From 765b017ac28ae5d39d475b59e67047cde212c817 Mon Sep 17 00:00:00 2001 From: pedro Date: Mon, 23 Sep 2024 13:09:48 -0300 Subject: [PATCH 01/22] good default django settings for deployments current docker deployment was not taking in account deployment with DOMAIN - domain is enforced, localhost by default - .env.example proposed (right now only with DOMAIN) --- .env.example | 1 + dhub/settings.py | 11 +++++++++-- docker-compose.yml | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..18f815d --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +DOMAIN=localhost diff --git a/dhub/settings.py b/dhub/settings.py index 3770353..1c6dce1 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -27,10 +27,17 @@ BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = "django-insecure-1p8rs@qf$$l^!vsbetagojw23kw@1ez(qi8^(s0t!wyh!l3" # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = config('DEBUG', default=False, cast=bool) -ALLOWED_HOSTS = config('ALLOWED_HOSTS', default='[]', cast=Csv()) +DOMAIN = config("DOMAIN") +assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" +# this var is very important, we print it +print("DOMAIN: " + DOMAIN) +ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) +assert DOMAIN in ALLOWED_HOSTS, "DOMAIN is not ALLOWED_HOST" + +CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) # Application definition diff --git a/docker-compose.yml b/docker-compose.yml index a106f6e..d0d6e44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: dockerfile: docker/devicehub-django.Dockerfile environment: - DEBUG=true - - ALLOWED_HOSTS=* + - DOMAIN=${DOMAIN:-localhost} volumes: - .:/opt/devicehub-django ports: From 665310651c3005ee00226c1be762fc66dd65f899 Mon Sep 17 00:00:00 2001 From: pedro Date: Mon, 23 Sep 2024 22:03:51 -0300 Subject: [PATCH 02/22] docker: update to latest python, do lxc trick lxc cannot build, do a trick --- docker/devicehub-django.Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/devicehub-django.Dockerfile b/docker/devicehub-django.Dockerfile index 2f734b4..fadc751 100644 --- a/docker/devicehub-django.Dockerfile +++ b/docker/devicehub-django.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.7-slim-bookworm +FROM python:3.11.10-slim-bookworm # last line is dependencies for weasyprint (for generating pdfs in lafede pilot) https://doc.courtbouillon.org/weasyprint/stable/first_steps.html#debian-11 RUN apt update && \ @@ -22,7 +22,8 @@ compile = no no-cache-dir = True END -RUN pip install --upgrade pip +# upgrade pip, which might fail on lxc, then remove the "corrupted file" +RUN python -m pip install --upgrade pip || (rm -rf /usr/local/lib/python3.11/site-packages/pip-*.dist-info && python -m pip install --upgrade pip) COPY ./requirements.txt /opt/devicehub-django RUN pip install -r requirements.txt From b024dd1a11e4445c2861f8ce59174d698cad2e87 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 24 Sep 2024 09:50:57 -0300 Subject: [PATCH 03/22] docker: add optional DEMO env var that includes the default snapshot import --- .env.example | 1 + docker-compose.yml | 1 + docker/devicehub-django.entrypoint.sh | 3 +++ 3 files changed, 5 insertions(+) diff --git a/.env.example b/.env.example index 18f815d..cf39121 100644 --- a/.env.example +++ b/.env.example @@ -1 +1,2 @@ DOMAIN=localhost +DEMO=false diff --git a/docker-compose.yml b/docker-compose.yml index d0d6e44..ee59640 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} + - DEMO=${DEMO:-n} volumes: - .:/opt/devicehub-django ports: diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 68e3760..a8c0351 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -24,6 +24,9 @@ deploy() { ./manage.py add_institution example-org # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user example-org user@example.org 1234 + if [ "${DEMO:-}" ]; then + ./manage.py up_snapshots example/snapshots/ user@example.org + fi fi } From bab540187c21cfd08949ae401ece19b4da9254b5 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 24 Sep 2024 09:52:18 -0300 Subject: [PATCH 04/22] docker: refactor init env vars --- docker/devicehub-django.entrypoint.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index a8c0351..920cdf0 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -21,24 +21,28 @@ deploy() { # inspired by https://medium.com/analytics-vidhya/django-with-docker-and-docker-compose-python-part-2-8415976470cc echo "INFO detected NEW deployment" ./manage.py migrate - ./manage.py add_institution example-org + INIT_ORG="${INIT_ORG:-example-org}" + INIT_USER="${INIT_USER:-user@example.org}" + INIT_PASSWD="${INIT_PASSWD:-1234}" + ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user example-org user@example.org 1234 + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" + if [ "${DEMO:-}" ]; then - ./manage.py up_snapshots example/snapshots/ user@example.org + ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi } runserver() { PORT="${PORT:-8000}" - if [ "${DEBUG:-}" = "true" ]; then + if [ "${DEBUG:-}" ]; then ./manage.py runserver 0.0.0.0:${PORT} else # TODO #./manage.py collectstatic true - if [ "${EXPERIMENTAL:-}" = "true" ]; then + if [ "${EXPERIMENTAL:-}" ]; then # TODO # reloading on source code changing is a debugging future, maybe better then use debug # src https://stackoverflow.com/questions/12773763/gunicorn-autoreload-on-source-change/24893069#24893069 From 54ef0bb41c54ed2baaeac68ffc940b7798996b63 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 25 Sep 2024 12:51:08 +0200 Subject: [PATCH 05/22] add parsing components with new workbench --- device/models.py | 7 + device/templates/details.html | 2 +- evidence/__init__.py | 25 + evidence/models.py | 31 +- evidence/parse.py | 2 +- evidence/parse_details.py | 492 ++++++++++++++++++++ evidence/unit_registry/base2.quantities.txt | 4 + evidence/unit_registry/quantities.txt | 9 + requirements.txt | 1 + user/views.py | 33 +- utils/constants.py | 8 + 11 files changed, 599 insertions(+), 15 deletions(-) create mode 100644 evidence/parse_details.py create mode 100644 evidence/unit_registry/base2.quantities.txt create mode 100644 evidence/unit_registry/quantities.txt diff --git a/device/models.py b/device/models.py index 6e0bc50..e2378c2 100644 --- a/device/models.py +++ b/device/models.py @@ -155,3 +155,10 @@ class Device: self.get_last_evidence() return self.last_evidence.get_model() + @property + def components(self): + if not self.last_evidence: + self.get_last_evidence() + return self.last_evidence.get_components() + + diff --git a/device/templates/details.html b/device/templates/details.html index 3881c1a..de4ae85 100644 --- a/device/templates/details.html +++ b/device/templates/details.html @@ -173,7 +173,7 @@
Components last evidence
- {% for c in object.last_evidence.doc.components %} + {% for c in object.components %}
{{ c.type }}
diff --git a/evidence/__init__.py b/evidence/__init__.py index e69de29..b76b6a9 100644 --- a/evidence/__init__.py +++ b/evidence/__init__.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from pint import UnitRegistry + +# Sets up the unit handling +unit_registry = Path(__file__).parent / 'unit_registry' + +unit = UnitRegistry() +unit.load_definitions(str(unit_registry / 'quantities.txt')) +TB = unit.TB +GB = unit.GB +MB = unit.MB +Mbs = unit.Mbit / unit.s +MBs = unit.MB / unit.s +Hz = unit.Hz +GHz = unit.GHz +MHz = unit.MHz +Inch = unit.inch +mAh = unit.hour * unit.mA +mV = unit.mV + +base2 = UnitRegistry() +base2.load_definitions(str(unit_registry / 'base2.quantities.txt')) + +GiB = base2.GiB diff --git a/evidence/models.py b/evidence/models.py index 30625aa..031641f 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -5,6 +5,7 @@ from django.db import models from utils.constants import STR_SM_SIZE, STR_EXTEND_SIZE, CHASSIS_DH from evidence.xapian import search +from evidence.parse_details import ParseSnapshot from user.models import Institution @@ -35,6 +36,8 @@ class Evidence: self.created = None self.dmi = None self.annotations = [] + self.components = [] + self.default = "n/a" self.get_owner() self.get_time() @@ -60,12 +63,11 @@ class Evidence: for xa in matches: self.doc = json.loads(xa.document.get_data()) - + if self.doc.get("software") == "EreuseWorkbench": dmidecode_raw = self.doc["data"]["dmidecode"] self.dmi = DMIParse(dmidecode_raw) - def get_time(self): if not self.doc: self.get_doc() @@ -74,38 +76,43 @@ class Evidence: if not self.created: self.created = self.annotations.last().created - def components(self): - return self.doc.get('components', []) + def get_components(self): + if self.doc.get("software") != "EreuseWorkbench": + return self.doc.get('components', []) + self.set_components() + return self.components def get_manufacturer(self): if self.doc.get("software") != "EreuseWorkbench": return self.doc['device']['manufacturer'] - + return self.dmi.manufacturer().strip() - + def get_model(self): if self.doc.get("software") != "EreuseWorkbench": return self.doc['device']['model'] - + return self.dmi.model().strip() def get_chassis(self): if self.doc.get("software") != "EreuseWorkbench": return self.doc['device']['model'] - - chassis = self.dmi.get("Chassis")[0].get("Type", '_virtual') + + chassis = self.dmi.get("Chassis")[0].get("Type", '_virtual') lower_type = chassis.lower() - + for k, v in CHASSIS_DH.items(): if lower_type in v: return k return "" - - @classmethod def get_all(cls, user): return Annotation.objects.filter( owner=user.institution, type=Annotation.Type.SYSTEM, ).order_by("-created").values_list("uuid", flat=True).distinct() + + def set_components(self): + snapshot = ParseSnapshot(self.doc).snapshot_json + self.components = snapshot['components'] diff --git a/evidence/parse.py b/evidence/parse.py index 2306ac4..6de5ca4 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -6,7 +6,7 @@ import hashlib from datetime import datetime from dmidecode import DMIParse from evidence.xapian import search, index -from evidence.models import Evidence, Annotation +from evidence.models import Annotation from utils.constants import ALGOS, CHASSIS_DH diff --git a/evidence/parse_details.py b/evidence/parse_details.py new file mode 100644 index 0000000..cddd293 --- /dev/null +++ b/evidence/parse_details.py @@ -0,0 +1,492 @@ +import json +import numpy as np + +from datetime import datetime +from dmidecode import DMIParse +from evidence import base2, unit +from utils.constants import CHASSIS_DH, DATASTORAGEINTERFACE + + +def get_lshw_child(child, nets, component): + if child.get('id') == component: + nets.append(child) + if child.get('children'): + [get_lshw_child(x, nets, component) for x in child['children']] + + +class ParseSnapshot: + def __init__(self, snapshot, default="n/a"): + self.default = default + self.dmidecode_raw = snapshot["data"].get("dmidecode", "{}") + self.smart_raw = snapshot["data"].get("disks", []) + self.hwinfo_raw = snapshot["data"].get("hwinfo", "") + self.lshw_raw = snapshot["data"].get("lshw", {}) or {} + self.lscpi_raw = snapshot["data"].get("lspci", "") + self.device = {"actions": []} + self.components = [] + self.monitors = [] + + self.dmi = DMIParse(self.dmidecode_raw) + self.smart = self.loads(self.smart_raw) + self.lshw = self.loads(self.lshw_raw) + self.hwinfo = self.parse_hwinfo() + + self.set_computer() + self.get_hwinfo_monitors() + self.set_components() + self.snapshot_json = { + "type": "Snapshot", + "device": self.device, + "software": snapshot["software"], + "components": self.components, + "uuid": snapshot['uuid'], + "version": snapshot['version'], + "endTime": snapshot["timestamp"], + "elapsed": 1, + } + + def set_computer(self): + self.device['manufacturer'] = self.dmi.manufacturer().strip() + self.device['model'] = self.dmi.model().strip() + self.device['serialNumber'] = self.dmi.serial_number() + self.device['type'] = self.get_type() + self.device['sku'] = self.get_sku() + self.device['version'] = self.get_version() + self.device['system_uuid'] = self.get_uuid() + self.device['family'] = self.get_family() + self.device['chassis'] = self.get_chassis_dh() + + def set_components(self): + self.get_cpu() + self.get_ram() + self.get_mother_board() + self.get_graphic() + self.get_data_storage() + self.get_display() + self.get_sound_card() + self.get_networks() + + def get_cpu(self): + for cpu in self.dmi.get('Processor'): + serial = cpu.get('Serial Number') + if serial == 'Not Specified' or not serial: + serial = cpu.get('ID').replace(' ', '') + self.components.append( + { + "actions": [], + "type": "Processor", + "speed": self.get_cpu_speed(cpu), + "cores": int(cpu.get('Core Count', 1)), + "model": cpu.get('Version'), + "threads": int(cpu.get('Thread Count', 1)), + "manufacturer": cpu.get('Manufacturer'), + "serialNumber": serial, + "generation": None, + "brand": cpu.get('Family'), + "address": self.get_cpu_address(cpu), + } + ) + + def get_ram(self): + for ram in self.dmi.get("Memory Device"): + if ram.get('size') == 'No Module Installed': + continue + if not ram.get("Speed"): + continue + + self.components.append( + { + "actions": [], + "type": "RamModule", + "size": self.get_ram_size(ram), + "speed": self.get_ram_speed(ram), + "manufacturer": ram.get("Manufacturer", self.default), + "serialNumber": ram.get("Serial Number", self.default), + "interface": ram.get("Type", "DDR"), + "format": ram.get("Form Factor", "DIMM"), + "model": ram.get("Part Number", self.default), + } + ) + + def get_mother_board(self): + for moder_board in self.dmi.get("Baseboard"): + self.components.append( + { + "actions": [], + "type": "Motherboard", + "version": moder_board.get("Version"), + "serialNumber": moder_board.get("Serial Number", "").strip(), + "manufacturer": moder_board.get("Manufacturer", "").strip(), + "biosDate": self.get_bios_date(), + "ramMaxSize": self.get_max_ram_size(), + "ramSlots": len(self.dmi.get("Memory Device")), + "slots": self.get_ram_slots(), + "model": moder_board.get("Product Name", "").strip(), + "firewire": self.get_firmware_num(), + "pcmcia": self.get_pcmcia_num(), + "serial": self.get_serial_num(), + "usb": self.get_usb_num(), + } + ) + + def get_graphic(self): + displays = [] + get_lshw_child(self.lshw, displays, 'display') + + for c in displays: + if not c['configuration'].get('driver', None): + continue + + self.components.append( + { + "actions": [], + "type": "GraphicCard", + "memory": self.get_memory_video(c), + "manufacturer": c.get("vendor", self.default), + "model": c.get("product", self.default), + "serialNumber": c.get("serial", self.default), + } + ) + + def get_memory_video(self, c): + # get info of lspci + # pci_id = c['businfo'].split('@')[1] + # lspci.get(pci_id) | grep size + # lspci -v -s 00:02.0 + return None + + def get_data_storage(self): + for sm in self.smart: + if sm.get('smartctl', {}).get('exit_status') == 1: + continue + model = sm.get('model_name') + manufacturer = None + if model and len(model.split(" ")) > 1: + mm = model.split(" ") + model = mm[-1] + manufacturer = " ".join(mm[:-1]) + + self.components.append( + { + "actions": self.sanitize(sm), + "type": self.get_data_storage_type(sm), + "model": model, + "manufacturer": manufacturer, + "serialNumber": sm.get('serial_number'), + "size": self.get_data_storage_size(sm), + "variant": sm.get("firmware_version"), + "interface": self.get_data_storage_interface(sm), + } + ) + + def sanitize(self, action): + return [] + + def get_networks(self): + networks = [] + get_lshw_child(self.lshw, networks, 'network') + + for c in networks: + capacity = c.get('capacity') + units = c.get('units') + speed = None + if capacity and units: + speed = unit.Quantity(capacity, units).to('Mbit/s').m + wireless = bool(c.get('configuration', {}).get('wireless', False)) + self.components.append( + { + "actions": [], + "type": "NetworkAdapter", + "model": c.get('product'), + "manufacturer": c.get('vendor'), + "serialNumber": c.get('serial'), + "speed": speed, + "variant": c.get('version', 1), + "wireless": wireless, + } + ) + + def get_sound_card(self): + multimedias = [] + get_lshw_child(self.lshw, multimedias, 'multimedia') + + for c in multimedias: + self.components.append( + { + "actions": [], + "type": "SoundCard", + "model": c.get('product'), + "manufacturer": c.get('vendor'), + "serialNumber": c.get('serial'), + } + ) + + def get_display(self): # noqa: C901 + TECHS = 'CRT', 'TFT', 'LED', 'PDP', 'LCD', 'OLED', 'AMOLED' + + for c in self.monitors: + resolution_width, resolution_height = (None,) * 2 + refresh, serial, model, manufacturer, size = (None,) * 5 + year, week, production_date = (None,) * 3 + + for x in c: + if "Vendor: " in x: + manufacturer = x.split('Vendor: ')[-1].strip() + if "Model: " in x: + model = x.split('Model: ')[-1].strip() + if "Serial ID: " in x: + serial = x.split('Serial ID: ')[-1].strip() + if " Resolution: " in x: + rs = x.split(' Resolution: ')[-1].strip() + if 'x' in rs: + resolution_width, resolution_height = [ + int(r) for r in rs.split('x') + ] + if "Frequencies: " in x: + try: + refresh = int(float(x.split(',')[-1].strip()[:-3])) + except Exception: + pass + if 'Year of Manufacture' in x: + year = x.split(': ')[1] + + if 'Week of Manufacture' in x: + week = x.split(': ')[1] + + if "Size: " in x: + size = self.get_size_monitor(x) + technology = next((t for t in TECHS if t in c[0]), None) + + if year and week: + d = '{} {} 0'.format(year, week) + production_date = datetime.strptime(d, '%Y %W %w').isoformat() + + self.components.append( + { + "actions": [], + "type": "Display", + "model": model, + "manufacturer": manufacturer, + "serialNumber": serial, + 'size': size, + 'resolutionWidth': resolution_width, + 'resolutionHeight': resolution_height, + "productionDate": production_date, + 'technology': technology, + 'refreshRate': refresh, + } + ) + + def get_hwinfo_monitors(self): + for c in self.hwinfo: + monitor = None + external = None + for x in c: + if 'Hardware Class: monitor' in x: + monitor = c + if 'Driver Info' in x: + external = c + + if monitor and not external: + self.monitors.append(c) + + def get_size_monitor(self, x): + i = 1 / 25.4 + t = x.split('Size: ')[-1].strip() + tt = t.split('mm') + if not tt: + return 0 + sizes = tt[0].strip() + if 'x' not in sizes: + return 0 + w, h = [int(x) for x in sizes.split('x')] + return np.sqrt(w**2 + h**2) * i + + def get_cpu_address(self, cpu): + default = 64 + for ch in self.lshw.get('children', []): + for c in ch.get('children', []): + if c['class'] == 'processor': + return c.get('width', default) + return default + + def get_usb_num(self): + return len( + [ + u + for u in self.dmi.get("Port Connector") + if "USB" in u.get("Port Type", "").upper() + ] + ) + + def get_serial_num(self): + return len( + [ + u + for u in self.dmi.get("Port Connector") + if "SERIAL" in u.get("Port Type", "").upper() + ] + ) + + def get_firmware_num(self): + return len( + [ + u + for u in self.dmi.get("Port Connector") + if "FIRMWARE" in u.get("Port Type", "").upper() + ] + ) + + def get_pcmcia_num(self): + return len( + [ + u + for u in self.dmi.get("Port Connector") + if "PCMCIA" in u.get("Port Type", "").upper() + ] + ) + + def get_bios_date(self): + return self.dmi.get("BIOS")[0].get("Release Date", self.default) + + def get_firmware(self): + return self.dmi.get("BIOS")[0].get("Firmware Revision", '1') + + def get_max_ram_size(self): + size = 0 + for slot in self.dmi.get("Physical Memory Array"): + capacity = slot.get("Maximum Capacity", '0').split(" ")[0] + size += int(capacity) + + return size + + def get_ram_slots(self): + slots = 0 + for x in self.dmi.get("Physical Memory Array"): + slots += int(x.get("Number Of Devices", 0)) + return slots + + def get_ram_size(self, ram): + try: + memory = ram.get("Size", "0") + memory = memory.split(' ') + if len(memory) > 1: + size = int(memory[0]) + units = memory[1] + return base2.Quantity(size, units).to('MiB').m + return int(size.split(" ")[0]) + except Exception as err: + logger.error("get_ram_size error: {}".format(err)) + return 0 + + def get_ram_speed(self, ram): + size = ram.get("Speed", "0") + return int(size.split(" ")[0]) + + def get_cpu_speed(self, cpu): + speed = cpu.get('Max Speed', "0") + return float(speed.split(" ")[0]) / 1024 + + def get_sku(self): + return self.dmi.get("System")[0].get("SKU Number", self.default).strip() + + def get_version(self): + return self.dmi.get("System")[0].get("Version", self.default).strip() + + def get_uuid(self): + return self.dmi.get("System")[0].get("UUID", '').strip() + + def get_family(self): + return self.dmi.get("System")[0].get("Family", '') + + def get_chassis(self): + return self.dmi.get("Chassis")[0].get("Type", '_virtual') + + def get_type(self): + chassis_type = self.get_chassis() + return self.translation_to_devicehub(chassis_type) + + def translation_to_devicehub(self, original_type): + lower_type = original_type.lower() + CHASSIS_TYPE = { + 'Desktop': [ + 'desktop', + 'low-profile', + 'tower', + 'docking', + 'all-in-one', + 'pizzabox', + 'mini-tower', + 'space-saving', + 'lunchbox', + 'mini', + 'stick', + ], + 'Laptop': [ + 'portable', + 'laptop', + 'convertible', + 'tablet', + 'detachable', + 'notebook', + 'handheld', + 'sub-notebook', + ], + 'Server': ['server'], + 'Computer': ['_virtual'], + } + for k, v in CHASSIS_TYPE.items(): + if lower_type in v: + return k + return self.default + + def get_chassis_dh(self): + chassis = self.get_chassis() + lower_type = chassis.lower() + for k, v in CHASSIS_DH.items(): + if lower_type in v: + return k + return self.default + + def get_data_storage_type(self, x): + # TODO @cayop add more SSDS types + SSDS = ["nvme"] + SSD = 'SolidStateDrive' + HDD = 'HardDrive' + type_dev = x.get('device', {}).get('type') + trim = x.get('trim', {}).get("supported") in [True, "true"] + return SSD if type_dev in SSDS or trim else HDD + + def get_data_storage_interface(self, x): + interface = x.get('device', {}).get('protocol', 'ATA') + if interface.upper() in DATASTORAGEINTERFACE: + return interface.upper() + + txt = "Sid: {}, interface {} is not in DataStorageInterface Enum".format( + self.sid, interface + ) + self.errors("{}".format(err)) + + def get_data_storage_size(self, x): + total_capacity = x.get('user_capacity', {}).get('bytes') + if not total_capacity: + return 1 + # convert bytes to Mb + return total_capacity / 1024**2 + + def parse_hwinfo(self): + hw_blocks = self.hwinfo_raw.split("\n\n") + return [x.split("\n") for x in hw_blocks] + + def loads(self, x): + if isinstance(x, str): + return json.loads(x) + return x + + def errors(self, txt=None): + if not txt: + return self._errors + + logger.error(txt) + self._errors.append(txt) + diff --git a/evidence/unit_registry/base2.quantities.txt b/evidence/unit_registry/base2.quantities.txt new file mode 100644 index 0000000..2c724a2 --- /dev/null +++ b/evidence/unit_registry/base2.quantities.txt @@ -0,0 +1,4 @@ +K = KiB = k = kb = KB +M = MiB = m = mb = MB +G = GiB = g = gb = GB +T = TiB = t = tb = TB diff --git a/evidence/unit_registry/quantities.txt b/evidence/unit_registry/quantities.txt new file mode 100644 index 0000000..d658ab2 --- /dev/null +++ b/evidence/unit_registry/quantities.txt @@ -0,0 +1,9 @@ +HZ = hertz = hz +KHZ = kilohertz = khz +MHZ = megahertz = mhz +GHZ = gigahertz = ghz +B = byte = b = UNIT = unit +KB = kilobyte = kb = K = k +MB = megabyte = mb = M = m +GB = gigabyte = gb = G = g +T = terabyte = tb = T = t diff --git a/requirements.txt b/requirements.txt index 217d120..2d2537c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ pandas==2.2.2 xlrd==2.0.1 odfpy==1.4.1 pytz==2024.2 +Pint==0.24.3 diff --git a/user/views.py b/user/views.py index e7f3289..bb6bdd2 100644 --- a/user/views.py +++ b/user/views.py @@ -1,6 +1,37 @@ from django.shortcuts import render from django.utils.translation import gettext_lazy as _ -from dashboard.mixins import InventaryMixin, DetailsMixin +from dashboard.mixins import DashboardView +class ProfileView(DashboardView): + template_name = "profile.html" + subtitle = _('My personal data') + icon = 'bi bi-person-gear' + fields = ('first_name', 'last_name', 'email') + success_url = reverse_lazy('idhub:user_profile') + model = User + + def get_queryset(self, **kwargs): + queryset = Membership.objects.select_related('user').filter( + user=self.request.user) + + return queryset + + def get_object(self): + return self.request.user + + def get_form(self): + form = super().get_form() + return form + + def form_valid(self, form): + return super().form_valid(form) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context.update({ + 'lang': self.request.LANGUAGE_CODE, + }) + return context + diff --git a/utils/constants.py b/utils/constants.py index e481d6c..bc43922 100644 --- a/utils/constants.py +++ b/utils/constants.py @@ -38,3 +38,11 @@ CHASSIS_DH = { 'Tablet': {'tablet'}, 'Virtual': {'_virtual'}, } + + +DATASTORAGEINTERFACE = [ + 'ATA', + 'USB', + 'PCI', + 'NVME', +] From 59f6ac705ca752684fa9dc93292e0181d19fae02 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 25 Sep 2024 19:04:29 +0200 Subject: [PATCH 06/22] fix components --- device/models.py | 5 +-- device/templates/details.html | 9 ++++-- device/views.py | 5 ++- evidence/parse.py | 6 +++- evidence/parse_details.py | 57 ++++++++++++++++++----------------- 5 files changed, 47 insertions(+), 35 deletions(-) diff --git a/device/models.py b/device/models.py index e2378c2..c9e04ec 100644 --- a/device/models.py +++ b/device/models.py @@ -102,8 +102,9 @@ class Device: def get_last_evidence(self): annotations = self.get_annotations() - if annotations: - annotation = annotations.first() + if not annotations.count(): + return + annotation = annotations.first() self.last_evidence = Evidence(annotation.uuid) def last_uuid(self): diff --git a/device/templates/details.html b/device/templates/details.html index de4ae85..b757b1f 100644 --- a/device/templates/details.html +++ b/device/templates/details.html @@ -180,9 +180,12 @@ {{ evidence.created }}

- {{ c.manufacturer }}
- {{ c.model }}
- {{ c.serialNumber }}
+ {% for k, v in c.items %} + {% if k not in "actions,type" %} + {{ k }}: {{ v }}
+ {% endif %} + {% endfor %} +

diff --git a/device/views.py b/device/views.py index 0fd9365..8d4c024 100644 --- a/device/views.py +++ b/device/views.py @@ -1,7 +1,7 @@ import json from django.urls import reverse_lazy -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, Http404 from django.utils.translation import gettext_lazy as _ from django.views.generic.edit import ( CreateView, @@ -87,6 +87,9 @@ class DetailsView(DashboardView, TemplateView): def get(self, request, *args, **kwargs): self.pk = kwargs['pk'] self.object = Device(id=self.pk) + if not self.object.last_evidence: + raise Http404 + return super().get(request, *args, **kwargs) def get_context_data(self, **kwargs): diff --git a/evidence/parse.py b/evidence/parse.py index 6de5ca4..d07f313 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -19,8 +19,12 @@ def get_network_cards(child, nets): def get_mac(lshw): nets = [] + try: + get_network_cards(json.loads(lshw), nets) + except Exception as ss: + print("WARNING!! {}".format(ss)) + return - get_network_cards(json.loads(lshw), nets) nets_sorted = sorted(nets, key=lambda x: x['businfo']) # This funcion get the network card integrated in motherboard # integrate = [x for x in nets if "pci@0000:00:" in x.get('businfo', '')] diff --git a/evidence/parse_details.py b/evidence/parse_details.py index cddd293..8465403 100644 --- a/evidence/parse_details.py +++ b/evidence/parse_details.py @@ -3,7 +3,6 @@ import numpy as np from datetime import datetime from dmidecode import DMIParse -from evidence import base2, unit from utils.constants import CHASSIS_DH, DATASTORAGEINTERFACE @@ -81,9 +80,9 @@ class ParseSnapshot: "threads": int(cpu.get('Thread Count', 1)), "manufacturer": cpu.get('Manufacturer'), "serialNumber": serial, - "generation": None, "brand": cpu.get('Family'), "address": self.get_cpu_address(cpu), + "bogomips": self.get_bogomips(), } ) @@ -182,16 +181,26 @@ class ParseSnapshot: def sanitize(self, action): return [] + def get_bogomips(self): + if not self.hwinfo: + return self.default + + bogomips = 0 + for row in self.hwinfo: + for cel in row: + if 'BogoMips' in cel: + try: + bogomips += float(cel.split(":")[-1]) + except: + pass + return bogomips + def get_networks(self): networks = [] get_lshw_child(self.lshw, networks, 'network') for c in networks: capacity = c.get('capacity') - units = c.get('units') - speed = None - if capacity and units: - speed = unit.Quantity(capacity, units).to('Mbit/s').m wireless = bool(c.get('configuration', {}).get('wireless', False)) self.components.append( { @@ -200,9 +209,10 @@ class ParseSnapshot: "model": c.get('product'), "manufacturer": c.get('vendor'), "serialNumber": c.get('serial'), - "speed": speed, + "speed": capacity, "variant": c.get('version', 1), - "wireless": wireless, + "wireless": wireless or False, + "integrated": "PCI:0000:00" in c.get("businfo", ""), } ) @@ -300,7 +310,7 @@ class ParseSnapshot: if 'x' not in sizes: return 0 w, h = [int(x) for x in sizes.split('x')] - return np.sqrt(w**2 + h**2) * i + return "{:.2f}".format(np.sqrt(w**2 + h**2) * i) def get_cpu_address(self, cpu): default = 64 @@ -367,25 +377,16 @@ class ParseSnapshot: return slots def get_ram_size(self, ram): - try: - memory = ram.get("Size", "0") - memory = memory.split(' ') - if len(memory) > 1: - size = int(memory[0]) - units = memory[1] - return base2.Quantity(size, units).to('MiB').m - return int(size.split(" ")[0]) - except Exception as err: - logger.error("get_ram_size error: {}".format(err)) - return 0 + memory = ram.get("Size", "0") + return memory def get_ram_speed(self, ram): size = ram.get("Speed", "0") - return int(size.split(" ")[0]) + return size def get_cpu_speed(self, cpu): speed = cpu.get('Max Speed', "0") - return float(speed.split(" ")[0]) / 1024 + return speed def get_sku(self): return self.dmi.get("System")[0].get("SKU Number", self.default).strip() @@ -468,11 +469,7 @@ class ParseSnapshot: self.errors("{}".format(err)) def get_data_storage_size(self, x): - total_capacity = x.get('user_capacity', {}).get('bytes') - if not total_capacity: - return 1 - # convert bytes to Mb - return total_capacity / 1024**2 + return x.get('user_capacity', {}).get('bytes') def parse_hwinfo(self): hw_blocks = self.hwinfo_raw.split("\n\n") @@ -480,7 +477,11 @@ class ParseSnapshot: def loads(self, x): if isinstance(x, str): - return json.loads(x) + try: + return json.loads(x) + except Exception as ss: + print("WARNING!! {}".format(ss)) + return {} return x def errors(self, txt=None): From b5b90c50f6ac5ccd79ab173e262f3e5e47b7d061 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 25 Sep 2024 22:32:26 -0300 Subject: [PATCH 07/22] placeholder form: route to unassigned devices An empty form can be filled, and if you press multiple times save, it saves multiple devices/placeholders edit device goes to unassigned devices, so probably is the same for new device --- device/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/device/views.py b/device/views.py index 8d4c024..60412de 100644 --- a/device/views.py +++ b/device/views.py @@ -21,7 +21,7 @@ class NewDeviceView(DashboardView, FormView): template_name = "new_device.html" title = _("New Device") breadcrumb = "Device / New Device" - success_url = reverse_lazy('device:add') + success_url = reverse_lazy('dashboard:unassigned_devices') form_class = DeviceFormSet def form_valid(self, form): From 5b84d815903238fb2db4cae221390d32345633ec Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 25 Sep 2024 22:48:03 -0300 Subject: [PATCH 08/22] new_device: UX: clarify add component details --- device/templates/new_device.html | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/device/templates/new_device.html b/device/templates/new_device.html index bdb2911..5f76e69 100644 --- a/device/templates/new_device.html +++ b/device/templates/new_device.html @@ -40,15 +40,6 @@ {% endif %} {{ form.management_form }}
-
-
-
- - - {% trans 'Add' %} - -
-
{% bootstrap_field form.0.type %} @@ -64,6 +55,17 @@ {% bootstrap_field form.0.customer_id %}
+
+
+ {% trans 'Component details' %} +
+
+ + + {% trans 'Add component' %} + +
+
{% for f in form %}
From e3b0d70f042d74732703c41c80b9cb0cada8e88a Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 25 Sep 2024 23:10:41 -0300 Subject: [PATCH 09/22] fix typo: customer_id -> custom_id --- device/forms.py | 6 +++--- device/templates/new_device.html | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/device/forms.py b/device/forms.py index b6c1d21..21c9a9a 100644 --- a/device/forms.py +++ b/device/forms.py @@ -23,7 +23,7 @@ DEVICE_TYPES = [ class DeviceForm(forms.Form): type = forms.ChoiceField(choices = DEVICE_TYPES, required=False) amount = forms.IntegerField(required=False, initial=1) - customer_id = forms.CharField(required=False) + custom_id = forms.CharField(required=False) name = forms.CharField(required=False) value = forms.CharField(required=False) @@ -49,8 +49,8 @@ class BaseDeviceFormSet(forms.BaseFormSet): row["amount"] = d["amount"] if d.get("name"): row[d["name"]] = d.get("value", '') - if d.get("customer_id"): - row['CUSTOMER_ID']= d["customer_id"] + if d.get("custom_id"): + row['CUSTOM_ID']= d["custom_id"] doc = create_doc(row) if not commit: diff --git a/device/templates/new_device.html b/device/templates/new_device.html index 5f76e69..320790d 100644 --- a/device/templates/new_device.html +++ b/device/templates/new_device.html @@ -52,7 +52,7 @@
- {% bootstrap_field form.0.customer_id %} + {% bootstrap_field form.0.custom_id %}
From 58c0e9059a415dbf9e81d70320b19e635d74c96e Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 26 Sep 2024 00:10:48 -0300 Subject: [PATCH 10/22] device: avoid hids duplication --- device/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device/models.py b/device/models.py index c9e04ec..9972ef5 100644 --- a/device/models.py +++ b/device/models.py @@ -89,10 +89,10 @@ class Device: def get_hids(self): annotations = self.get_annotations() - self.hids = annotations.filter( + self.hids = list(set(annotations.filter( type=Annotation.Type.SYSTEM, key__in=ALGOS.keys(), - ).values_list("value", flat=True) + ).values_list("value", flat=True))) def get_evidences(self): if not self.uuids: From e2f5358d830943725bae3109f16011583a053974 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 26 Sep 2024 00:22:21 -0300 Subject: [PATCH 11/22] add shortid and refactor unassigned devices table --- dashboard/templates/unassigned_devices.html | 25 +++++++++++++++++++-- device/models.py | 1 + device/templates/details.html | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/dashboard/templates/unassigned_devices.html b/dashboard/templates/unassigned_devices.html index c21a2be..8d894ed 100644 --- a/dashboard/templates/unassigned_devices.html +++ b/dashboard/templates/unassigned_devices.html @@ -33,7 +33,19 @@ - Title + select + + + shortid + + + type + + + manufacturer + + + model @@ -45,9 +57,18 @@ - {{ dev.type }} {{ dev.manufacturer }} {{ dev.model }} + {{ dev.shortid }} + + {{ dev.type }} + + + {{ dev.manufacturer }} + + + {{ dev.model }} + {% endfor %} diff --git a/device/models.py b/device/models.py index c9e04ec..1bf4bcc 100644 --- a/device/models.py +++ b/device/models.py @@ -27,6 +27,7 @@ class Device: # the id is the chid of the device self.id = kwargs["id"] self.pk = self.id + self.shortid = self.pk[:6] self.algorithm = None self.owner = None self.annotations = [] diff --git a/device/templates/details.html b/device/templates/details.html index b757b1f..37f7487 100644 --- a/device/templates/details.html +++ b/device/templates/details.html @@ -4,7 +4,7 @@ {% block content %}
-

{{ object.id }}

+

{{ object.shortid }}

From 1a28e071e937d4aba40bd86a5d048d79ba93c136 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 26 Sep 2024 10:10:15 +0200 Subject: [PATCH 12/22] fix bug remove usb nets iface --- evidence/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evidence/parse.py b/evidence/parse.py index d07f313..1ec972c 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -11,7 +11,7 @@ from utils.constants import ALGOS, CHASSIS_DH def get_network_cards(child, nets): - if child['id'] == 'network': + if child['id'] == 'network' and "PCI:" in child.get("businfo"): nets.append(child) if child.get('children'): [get_network_cards(x, nets) for x in child['children']] From dffe663e0516347f5bd82891d06de44643b3a3f1 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 26 Sep 2024 12:04:13 +0200 Subject: [PATCH 13/22] remove units and quantities addapted --- evidence/__init__.py | 25 --------------------- evidence/unit_registry/base2.quantities.txt | 4 ---- evidence/unit_registry/quantities.txt | 9 -------- requirements.txt | 2 +- 4 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 evidence/unit_registry/base2.quantities.txt delete mode 100644 evidence/unit_registry/quantities.txt diff --git a/evidence/__init__.py b/evidence/__init__.py index b76b6a9..e69de29 100644 --- a/evidence/__init__.py +++ b/evidence/__init__.py @@ -1,25 +0,0 @@ -from pathlib import Path - -from pint import UnitRegistry - -# Sets up the unit handling -unit_registry = Path(__file__).parent / 'unit_registry' - -unit = UnitRegistry() -unit.load_definitions(str(unit_registry / 'quantities.txt')) -TB = unit.TB -GB = unit.GB -MB = unit.MB -Mbs = unit.Mbit / unit.s -MBs = unit.MB / unit.s -Hz = unit.Hz -GHz = unit.GHz -MHz = unit.MHz -Inch = unit.inch -mAh = unit.hour * unit.mA -mV = unit.mV - -base2 = UnitRegistry() -base2.load_definitions(str(unit_registry / 'base2.quantities.txt')) - -GiB = base2.GiB diff --git a/evidence/unit_registry/base2.quantities.txt b/evidence/unit_registry/base2.quantities.txt deleted file mode 100644 index 2c724a2..0000000 --- a/evidence/unit_registry/base2.quantities.txt +++ /dev/null @@ -1,4 +0,0 @@ -K = KiB = k = kb = KB -M = MiB = m = mb = MB -G = GiB = g = gb = GB -T = TiB = t = tb = TB diff --git a/evidence/unit_registry/quantities.txt b/evidence/unit_registry/quantities.txt deleted file mode 100644 index d658ab2..0000000 --- a/evidence/unit_registry/quantities.txt +++ /dev/null @@ -1,9 +0,0 @@ -HZ = hertz = hz -KHZ = kilohertz = khz -MHZ = megahertz = mhz -GHZ = gigahertz = ghz -B = byte = b = UNIT = unit -KB = kilobyte = kb = K = k -MB = megabyte = mb = M = m -GB = gigabyte = gb = G = g -T = terabyte = tb = T = t diff --git a/requirements.txt b/requirements.txt index 2d2537c..1576d4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ pandas==2.2.2 xlrd==2.0.1 odfpy==1.4.1 pytz==2024.2 -Pint==0.24.3 + From f39cdefc75b69e176509f099f09bf690d0345281 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 26 Sep 2024 12:09:45 +0200 Subject: [PATCH 14/22] remove trash --- dashboard/templates/dashboard.html-bk | 213 -------------------------- 1 file changed, 213 deletions(-) delete mode 100644 dashboard/templates/dashboard.html-bk diff --git a/dashboard/templates/dashboard.html-bk b/dashboard/templates/dashboard.html-bk deleted file mode 100644 index 47289f0..0000000 --- a/dashboard/templates/dashboard.html-bk +++ /dev/null @@ -1,213 +0,0 @@ -{% load i18n static %} - - - - - {% block head %} - {% block meta %} - - - - - - {% endblock %} - {% block title %}{% if title %}{{ title }} – {% endif %}DeviceHub{% endblock %} - - - {% block style %} - - - - - - - - - - {% endblock %} - {% endblock %} - - -
- DEVICE HUB -
- PANGEA -
-
-
- - {{ user.email }} - - - -
-
-
- -
-
- - -
- {% block messages %} - {% for message in messages %} - - {% endfor %} - {% endblock messages %} -
-

{{ title }}

-
-
-
- - {% block content %} - {% endblock content %} - -
-
-
- - -
-
- {{ commit_id }} -
-
- - {% block script %} - - - - {% block extrascript %}{% endblock %} - {% endblock %} - - From e96afda483bca4292fb066e07f194951bedc64b2 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 26 Sep 2024 13:09:55 +0200 Subject: [PATCH 15/22] add details in dashboard for placeholder --- device/models.py | 11 ++++++----- evidence/models.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/device/models.py b/device/models.py index 683da53..6ffae4a 100644 --- a/device/models.py +++ b/device/models.py @@ -131,14 +131,14 @@ class Device: def is_websnapshot(self): if not self.last_evidence: self.get_last_evidence() - return self.last_evidence.doc['type'] == "WebSnapshot" - + return self.last_evidence.doc['type'] == "WebSnapshot" + @property def last_user_evidence(self): if not self.last_evidence: self.get_last_evidence() return self.last_evidence.doc['kv'].items() - + @property def manufacturer(self): if not self.last_evidence: @@ -147,6 +147,9 @@ class Device: @property def type(self): + if self.last_evidence.doc['type'] == "WebSnapshot": + return self.last_evidence.doc.get("device", {}).get("type", "") + if not self.last_evidence: self.get_last_evidence() return self.last_evidence.get_chassis() @@ -162,5 +165,3 @@ class Device: if not self.last_evidence: self.get_last_evidence() return self.last_evidence.get_components() - - diff --git a/evidence/models.py b/evidence/models.py index 031641f..2f0c703 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -83,12 +83,24 @@ class Evidence: return self.components def get_manufacturer(self): + if self.doc.get("type") == "WebSnapshot": + kv = self.doc.get('kv', {}) + if len(kv) < 1: + return "" + return list(self.doc.get('kv').values())[0] + if self.doc.get("software") != "EreuseWorkbench": return self.doc['device']['manufacturer'] return self.dmi.manufacturer().strip() def get_model(self): + if self.doc.get("type") == "WebSnapshot": + kv = self.doc.get('kv', {}) + if len(kv) < 2: + return "" + return list(self.doc.get('kv').values())[1] + if self.doc.get("software") != "EreuseWorkbench": return self.doc['device']['model'] From b47b5d3b4b8f471ef84ba37dda2c57a2fb56b01e Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 26 Sep 2024 20:06:37 -0300 Subject: [PATCH 16/22] docker-reset: add detach arg for demo deployment --- docker-reset.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-reset.sh b/docker-reset.sh index 9264b93..77d63b1 100755 --- a/docker-reset.sh +++ b/docker-reset.sh @@ -9,11 +9,14 @@ set -u set -x main() { + if [ "${DETACH:-}" ]; then + detach_arg='-d' + fi # remove old database sudo rm -vf db/* docker compose down docker compose build - docker compose up + docker compose up ${detach_arg:-} } main "${@}" From 94470cbdd1791e50d3b790daa5d6b822b2f46f00 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 26 Sep 2024 20:35:41 -0300 Subject: [PATCH 17/22] add modelo datos --- docs/es/modelo-datos.md | 80 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 docs/es/modelo-datos.md diff --git a/docs/es/modelo-datos.md b/docs/es/modelo-datos.md new file mode 100644 index 0000000..bb2cb22 --- /dev/null +++ b/docs/es/modelo-datos.md @@ -0,0 +1,80 @@ +Modelo de datos *abstracto* de devicehub que ayuda a tener una idea de cómo funciona + +Recordad que por ser este un proyecto de django, se puede obtener de forma automatizada un diagrama de datos con el comando `graph_models` (más adelante vemos de documentar mejor cómo generarlo) + +```mermaid +erDiagram + + %% los snapshots/placeholders son ficheros de FS inmutables, se insertan en xapian + %% y via su uuid se anotan + %% placeholders también se pueden firmar (como un spnashot, otra fuente) + EVIDENCE { + json obj "its uuid is the PK" + } + + USER { + int id PK + string personal-data-etc + } + + %% includes the relevant CHID with algorithm for the device build + EVIDENCE_ANNOTATION { + int id PK + uuid uuid "ref evidence (snapshot,placeholder)" + string key + string value + int type "0: sys_deviceid, 1: usr_deviceid, 2: user" + ts created + int owner FK + } + + ALGORITHM { + string algorithm + } + + %% todas las anotaciones que tienen CHID + %% y su key es un algoritmo de los que tenemos + + %% un device es una evaluación + + DEVICE { + string CHID + } + + DEVICE_ANNOTATION { + string CHID FK + string key + string value + uuid uuid "from last snapshot" + } + + LOT { + int id PK + string name + string code "id alt legacy" + string description + bool closed + int owner FK + ts created + ts updated + + } + + LOT_ANNOTATION { + string id FK + string key + string value + } + + SNAPSHOT ||--|| EVIDENCE: "via workbench" + PLACEHOLDER ||--|| EVIDENCE: "via webform" + + EVIDENCE ||--|{ EVIDENCE_ANNOTATION: "are interpreted" + USER ||--|{ EVIDENCE_ANNOTATION: "manually entered" + ALGORITHM ||--|{ EVIDENCE_ANNOTATION: "automatically entered" + EVIDENCE_ANNOTATION }|--|{ DEVICE: "aggregates" + DEVICE }|--|{ LOT: "aggregates" + + DEVICE ||--|| DEVICE_ANNOTATION: "enriches data" + LOT ||--|| LOT_ANNOTATION: "enriches data" +``` From b8ab991a4f3287b0e9b2e2a2567e063d1444501e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 27 Sep 2024 09:48:38 +0200 Subject: [PATCH 18/22] . --- evidence/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/evidence/views.py b/evidence/views.py index d98bddb..4535012 100644 --- a/evidence/views.py +++ b/evidence/views.py @@ -8,6 +8,7 @@ from django.views.generic.edit import ( FormView, ) + from dashboard.mixins import DashboardView, Http403 from evidence.models import Evidence from evidence.forms import UploadForm, UserTagForm, ImportForm From 9dfaf0f60b0052533f18d88c8243cbb307d39e37 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 27 Sep 2024 14:23:25 +0200 Subject: [PATCH 19/22] token api activate --- api/views.py | 14 +++++++------- user/management/commands/add_user.py | 6 ++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/api/views.py b/api/views.py index fbccb79..f852c05 100644 --- a/api/views.py +++ b/api/views.py @@ -29,14 +29,14 @@ def NewSnapshot(request): return JsonResponse({'error': 'Invalid request method'}, status=400) # Authentication - # auth_header = request.headers.get('Authorization') - # if not auth_header or not auth_header.startswith('Bearer '): - # return JsonResponse({'error': 'Invalid or missing token'}, status=401) + auth_header = request.headers.get('Authorization') + if not auth_header or not auth_header.startswith('Bearer '): + return JsonResponse({'error': 'Invalid or missing token'}, status=401) - # token = auth_header.split(' ')[1] - # tk = Token.objects.filter(token=token).first() - # if not tk: - # return JsonResponse({'error': 'Invalid or missing token'}, status=401) + token = auth_header.split(' ')[1] + tk = Token.objects.filter(token=token).first() + if not tk: + return JsonResponse({'error': 'Invalid or missing token'}, status=401) # Validation snapshot try: diff --git a/user/management/commands/add_user.py b/user/management/commands/add_user.py index 499adfa..cc40ac1 100644 --- a/user/management/commands/add_user.py +++ b/user/management/commands/add_user.py @@ -1,6 +1,9 @@ +from uuid import uuid4 + from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model from user.models import Institution +from api.models import Token User = get_user_model() @@ -28,3 +31,6 @@ class Command(BaseCommand): ) self.u.set_password(self.password) self.u.save() + token = uuid4() + Token.objects.create(token=token, owner=self.u) + print(f"TOKEN: {token}") From 665fbf852aab412ff14b0f3a72bfa162b5284510 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 27 Sep 2024 14:25:00 +0200 Subject: [PATCH 20/22] fix link snapshot with user --- api/views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index f852c05..4013012 100644 --- a/api/views.py +++ b/api/views.py @@ -60,9 +60,7 @@ def NewSnapshot(request): # save_in_disk(data, tk.user) try: - # Build(data, tk.user) - user = User.objects.get(email="user@example.org") - Build(data, user) + Build(data, tk.user) except Exception: return JsonResponse({'status': 'fail'}, status=200) From 029640347d3af859f1c621b7bd5d3644fd05a618 Mon Sep 17 00:00:00 2001 From: pedro Date: Sat, 28 Sep 2024 12:11:20 -0300 Subject: [PATCH 21/22] docker-reset.sh: arg to make it easier to deploy --- docker-reset.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-reset.sh b/docker-reset.sh index 77d63b1..cdbb971 100755 --- a/docker-reset.sh +++ b/docker-reset.sh @@ -14,7 +14,7 @@ main() { fi # remove old database sudo rm -vf db/* - docker compose down + docker compose down -v docker compose build docker compose up ${detach_arg:-} } From dcf2e43d579f767f88949f436b27a7a39fe87ea7 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 14 Oct 2024 11:34:52 +0200 Subject: [PATCH 22/22] we don't need for the moment edit profile of user --- user/views.py | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/user/views.py b/user/views.py index d64d2e9..23ebb27 100644 --- a/user/views.py +++ b/user/views.py @@ -1,5 +1,5 @@ from decouple import config -from django.urls import reverse +from django.urls import reverse, reverse_lazy from django.http import HttpResponse from django.shortcuts import render from django.utils.translation import gettext_lazy as _ @@ -48,34 +48,3 @@ class SettingsView(DashboardView, FormView): kwargs['tokens'] = tokens return kwargs -class ProfileView(DashboardView): - template_name = "profile.html" - subtitle = _('My personal data') - icon = 'bi bi-person-gear' - fields = ('first_name', 'last_name', 'email') - success_url = reverse_lazy('idhub:user_profile') - model = User - - def get_queryset(self, **kwargs): - queryset = Membership.objects.select_related('user').filter( - user=self.request.user) - - return queryset - - def get_object(self): - return self.request.user - - def get_form(self): - form = super().get_form() - return form - - def form_valid(self, form): - return super().form_valid(form) - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context.update({ - 'lang': self.request.LANGUAGE_CODE, - }) - return context -