Add reindex from snapshots files

This commit is contained in:
Cayo Puigdefabregas 2024-10-25 17:36:13 +02:00
parent 1a2f1249ff
commit fcb07a7337
6 changed files with 137 additions and 20 deletions

View File

@ -66,7 +66,7 @@ class SearchView(InventaryMixin):
limit limit
) )
if not matches.size(): if not matches or not matches.size():
return self.search_hids(query, offset, limit) return self.search_hids(query, offset, limit)
devices = [] devices = []

View File

@ -0,0 +1,86 @@
import os
import json
import logging
from django.core.management.base import BaseCommand
from django.conf import settings
from utils.device import create_annotation, create_doc, create_index
from user.models import Institution
from evidence.parse import Build
logger = logging.getLogger('django')
class Command(BaseCommand):
help = "Reindex snapshots"
snapshots = []
EVIDENCES = settings.EVIDENCES_DIR
def handle(self, *args, **kwargs):
if os.path.isdir(self.EVIDENCES):
self.read_files(self.EVIDENCES)
self.parsing()
def read_files(self, directory):
for filename in os.listdir(directory):
filepath = os.path.join(directory, filename)
if not os.path.isdir(filepath):
continue
institution = Institution.objects.filter(name=filename).first()
if not institution:
continue
user = institution.user_set.filter(is_admin=True).first()
if not user:
txt = "Error No there are Admins for the institution: {}".format(
institution.name
)
logger.exception(txt)
continue
snapshots_path = os.path.join(filepath, "snapshots")
placeholders_path = os.path.join(filepath, "placeholders")
for f in os.listdir(snapshots_path):
f_path = os.path.join(snapshots_path, f)
if f_path[-5:] == ".json" and os.path.isfile(f_path):
self.open(f_path, user)
for f in os.listdir(placeholders_path):
f_path = os.path.join(placeholders_path, f)
if f_path[-5:] == ".json" and os.path.isfile(f_path):
self.open(f_path, user)
def open(self, filepath, user):
with open(filepath, 'r') as file:
content = json.loads(file.read())
self.snapshots.append((content, user, filepath))
def parsing(self):
for s, user, f_path in self.snapshots:
if s.get("type") == "Websnapshot":
self.build_placeholder(s, user, f_path)
else:
self.build_snapshot(s, user, f_path)
def build_placeholder(self, s, user, f_path):
try:
create_index(s, user)
create_annotation(s, user, commit=True)
except Exception as err:
txt = "Error: in placeholder {} \n{}".format(f_path, err)
logger.exception(txt)
def build_snapshot(self, s, user, f_path):
try:
Build(s, user)
except Exception as err:
txt = "Error: in Snapshot {} \n{}".format(f_path, err)
logger.exception(txt)

View File

@ -61,7 +61,7 @@ class Evidence:
self.get_owner() self.get_owner()
qry = 'uuid:"{}"'.format(self.uuid) qry = 'uuid:"{}"'.format(self.uuid)
matches = search(self.owner, qry, limit=1) matches = search(self.owner, qry, limit=1)
if matches.size() < 0: if matches and matches.size() < 0:
return return
for xa in matches: for xa in matches:

View File

@ -1,15 +1,16 @@
import os
import json import json
import shutil
import hashlib import hashlib
import logging
from datetime import datetime
from dmidecode import DMIParse from dmidecode import DMIParse
from json_repair import repair_json from json_repair import repair_json
from evidence.models import Annotation from evidence.models import Annotation
from evidence.xapian import index from evidence.xapian import index
from utils.constants import ALGOS, CHASSIS_DH from utils.constants import CHASSIS_DH
logger = logging.getLogger('django')
def get_network_cards(child, nets): def get_network_cards(child, nets):
@ -17,15 +18,18 @@ def get_network_cards(child, nets):
nets.append(child) nets.append(child)
if child.get('children'): if child.get('children'):
[get_network_cards(x, nets) for x in child['children']] [get_network_cards(x, nets) for x in child['children']]
def get_mac(lshw): def get_mac(lshw):
nets = [] nets = []
try: try:
hw = json.loads(lshw) if type(lshw) is dict:
hw = lshw
else:
hw = json.loads(lshw)
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
hw = json.loads(repair_json(lshw)) hw = json.loads(repair_json(lshw))
try: try:
get_network_cards(hw, nets) get_network_cards(hw, nets)
except Exception as ss: except Exception as ss:
@ -74,10 +78,21 @@ class Build:
serial_number = device.get("serialNumber", '') serial_number = device.get("serialNumber", '')
sku = device.get("sku", '') sku = device.get("sku", '')
hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}" hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}"
return hashlib.sha3_256(hid.encode()).hexdigest() return hashlib.sha3_256(hid.encode()).hexdigest()
def create_annotations(self): def create_annotations(self):
annotation = Annotation.objects.filter(
uuid=self.uuid,
owner=self.user.institution,
type=Annotation.Type.SYSTEM,
)
if annotation:
txt = "Warning: Snapshot {} exist as annotation !!".format(self.uuid)
logger.exception(txt)
return
for k, v in self.algorithms.items(): for k, v in self.algorithms.items():
Annotation.objects.create( Annotation.objects.create(
@ -99,7 +114,7 @@ class Build:
def get_sku(self): def get_sku(self):
return self.dmi.get("System")[0].get("SKU Number", "n/a").strip() return self.dmi.get("System")[0].get("SKU Number", "n/a").strip()
def get_chassis(self): def get_chassis(self):
return self.dmi.get("Chassis")[0].get("Type", '_virtual') return self.dmi.get("Chassis")[0].get("Type", '_virtual')
@ -115,7 +130,7 @@ class Build:
if not snapshot["data"].get('lshw'): if not snapshot["data"].get('lshw'):
return f"{manufacturer}{model}{chassis}{serial_number}{sku}" return f"{manufacturer}{model}{chassis}{serial_number}{sku}"
lshw = snapshot["data"]["lshw"] lshw = snapshot["data"]["lshw"]
# mac = get_mac2(hwinfo_raw) or "" # mac = get_mac2(hwinfo_raw) or ""
mac = get_mac(lshw) or "" mac = get_mac(lshw) or ""

View File

@ -11,7 +11,10 @@ import xapian
def search(institution, qs, offset=0, limit=10): def search(institution, qs, offset=0, limit=10):
database = xapian.Database("db") try:
database = xapian.Database("db")
except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError):
return
qp = xapian.QueryParser() qp = xapian.QueryParser()
qp.set_database(database) qp.set_database(database)
@ -31,12 +34,9 @@ def search(institution, qs, offset=0, limit=10):
def index(institution, uuid, snap): def index(institution, uuid, snap):
uuid = 'uuid:"{}"'.format(uuid) uuid = 'uuid:"{}"'.format(uuid)
try: matches = search(institution, uuid, limit=1)
matches = search(institution, uuid, limit=1) if matches and matches.size() > 0:
if matches.size() > 0: return
return
except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError):
pass
database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN) database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN)
indexer = xapian.TermGenerator() indexer = xapian.TermGenerator()

View File

@ -2,12 +2,17 @@ import json
import uuid import uuid
import hashlib import hashlib
import datetime import datetime
import logging
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from evidence.xapian import index from evidence.xapian import index
from evidence.models import Annotation from evidence.models import Annotation
from device.models import Device from device.models import Device
logger = logging.getLogger('django')
def create_doc(data): def create_doc(data):
if not data: if not data:
return return
@ -76,6 +81,17 @@ def create_annotation(doc, user, commit=False):
'value': doc['CUSTOMER_ID'], 'value': doc['CUSTOMER_ID'],
} }
if commit: if commit:
annotation = Annotation.objects.filter(
uuid=doc["uuid"],
owner=user.institution,
type=Annotation.Type.SYSTEM,
)
if annotation:
txt = "Warning: Snapshot {} exist as annotation !!".format(doc["uuid"])
logger.exception(txt)
return annotation
return Annotation.objects.create(**data) return Annotation.objects.create(**data)
return Annotation(**data) return Annotation(**data)