Add reindex from snapshots files

This commit is contained in:
Cayo Puigdefabregas 2024-10-25 17:36:13 +02:00
parent 1a2f1249ff
commit fcb07a7337
6 changed files with 137 additions and 20 deletions

View File

@ -66,7 +66,7 @@ class SearchView(InventaryMixin):
limit
)
if not matches.size():
if not matches or not matches.size():
return self.search_hids(query, offset, limit)
devices = []

View File

@ -0,0 +1,86 @@
import os
import json
import logging
from django.core.management.base import BaseCommand
from django.conf import settings
from utils.device import create_annotation, create_doc, create_index
from user.models import Institution
from evidence.parse import Build
logger = logging.getLogger('django')
class Command(BaseCommand):
help = "Reindex snapshots"
snapshots = []
EVIDENCES = settings.EVIDENCES_DIR
def handle(self, *args, **kwargs):
if os.path.isdir(self.EVIDENCES):
self.read_files(self.EVIDENCES)
self.parsing()
def read_files(self, directory):
for filename in os.listdir(directory):
filepath = os.path.join(directory, filename)
if not os.path.isdir(filepath):
continue
institution = Institution.objects.filter(name=filename).first()
if not institution:
continue
user = institution.user_set.filter(is_admin=True).first()
if not user:
txt = "Error No there are Admins for the institution: {}".format(
institution.name
)
logger.exception(txt)
continue
snapshots_path = os.path.join(filepath, "snapshots")
placeholders_path = os.path.join(filepath, "placeholders")
for f in os.listdir(snapshots_path):
f_path = os.path.join(snapshots_path, f)
if f_path[-5:] == ".json" and os.path.isfile(f_path):
self.open(f_path, user)
for f in os.listdir(placeholders_path):
f_path = os.path.join(placeholders_path, f)
if f_path[-5:] == ".json" and os.path.isfile(f_path):
self.open(f_path, user)
def open(self, filepath, user):
with open(filepath, 'r') as file:
content = json.loads(file.read())
self.snapshots.append((content, user, filepath))
def parsing(self):
for s, user, f_path in self.snapshots:
if s.get("type") == "Websnapshot":
self.build_placeholder(s, user, f_path)
else:
self.build_snapshot(s, user, f_path)
def build_placeholder(self, s, user, f_path):
try:
create_index(s, user)
create_annotation(s, user, commit=True)
except Exception as err:
txt = "Error: in placeholder {} \n{}".format(f_path, err)
logger.exception(txt)
def build_snapshot(self, s, user, f_path):
try:
Build(s, user)
except Exception as err:
txt = "Error: in Snapshot {} \n{}".format(f_path, err)
logger.exception(txt)

View File

@ -61,7 +61,7 @@ class Evidence:
self.get_owner()
qry = 'uuid:"{}"'.format(self.uuid)
matches = search(self.owner, qry, limit=1)
if matches.size() < 0:
if matches and matches.size() < 0:
return
for xa in matches:

View File

@ -1,15 +1,16 @@
import os
import json
import shutil
import hashlib
import logging
from datetime import datetime
from dmidecode import DMIParse
from json_repair import repair_json
from evidence.models import Annotation
from evidence.xapian import index
from utils.constants import ALGOS, CHASSIS_DH
from utils.constants import CHASSIS_DH
logger = logging.getLogger('django')
def get_network_cards(child, nets):
@ -17,15 +18,18 @@ def get_network_cards(child, nets):
nets.append(child)
if child.get('children'):
[get_network_cards(x, nets) for x in child['children']]
def get_mac(lshw):
nets = []
try:
hw = json.loads(lshw)
if type(lshw) is dict:
hw = lshw
else:
hw = json.loads(lshw)
except json.decoder.JSONDecodeError:
hw = json.loads(repair_json(lshw))
try:
get_network_cards(hw, nets)
except Exception as ss:
@ -74,10 +78,21 @@ class Build:
serial_number = device.get("serialNumber", '')
sku = device.get("sku", '')
hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}"
return hashlib.sha3_256(hid.encode()).hexdigest()
def create_annotations(self):
annotation = Annotation.objects.filter(
uuid=self.uuid,
owner=self.user.institution,
type=Annotation.Type.SYSTEM,
)
if annotation:
txt = "Warning: Snapshot {} exist as annotation !!".format(self.uuid)
logger.exception(txt)
return
for k, v in self.algorithms.items():
Annotation.objects.create(
@ -99,7 +114,7 @@ class Build:
def get_sku(self):
return self.dmi.get("System")[0].get("SKU Number", "n/a").strip()
def get_chassis(self):
return self.dmi.get("Chassis")[0].get("Type", '_virtual')
@ -115,7 +130,7 @@ class Build:
if not snapshot["data"].get('lshw'):
return f"{manufacturer}{model}{chassis}{serial_number}{sku}"
lshw = snapshot["data"]["lshw"]
# mac = get_mac2(hwinfo_raw) or ""
mac = get_mac(lshw) or ""

View File

@ -11,7 +11,10 @@ import xapian
def search(institution, qs, offset=0, limit=10):
database = xapian.Database("db")
try:
database = xapian.Database("db")
except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError):
return
qp = xapian.QueryParser()
qp.set_database(database)
@ -31,12 +34,9 @@ def search(institution, qs, offset=0, limit=10):
def index(institution, uuid, snap):
uuid = 'uuid:"{}"'.format(uuid)
try:
matches = search(institution, uuid, limit=1)
if matches.size() > 0:
return
except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError):
pass
matches = search(institution, uuid, limit=1)
if matches and matches.size() > 0:
return
database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN)
indexer = xapian.TermGenerator()

View File

@ -2,12 +2,17 @@ import json
import uuid
import hashlib
import datetime
import logging
from django.core.exceptions import ValidationError
from evidence.xapian import index
from evidence.models import Annotation
from device.models import Device
logger = logging.getLogger('django')
def create_doc(data):
if not data:
return
@ -76,6 +81,17 @@ def create_annotation(doc, user, commit=False):
'value': doc['CUSTOMER_ID'],
}
if commit:
annotation = Annotation.objects.filter(
uuid=doc["uuid"],
owner=user.institution,
type=Annotation.Type.SYSTEM,
)
if annotation:
txt = "Warning: Snapshot {} exist as annotation !!".format(doc["uuid"])
logger.exception(txt)
return annotation
return Annotation.objects.create(**data)
return Annotation(**data)