Add reindex from snapshots files
This commit is contained in:
parent
1a2f1249ff
commit
fcb07a7337
|
@ -66,7 +66,7 @@ class SearchView(InventaryMixin):
|
||||||
limit
|
limit
|
||||||
)
|
)
|
||||||
|
|
||||||
if not matches.size():
|
if not matches or not matches.size():
|
||||||
return self.search_hids(query, offset, limit)
|
return self.search_hids(query, offset, limit)
|
||||||
|
|
||||||
devices = []
|
devices = []
|
||||||
|
|
|
@ -0,0 +1,86 @@
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
from django.conf import settings
|
||||||
|
|
||||||
|
from utils.device import create_annotation, create_doc, create_index
|
||||||
|
from user.models import Institution
|
||||||
|
from evidence.parse import Build
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger('django')
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = "Reindex snapshots"
|
||||||
|
snapshots = []
|
||||||
|
EVIDENCES = settings.EVIDENCES_DIR
|
||||||
|
|
||||||
|
def handle(self, *args, **kwargs):
|
||||||
|
if os.path.isdir(self.EVIDENCES):
|
||||||
|
self.read_files(self.EVIDENCES)
|
||||||
|
|
||||||
|
self.parsing()
|
||||||
|
|
||||||
|
def read_files(self, directory):
|
||||||
|
for filename in os.listdir(directory):
|
||||||
|
filepath = os.path.join(directory, filename)
|
||||||
|
if not os.path.isdir(filepath):
|
||||||
|
continue
|
||||||
|
|
||||||
|
institution = Institution.objects.filter(name=filename).first()
|
||||||
|
|
||||||
|
if not institution:
|
||||||
|
continue
|
||||||
|
user = institution.user_set.filter(is_admin=True).first()
|
||||||
|
if not user:
|
||||||
|
txt = "Error No there are Admins for the institution: {}".format(
|
||||||
|
institution.name
|
||||||
|
)
|
||||||
|
logger.exception(txt)
|
||||||
|
continue
|
||||||
|
|
||||||
|
snapshots_path = os.path.join(filepath, "snapshots")
|
||||||
|
placeholders_path = os.path.join(filepath, "placeholders")
|
||||||
|
|
||||||
|
for f in os.listdir(snapshots_path):
|
||||||
|
f_path = os.path.join(snapshots_path, f)
|
||||||
|
|
||||||
|
if f_path[-5:] == ".json" and os.path.isfile(f_path):
|
||||||
|
self.open(f_path, user)
|
||||||
|
|
||||||
|
for f in os.listdir(placeholders_path):
|
||||||
|
f_path = os.path.join(placeholders_path, f)
|
||||||
|
|
||||||
|
if f_path[-5:] == ".json" and os.path.isfile(f_path):
|
||||||
|
self.open(f_path, user)
|
||||||
|
|
||||||
|
def open(self, filepath, user):
|
||||||
|
with open(filepath, 'r') as file:
|
||||||
|
content = json.loads(file.read())
|
||||||
|
self.snapshots.append((content, user, filepath))
|
||||||
|
|
||||||
|
def parsing(self):
|
||||||
|
for s, user, f_path in self.snapshots:
|
||||||
|
if s.get("type") == "Websnapshot":
|
||||||
|
self.build_placeholder(s, user, f_path)
|
||||||
|
else:
|
||||||
|
self.build_snapshot(s, user, f_path)
|
||||||
|
|
||||||
|
def build_placeholder(self, s, user, f_path):
|
||||||
|
try:
|
||||||
|
create_index(s, user)
|
||||||
|
create_annotation(s, user, commit=True)
|
||||||
|
except Exception as err:
|
||||||
|
txt = "Error: in placeholder {} \n{}".format(f_path, err)
|
||||||
|
logger.exception(txt)
|
||||||
|
|
||||||
|
def build_snapshot(self, s, user, f_path):
|
||||||
|
try:
|
||||||
|
Build(s, user)
|
||||||
|
except Exception as err:
|
||||||
|
txt = "Error: in Snapshot {} \n{}".format(f_path, err)
|
||||||
|
logger.exception(txt)
|
||||||
|
|
|
@ -61,7 +61,7 @@ class Evidence:
|
||||||
self.get_owner()
|
self.get_owner()
|
||||||
qry = 'uuid:"{}"'.format(self.uuid)
|
qry = 'uuid:"{}"'.format(self.uuid)
|
||||||
matches = search(self.owner, qry, limit=1)
|
matches = search(self.owner, qry, limit=1)
|
||||||
if matches.size() < 0:
|
if matches and matches.size() < 0:
|
||||||
return
|
return
|
||||||
|
|
||||||
for xa in matches:
|
for xa in matches:
|
||||||
|
|
|
@ -1,15 +1,16 @@
|
||||||
import os
|
|
||||||
import json
|
import json
|
||||||
import shutil
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import logging
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from dmidecode import DMIParse
|
from dmidecode import DMIParse
|
||||||
from json_repair import repair_json
|
from json_repair import repair_json
|
||||||
|
|
||||||
from evidence.models import Annotation
|
from evidence.models import Annotation
|
||||||
from evidence.xapian import index
|
from evidence.xapian import index
|
||||||
from utils.constants import ALGOS, CHASSIS_DH
|
from utils.constants import CHASSIS_DH
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger('django')
|
||||||
|
|
||||||
|
|
||||||
def get_network_cards(child, nets):
|
def get_network_cards(child, nets):
|
||||||
|
@ -17,15 +18,18 @@ def get_network_cards(child, nets):
|
||||||
nets.append(child)
|
nets.append(child)
|
||||||
if child.get('children'):
|
if child.get('children'):
|
||||||
[get_network_cards(x, nets) for x in child['children']]
|
[get_network_cards(x, nets) for x in child['children']]
|
||||||
|
|
||||||
|
|
||||||
def get_mac(lshw):
|
def get_mac(lshw):
|
||||||
nets = []
|
nets = []
|
||||||
try:
|
try:
|
||||||
hw = json.loads(lshw)
|
if type(lshw) is dict:
|
||||||
|
hw = lshw
|
||||||
|
else:
|
||||||
|
hw = json.loads(lshw)
|
||||||
except json.decoder.JSONDecodeError:
|
except json.decoder.JSONDecodeError:
|
||||||
hw = json.loads(repair_json(lshw))
|
hw = json.loads(repair_json(lshw))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
get_network_cards(hw, nets)
|
get_network_cards(hw, nets)
|
||||||
except Exception as ss:
|
except Exception as ss:
|
||||||
|
@ -74,10 +78,21 @@ class Build:
|
||||||
serial_number = device.get("serialNumber", '')
|
serial_number = device.get("serialNumber", '')
|
||||||
sku = device.get("sku", '')
|
sku = device.get("sku", '')
|
||||||
hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}"
|
hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}"
|
||||||
|
|
||||||
|
|
||||||
return hashlib.sha3_256(hid.encode()).hexdigest()
|
return hashlib.sha3_256(hid.encode()).hexdigest()
|
||||||
|
|
||||||
def create_annotations(self):
|
def create_annotations(self):
|
||||||
|
annotation = Annotation.objects.filter(
|
||||||
|
uuid=self.uuid,
|
||||||
|
owner=self.user.institution,
|
||||||
|
type=Annotation.Type.SYSTEM,
|
||||||
|
)
|
||||||
|
|
||||||
|
if annotation:
|
||||||
|
txt = "Warning: Snapshot {} exist as annotation !!".format(self.uuid)
|
||||||
|
logger.exception(txt)
|
||||||
|
return
|
||||||
|
|
||||||
for k, v in self.algorithms.items():
|
for k, v in self.algorithms.items():
|
||||||
Annotation.objects.create(
|
Annotation.objects.create(
|
||||||
|
@ -99,7 +114,7 @@ class Build:
|
||||||
|
|
||||||
def get_sku(self):
|
def get_sku(self):
|
||||||
return self.dmi.get("System")[0].get("SKU Number", "n/a").strip()
|
return self.dmi.get("System")[0].get("SKU Number", "n/a").strip()
|
||||||
|
|
||||||
def get_chassis(self):
|
def get_chassis(self):
|
||||||
return self.dmi.get("Chassis")[0].get("Type", '_virtual')
|
return self.dmi.get("Chassis")[0].get("Type", '_virtual')
|
||||||
|
|
||||||
|
@ -115,7 +130,7 @@ class Build:
|
||||||
|
|
||||||
if not snapshot["data"].get('lshw'):
|
if not snapshot["data"].get('lshw'):
|
||||||
return f"{manufacturer}{model}{chassis}{serial_number}{sku}"
|
return f"{manufacturer}{model}{chassis}{serial_number}{sku}"
|
||||||
|
|
||||||
lshw = snapshot["data"]["lshw"]
|
lshw = snapshot["data"]["lshw"]
|
||||||
# mac = get_mac2(hwinfo_raw) or ""
|
# mac = get_mac2(hwinfo_raw) or ""
|
||||||
mac = get_mac(lshw) or ""
|
mac = get_mac(lshw) or ""
|
||||||
|
|
|
@ -11,7 +11,10 @@ import xapian
|
||||||
|
|
||||||
|
|
||||||
def search(institution, qs, offset=0, limit=10):
|
def search(institution, qs, offset=0, limit=10):
|
||||||
database = xapian.Database("db")
|
try:
|
||||||
|
database = xapian.Database("db")
|
||||||
|
except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError):
|
||||||
|
return
|
||||||
|
|
||||||
qp = xapian.QueryParser()
|
qp = xapian.QueryParser()
|
||||||
qp.set_database(database)
|
qp.set_database(database)
|
||||||
|
@ -31,12 +34,9 @@ def search(institution, qs, offset=0, limit=10):
|
||||||
|
|
||||||
def index(institution, uuid, snap):
|
def index(institution, uuid, snap):
|
||||||
uuid = 'uuid:"{}"'.format(uuid)
|
uuid = 'uuid:"{}"'.format(uuid)
|
||||||
try:
|
matches = search(institution, uuid, limit=1)
|
||||||
matches = search(institution, uuid, limit=1)
|
if matches and matches.size() > 0:
|
||||||
if matches.size() > 0:
|
return
|
||||||
return
|
|
||||||
except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN)
|
database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN)
|
||||||
indexer = xapian.TermGenerator()
|
indexer = xapian.TermGenerator()
|
||||||
|
|
|
@ -2,12 +2,17 @@ import json
|
||||||
import uuid
|
import uuid
|
||||||
import hashlib
|
import hashlib
|
||||||
import datetime
|
import datetime
|
||||||
|
import logging
|
||||||
|
|
||||||
from django.core.exceptions import ValidationError
|
from django.core.exceptions import ValidationError
|
||||||
from evidence.xapian import index
|
from evidence.xapian import index
|
||||||
from evidence.models import Annotation
|
from evidence.models import Annotation
|
||||||
from device.models import Device
|
from device.models import Device
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger('django')
|
||||||
|
|
||||||
|
|
||||||
def create_doc(data):
|
def create_doc(data):
|
||||||
if not data:
|
if not data:
|
||||||
return
|
return
|
||||||
|
@ -76,6 +81,17 @@ def create_annotation(doc, user, commit=False):
|
||||||
'value': doc['CUSTOMER_ID'],
|
'value': doc['CUSTOMER_ID'],
|
||||||
}
|
}
|
||||||
if commit:
|
if commit:
|
||||||
|
annotation = Annotation.objects.filter(
|
||||||
|
uuid=doc["uuid"],
|
||||||
|
owner=user.institution,
|
||||||
|
type=Annotation.Type.SYSTEM,
|
||||||
|
)
|
||||||
|
|
||||||
|
if annotation:
|
||||||
|
txt = "Warning: Snapshot {} exist as annotation !!".format(doc["uuid"])
|
||||||
|
logger.exception(txt)
|
||||||
|
return annotation
|
||||||
|
|
||||||
return Annotation.objects.create(**data)
|
return Annotation.objects.create(**data)
|
||||||
|
|
||||||
return Annotation(**data)
|
return Annotation(**data)
|
||||||
|
|
Loading…
Reference in New Issue