Delete lots; add LotDeviceDescendants view really fixing querying devices in lots

ereuse_devicehub/db.py

@@ -1,4 +1,6 @@
+from sqlalchemy import event
 from sqlalchemy.dialects import postgresql
+from sqlalchemy_utils import view
 from teal.db import SchemaSQLAlchemy
 
 
@@ -17,3 +19,21 @@ class SQLAlchemy(SchemaSQLAlchemy):
 
 
 db = SQLAlchemy(session_options={"autoflush": False})
+
+
+def create_view(name, selectable):
+    """Creates a view.
+
+    This is an adaptation from sqlalchemy_utils.view. See
+     `the test on sqlalchemy-utils <https://github.com/kvesteri/
+    sqlalchemy-utils/blob/master/tests/test_views.py>`_ for an
+    example on how to use.
+    """
+    table = view.create_table_from_selectable(name=name, selectable=selectable, metadata=None)
+
+    # We need to ensure views are created / destroyed before / after
+    # SchemaSQLAlchemy's listeners execute
+    # That is why insert=True in 'after_create'
+    event.listen(db.metadata, 'after_create', view.CreateView(name, selectable), insert=True)
+    event.listen(db.metadata, 'before_drop', view.DropView(name))
+    return table

ereuse_devicehub/resources/device/models.py

@@ -16,8 +16,8 @@ from sqlalchemy.orm import ColumnProperty, backref, relationship, validates
 from sqlalchemy.util import OrderedSet
 from sqlalchemy_utils import ColorType
 from stdnum import imei, meid
-from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, URL, check_lower, \
+from teal.db import CASCADE_DEL, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, URL, \
-    check_range
+    check_lower, check_range
 from teal.enums import Layouts
 from teal.marshmallow import ValidationError
 from teal.resource import url_for_resource
@@ -428,7 +428,7 @@ class Component(Device):
     parent = relationship(Computer,
                           backref=backref('components',
                                           lazy=True,
-                                          cascade=CASCADE,
+                                          cascade=CASCADE_DEL,
                                           order_by=lambda: Component.id,
                                           collection_class=OrderedSet),
                           primaryjoin=parent_id == Computer.id)

ereuse_devicehub/resources/device/views.py

@@ -12,10 +12,10 @@ from teal.resource import View
 from ereuse_devicehub import auth
 from ereuse_devicehub.db import db
 from ereuse_devicehub.resources import search
-from ereuse_devicehub.resources.device.models import Component, Computer, Device, Manufacturer
+from ereuse_devicehub.resources.device.models import Device, Manufacturer
 from ereuse_devicehub.resources.device.search import DeviceSearch
 from ereuse_devicehub.resources.event.models import Rate
-from ereuse_devicehub.resources.lot.models import Lot, LotDevice
+from ereuse_devicehub.resources.lot.models import LotDeviceDescendants
 from ereuse_devicehub.resources.tag.model import Tag
 
 
@@ -41,15 +41,10 @@ class TagQ(query.Query):
 
 
 class LotQ(query.Query):
-    id = query.Or(query.QueryField(Lot.descendantsq, fields.UUID()))
+    id = query.Or(query.Equal(LotDeviceDescendants.ancestor_lot_id, fields.UUID()))
 
 
 class Filters(query.Query):
-    _parent = Computer.__table__.alias()
-    _device_inside_lot = (Device.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
-    _parent_device_in_lot = (Device.id == Component.id) & (Component.parent_id == _parent.c.id) \
-                            & (_parent.c.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
-
     type = query.Or(OfType(Device.type))
     model = query.ILike(Device.model)
     manufacturer = query.ILike(Device.manufacturer)
@@ -59,7 +54,7 @@ class Filters(query.Query):
     # todo This part of the query is really slow
     # And forces usage of distinct, as it returns many rows
     # due to having multiple paths to the same
-    lot = query.Join(_device_inside_lot | _parent_device_in_lot, LotQ)
+    lot = query.Join(Device.id == LotDeviceDescendants.device_id, LotQ)
 
 
 class Sorting(query.Sort):

ereuse_devicehub/resources/event/models.py

@@ -18,7 +18,7 @@ from sqlalchemy.ext.orderinglist import ordering_list
 from sqlalchemy.orm import backref, relationship, validates
 from sqlalchemy.orm.events import AttributeEvents as Events
 from sqlalchemy.util import OrderedSet
-from teal.db import ArrayOfEnum, CASCADE, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \
+from teal.db import ArrayOfEnum, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \
     POLYMORPHIC_ON, StrictVersionType, URL, check_lower, check_range
 from teal.enums import Country, Currency, Subdivision
 from teal.marshmallow import ValidationError
@@ -219,7 +219,7 @@ class EventWithOneDevice(JoinedTableMixin, Event):
     device = relationship(Device,
                           backref=backref('events_one',
                                           lazy=True,
-                                          cascade=CASCADE,
+                                          cascade=CASCADE_OWN,
                                           order_by=lambda: EventWithOneDevice.created,
                                           collection_class=OrderedSet),
                           primaryjoin=Device.id == device_id)

ereuse_devicehub/resources/lot/models.py

@@ -1,5 +1,6 @@
 import uuid
 from datetime import datetime
+from typing import Union
 
 from boltons import urlutils
 from citext import CIText
@@ -9,11 +10,11 @@ from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.sql import expression as exp
 from sqlalchemy_utils import LtreeType
 from sqlalchemy_utils.types.ltree import LQUERY
-from teal.db import UUIDLtree
+from teal.db import CASCADE_OWN, UUIDLtree
 from teal.resource import url_for_resource
 
-from ereuse_devicehub.db import db
+from ereuse_devicehub.db import create_view, db
-from ereuse_devicehub.resources.device.models import Device
+from ereuse_devicehub.resources.device.models import Component, Computer, Device
 from ereuse_devicehub.resources.models import Thing
 from ereuse_devicehub.resources.user.models import User
 
@@ -89,6 +90,16 @@ class Lot(Thing):
         _id = UUIDLtree.convert(id)
         return (cls.id == Path.lot_id) & Path.path.lquery(exp.cast('*.{}.*'.format(_id), LQUERY))
 
+    @classmethod
+    def device_in_lotq(cls):
+        parent = Computer.__table__.alias()
+        device_inside_lot = (Device.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
+        parent_device_in_lot = (Device.id == Component.id) \
+                               & (Component.parent_id == parent.c.id) \
+                               & (parent.c.id == LotDevice.device_id) \
+                               & (Lot.id == LotDevice.lot_id)
+        return device_inside_lot | parent_device_in_lot
+
     @property
     def parents(self):
         return self.parentsq(self.id)
@@ -109,8 +120,28 @@ class Lot(Thing):
         """Gets the lots that are not under any other lot."""
         return cls.query.join(cls.paths).filter(db.func.nlevel(Path.path) == 1)
 
-    def __contains__(self, child: 'Lot'):
+    def delete(self):
-        return Path.has_lot(self.id, child.id)
+        """Deletes the lot.
+
+        This method removes the children lots and children
+        devices orphan from this lot and then marks this lot
+        for deletion.
+        """
+        for child in self.children:
+            self.remove_child(child)
+        db.session.delete(self)
+
+    def __contains__(self, child: Union['Lot', Device]):
+        if isinstance(child, Lot):
+            return Path.has_lot(self.id, child.id)
+        elif isinstance(child, Device):
+            device = db.session.query(LotDeviceDescendants) \
+                .filter(LotDeviceDescendants.device_id == child.id) \
+                .filter(LotDeviceDescendants.ancestor_lot_id == self.id) \
+                .one_or_none()
+            return device
+        else:
+            raise TypeError('Lot only contains devices and lots, not {}'.format(child.__class__))
 
     def __repr__(self) -> str:
         return '<Lot {0.name} devices={0.devices!r}>'.format(self)
@@ -136,7 +167,10 @@ class Path(db.Model):
                    server_default=db.text('gen_random_uuid()'))
     lot_id = db.Column(db.UUID(as_uuid=True), db.ForeignKey(Lot.id), nullable=False, index=True)
     lot = db.relationship(Lot,
-                          backref=db.backref('paths', lazy=True, collection_class=set),
+                          backref=db.backref('paths',
+                                             lazy=True,
+                                             collection_class=set,
+                                             cascade=CASCADE_OWN),
                           primaryjoin=Lot.id == lot_id)
     path = db.Column(LtreeType, nullable=False)
     created = db.Column(db.TIMESTAMP(timezone=True), server_default=db.text('CURRENT_TIMESTAMP'))
@@ -174,3 +208,54 @@ class Path(db.Model):
                 "SELECT 1 from path where path ~ '*.{}.*.{}.*'".format(parent_id, child_id)
             ).first()
         )
+
+
+class LotDeviceDescendants(db.Model):
+    """A view facilitating querying inclusion between devices and lots,
+    including components.
+
+    The view has 4 columns:
+    1. The ID of the device.
+    2. The ID of a lot containing the device.
+    3. The ID of the lot that directly contains the device.
+    4. If 1. is a component, the ID of the device that is inside the lot.
+    """
+
+    _ancestor = Lot.__table__.alias(name='ancestor')
+    """Ancestor lot table."""
+    _desc = Lot.__table__.alias()
+    """Descendant lot table."""
+    lot_device = _desc \
+        .join(LotDevice, _desc.c.id == LotDevice.lot_id) \
+        .join(Path, _desc.c.id == Path.lot_id)
+    """Join: Path -- Lot -- LotDevice"""
+
+    descendants = "path.path ~ (CAST('*.'|| replace(CAST({}.id as text), '-', '_') " \
+                  "|| '.*' AS LQUERY))".format(_ancestor.name)
+    """Query that gets the descendants of the ancestor lot."""
+    devices = db.select([
+        LotDevice.device_id,
+        _ancestor.c.id.label('ancestor_lot_id'),
+        _desc.c.id.label('parent_lot_id'),
+        None
+    ]).select_from(_ancestor).select_from(lot_device).where(descendants)
+
+    # Components
+    _parent_device = Device.__table__.alias(name='parent_device')
+    """The device that has the access to the lot."""
+    lot_device_component = lot_device \
+        .join(_parent_device, _parent_device.c.id == LotDevice.device_id) \
+        .join(Component, _parent_device.c.id == Component.parent_id)
+    """Join: Path -- Lot -- LotDevice -- ParentDevice (Device) -- Component"""
+
+    components = db.select([
+        Component.id.label('device_id'),
+        _ancestor.c.id.label('ancestor_lot_id'),
+        _desc.c.id.label('parent_lot_id'),
+        LotDevice.device_id.label('device_parent_id'),
+    ]).select_from(_ancestor).select_from(lot_device_component).where(descendants)
+
+    __table__ = create_view(
+        name='lot_device_descendants',
+        selectable=devices.union(components)
+    )

ereuse_devicehub/resources/lot/models.pyi

@@ -1,6 +1,6 @@
 import uuid
 from datetime import datetime
-from typing import Iterable, Set, Union
+from typing import Iterable, Optional, Set, Union
 from uuid import UUID
 
 from boltons import urlutils
@@ -8,6 +8,7 @@ from sqlalchemy import Column
 from sqlalchemy.orm import Query, relationship
 from sqlalchemy_utils import Ltree
 
+from ereuse_devicehub.db import db
 from ereuse_devicehub.resources.device.models import Device
 from ereuse_devicehub.resources.models import Thing
 
@@ -65,6 +66,9 @@ class Lot(Thing):
     def url(self) -> urlutils.URL:
         pass
 
+    def delete(self):
+        pass
+
 
 class Path:
     id = ...  # type: Column
@@ -79,3 +83,17 @@ class Path:
         self.lot = ...  # type: Lot
         self.path = ...  # type: Ltree
         self.created = ...  # type: datetime
+
+
+class LotDeviceDescendants(db.Model):
+    device_id = ...  # type: Column
+    ancestor_lot_id = ...  # type: Column
+    parent_lot_id = ...  # type: Column
+    device_parent_id = ...  # type: Column
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.device_id = ...  # type: int
+        self.ancestor_lot_id = ...  # type: UUID
+        self.parent_lot_id = ...  # type: UUID
+        self.device_parent_id = ...  # type: Optional[int]

ereuse_devicehub/resources/lot/views.py

@@ -97,6 +97,12 @@ class LotView(View):
             cls._p(nodes, path)
         return nodes
 
+    def delete(self, id):
+        lot = Lot.query.filter_by(id=id).one()
+        lot.delete()
+        db.session.commit()
+        return Response(status=204)
+
     @classmethod
     def _p(cls, nodes: List[dict], path: deque):
         """Recursively creates the nested lot structure.

ereuse_devicehub/resources/models.pyi

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from sqlalchemy import Column
+from sqlalchemy import Column, Table
 from teal.db import Model
 
 STR_SIZE = 64
@@ -10,6 +10,7 @@ STR_XSM_SIZE = 16
 
 
 class Thing(Model):
+    __table__ = ...  # type: Table
     t = ...  # type: str
     type = ...  # type: str
     updated = ...  # type: Column

requirements.txt

@@ -23,9 +23,9 @@ python-stdnum==1.9
 PyYAML==3.13
 requests==2.19.1
 requests-mock==1.5.2
-SQLAlchemy==1.2.11
+SQLAlchemy==1.2.14
-SQLAlchemy-Utils==0.33.3
+SQLAlchemy-Utils==0.33.6
-teal==0.2.0a29
+teal==0.2.0a30
 webargs==4.0.0
 Werkzeug==0.14.1
 sqlalchemy-citext==1.3.post0

setup.py

@@ -29,7 +29,7 @@ setup(
     long_description=long_description,
     long_description_content_type='text/markdown',
     install_requires=[
-        'teal>=0.2.0a29',  # teal always first
+        'teal>=0.2.0a30',  # teal always first
         'click',
         'click-spinner',
         'ereuse-utils[Naming]>=0.4b10',

tests/test_lot.py

@@ -23,7 +23,40 @@ In case of error, debug with:
 """
 
 
-def test_lot_modify_patch_endpoint(user: UserClient):
+@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
+def test_lot_model_children():
+    """Tests the property Lot.children
+
+    l1
+    |
+    l2
+    |
+    l3
+    """
+    lots = Lot('1'), Lot('2'), Lot('3')
+    l1, l2, l3 = lots
+    db.session.add_all(lots)
+    db.session.flush()
+
+    l1.add_child(l2)
+    db.session.flush()
+
+    assert list(l1.children) == [l2]
+
+    l2.add_child(l3)
+    assert list(l1.children) == [l2]
+
+    l2.delete()
+    db.session.flush()
+    assert not list(l1.children)
+
+    l1.delete()
+    db.session.flush()
+    l3b = Lot.query.one()
+    assert l3 == l3b
+
+
+def test_lot_modify_patch_endpoint_and_delete(user: UserClient):
     """Creates and modifies lot properties through the endpoint"""
     l, _ = user.post({'name': 'foo', 'description': 'baz'}, res=Lot)
     assert l['name'] == 'foo'
@@ -32,20 +65,17 @@ def test_lot_modify_patch_endpoint(user: UserClient):
     l_after, _ = user.get(res=Lot, item=l['id'])
     assert l_after['name'] == 'bar'
     assert l_after['description'] == 'bax'
+    user.delete(res=Lot, item=l['id'], status=204)
+    user.get(res=Lot, item=l['id'], status=404)
 
 
-@pytest.mark.xfail(reason='No DEL endpoint')
-def test_lot_delete_endpoint(user: UserClient):
-    pass
-
-
-@pytest.mark.xfail(reason='the IN comparison does not work for device')
 @pytest.mark.usefixtures(conftest.auth_app_context.__name__)
 def test_lot_device_relationship():
     device = Desktop(serial_number='foo',
                      model='bar',
                      manufacturer='foobar',
                      chassis=ComputerChassis.Lunchbox)
+    device.components.add(GraphicCard(serial_number='foo', model='bar1', manufacturer='baz'))
     child = Lot('child')
     child.devices.add(device)
     db.session.add(child)
@@ -253,21 +283,6 @@ def test_lot_roots():
     assert set(Lot.roots()) == {l1, l3}
 
 
-@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
-def test_lot_model_children():
-    """Tests the property Lot.children"""
-    lots = Lot('1'), Lot('2'), Lot('3')
-    l1, l2, l3 = lots
-    db.session.add_all(lots)
-    db.session.flush()
-
-    l1.add_child(l2)
-    db.session.flush()
-
-    children = l1.children
-    assert list(children) == [l2]
-
-
 def test_post_get_lot(user: UserClient):
     """Tests submitting and retreiving a basic lot."""
     l, _ = user.post({'name': 'Foo'}, res=Lot)