Move cxx to 7c4ac436 and read by index on an engine built with no names

device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py

@@ -181,7 +181,8 @@ def __init__(
         properties_list = RequiredPropertiesConfigSwig(restricted_properties_vector)
 
         # The property names the engine is built with. A property's index is
-        # its place in this list, which is fixed at build.
+        # its place in this list, which is fixed at build. An empty list
+        # means the engine takes its names from the native code once built.
         self.build_property_names = list(restricted_properties_vector)
 
         #  Assemble configuration
@@ -304,9 +305,7 @@ def init_engine(self):
         self.engine = engine
 
         # The table from each build-time name to its index, made once here
-        # so every read goes by index. An engine built with no names has no
-        # build-time list, so the table is empty and every read goes by
-        # name, which is the native behaviour it has today.
+        # so every read goes by index.
         self.property_indexes = self.read_property_indexes()
 
         # Get keys and add to evidenceKey filter
@@ -350,12 +349,17 @@ def read_property_indexes(self):
         The index of each property the engine was built with, keyed by its
         name in lower case. The native code gives a property the place of
         its name among the names the engine was built with, the first place
-        where a name is given twice. An engine built with no names has an
-        empty table, so every read of its properties goes by name.
+        where a name is given twice. An engine built with no names takes
+        the names the native code gives after the first load, which are
+        every property in that data set sorted by name and which a reload
+        does not change.
         """
 
+        names = self.build_property_names
+        if not names:
+            names = list(self.engine.getRequiredProperties())
         indexes = {}
-        for index, name in enumerate(self.build_property_names):
+        for index, name in enumerate(names):
             indexes.setdefault(name.lower(), index)
         return indexes
 

device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py

@@ -23,7 +23,8 @@
 """An engine built with a list of property names reads each one by its
 index in that list, with the native engine and the Lite data file. Also
 prints how long a read by name and a read by index take. An engine built
-with no names keeps reading by name until the native change lands."""
+with no names takes the names the native code gives after the first load as
+its list, and reads by index too."""
 
 import time
 import unittest
@@ -39,6 +40,8 @@
              "/11D167 Safari/9537.53")
 # Reads timed for the figures in the warnings.
 TIMED_READS = 20000
+# Builds of the index table timed for the figures in the warnings.
+TIMED_TABLE_BUILDS = 200
 
 
 def shown(value):
@@ -58,6 +61,15 @@ def process(engine):
     return pipeline, flowdata
 
 
+def first_places(names):
+    """Each name in lower case against its first place in the list, which
+    is the index the native code gives."""
+    indexes = {}
+    for index, name in enumerate(names):
+        indexes.setdefault(name.lower(), index)
+    return indexes
+
+
 def data_set_names(engine):
     """The names the data set holds, in the data set's own order. Only the
     tests use this order, to build engines whose name lists differ from it.
@@ -80,19 +92,95 @@ def names_from_the_data_set(self):
         self.addCleanup(flowdata.close)
         return data_set_names(engine), flowdata
 
-    def test_an_engine_with_no_names_keeps_reading_by_name(self):
-        """An engine built with no names has an empty table, so nothing
-        about it changes until the native change lands."""
+    def no_names_engine(self):
         engine = DeviceDetectionOnPremise(
             data_file_path=data_file, licence_keys="", auto_update=False)
         pipeline, flowdata = process(engine)
         self.addCleanup(pipeline.close)
         self.addCleanup(flowdata.close)
-        self.assertEqual([], engine.build_property_names)
-        self.assertEqual({}, engine.property_indexes)
-        # Reading still works, by name, exactly as before.
+        return engine, pipeline, flowdata
+
+    def test_an_engine_with_no_names_reads_by_index(self):
+        """An engine built with no names takes the names the native code
+        gives after the first load as its list, so its table holds every
+        one of them and each index names its property."""
+        engine, _, flowdata = self.no_names_engine()
+        required = list(engine.engine.getRequiredProperties())
+        self.assertIn("IsMobile", required)
+        self.assertEqual(first_places(required), engine.property_indexes)
+        results = flowdata.device.swig_results
+        for name, index in engine.property_indexes.items():
+            self.assertEqual(name, results.getPropertyName(index).lower())
+        # Reading through the table gives what an engine built with the
+        # same names gives.
+        named = DeviceDetectionOnPremise(
+            data_file_path=data_file, licence_keys="", auto_update=False,
+            restricted_properties=list(required))
+        second, named_data = process(named)
+        self.addCleanup(second.close)
+        self.addCleanup(named_data.close)
+        for name in ("ismobile", "browsername", "platformname"):
+            self.assertEqual(
+                shown(named_data.device.get(name)),
+                shown(flowdata.device.get(name)), name)
         self.assertTrue(flowdata.device.ismobile.value())
 
+    def test_a_reload_keeps_the_table_of_an_engine_with_no_names(self):
+        """A reload leaves an engine built with no names with its table,
+        and each index still names the same property."""
+        engine, pipeline, _ = self.no_names_engine()
+        required = list(engine.engine.getRequiredProperties())
+        table = dict(engine.property_indexes)
+        self.assertEqual(first_places(required), table)
+        engine.engine.refreshData()
+        self.assertEqual(required, list(engine.engine.getRequiredProperties()))
+        self.assertEqual(table, engine.property_indexes)
+        flowdata = pipeline.create_flowdata()
+        self.addCleanup(flowdata.close)
+        flowdata.evidence.add("header.user-agent", mobile_ua)
+        flowdata.process()
+        results = flowdata.device.swig_results
+        for name, index in table.items():
+            self.assertEqual(name, results.getPropertyName(index).lower())
+        self.assertTrue(flowdata.device.ismobile.value())
+
+    def test_an_engine_with_no_names_times_each_read(self):
+        """With no names, a read by name, a read by index and the typed
+        read give the same value, with the time each read takes and the
+        time to build the index table, for the figures."""
+        engine, _, flowdata = self.no_names_engine()
+        results = flowdata.device.swig_results
+        index = list(engine.engine.getRequiredProperties()).index("IsMobile")
+        self.assertEqual(
+            results.getValueAsBool("IsMobile").getValue(),
+            results.getValueAsBool(index).getValue())
+        self.assertEqual(
+            results.getValueAsBool("IsMobile").getValue(),
+            flowdata.device.ismobile.value())
+
+        start = time.perf_counter()
+        for _ in range(TIMED_READS):
+            results.getValueAsBool("IsMobile")
+        by_name = time.perf_counter() - start
+        start = time.perf_counter()
+        for _ in range(TIMED_READS):
+            results.getValueAsBool(index)
+        by_index = time.perf_counter() - start
+        start = time.perf_counter()
+        for _ in range(TIMED_READS):
+            flowdata.device.get("ismobile")
+        typed = time.perf_counter() - start
+        start = time.perf_counter()
+        for _ in range(TIMED_TABLE_BUILDS):
+            engine.read_property_indexes()
+        table = time.perf_counter() - start
+        warnings.warn(
+            f"No names, IsMobile read {TIMED_READS} times, native by name "
+            f"{by_name * 1e9 / TIMED_READS:.0f} ns a read, native by index "
+            f"{by_index * 1e9 / TIMED_READS:.0f} ns a read, typed read "
+            f"{typed * 1e9 / TIMED_READS:.0f} ns a read, index table "
+            f"{table * 1e9 / TIMED_TABLE_BUILDS:.0f} ns a build")
+
     def test_the_table_follows_the_order_the_names_were_given_in(self):
         names, _ = self.names_from_the_data_set()
         given = list(reversed(names[:6]))

ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py

@@ -150,7 +150,8 @@ def __init__(
         properties_list = RequiredPropertiesConfigSwig(restricted_properties_vector)
 
         # The property names the engine is built with. A property's index is
-        # its place in this list, which is fixed at build.
+        # its place in this list, which is fixed at build. An empty list
+        # means the engine takes its names from the native code once built.
         self.build_property_names = list(restricted_properties_vector)
 
         #  Assemble configuration
@@ -245,9 +246,7 @@ def init_engine(self):
         self.engine = engine
 
         # The table from each build-time name to its index, made once here
-        # so every read goes by index. An engine built with no names has no
-        # build-time list, so the table is empty and every read goes by
-        # name, which is the native behaviour it has today.
+        # so every read goes by index.
         self.property_indexes = self.read_property_indexes()
 
         # Get keys and add to evidenceKey filter
@@ -295,12 +294,17 @@ def read_property_indexes(self):
         The index of each property the engine was built with, keyed by its
         name in lower case. The native code gives a property the place of
         its name among the names the engine was built with, the first place
-        where a name is given twice. An engine built with no names has an
-        empty table, so every read of its properties goes by name.
+        where a name is given twice. An engine built with no names takes
+        the names the native code gives after the first load, which are
+        every property in that data set sorted by name and which a reload
+        does not change.
         """
 
+        names = self.build_property_names
+        if not names:
+            names = list(self.engine.getRequiredProperties())
         indexes = {}
-        for index, name in enumerate(self.build_property_names):
+        for index, name in enumerate(names):
             indexes.setdefault(name.lower(), index)
         return indexes
 

ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py

@@ -22,8 +22,8 @@
 
 """An engine built with a list of property names reads each one by its
 index in that list, with the native engine and the data file the tests use.
-An engine built with no names keeps reading by name until the native
-change lands."""
+An engine built with no names takes the names the native code gives after
+the first load as its list, and reads by index too."""
 
 import os
 import unittest
@@ -54,6 +54,15 @@ def process(engine):
     return pipeline, flowdata
 
 
+def first_places(names):
+    """Each name in lower case against its first place in the list, which
+    is the index the native code gives."""
+    indexes = {}
+    for index, name in enumerate(names):
+        indexes.setdefault(name.lower(), index)
+    return indexes
+
+
 def data_set_names(engine):
     """The names the data set holds, in the data set's own order. Only the
     tests use this order, to build engines whose name lists differ from it.
@@ -79,18 +88,57 @@ def names_from_the_data_set(self):
         self.addCleanup(flowdata.close)
         return data_set_names(engine), flowdata
 
-    def test_an_engine_with_no_names_keeps_reading_by_name(self):
-        """An engine built with no names has an empty table, so nothing
-        about it changes until the native change lands."""
+    def no_names_engine(self):
         engine = IpIntelligenceOnPremise(
             data_file_path=data_file, licence_keys="")
         pipeline, flowdata = process(engine)
         self.addCleanup(pipeline.close)
         self.addCleanup(flowdata.close)
-        self.assertEqual([], engine.build_property_names)
-        self.assertEqual({}, engine.property_indexes)
-        # Reading still works, by name, exactly as before.
-        self.assertTrue(flowdata.ip.get(data_set_names(engine)[0]))
+        return engine, pipeline, flowdata
+
+    def test_an_engine_with_no_names_reads_by_index(self):
+        """An engine built with no names takes the names the native code
+        gives after the first load as its list, so its table holds every
+        one of them and each index names its property."""
+        engine, _, flowdata = self.no_names_engine()
+        required = list(engine.engine.getRequiredProperties())
+        self.assertNotEqual([], required)
+        self.assertEqual(first_places(required), engine.property_indexes)
+        results = flowdata.ip.swig_results
+        for name, index in engine.property_indexes.items():
+            self.assertEqual(name, results.getPropertyName(index).lower())
+        # Reading through the table gives what an engine built with the
+        # same names gives.
+        named = IpIntelligenceOnPremise(
+            data_file_path=data_file, licence_keys="",
+            restricted_properties=list(required))
+        second, named_data = process(named)
+        self.addCleanup(second.close)
+        self.addCleanup(named_data.close)
+        for name in engine.property_indexes:
+            if name in engine.properties:
+                self.assertEqual(
+                    shown(named_data.ip.get(name)),
+                    shown(flowdata.ip.get(name)), name)
+
+    def test_a_reload_keeps_the_table_of_an_engine_with_no_names(self):
+        """A reload leaves an engine built with no names with its table,
+        and each index still names the same property."""
+        engine, pipeline, _ = self.no_names_engine()
+        required = list(engine.engine.getRequiredProperties())
+        table = dict(engine.property_indexes)
+        self.assertEqual(first_places(required), table)
+        self.assertNotEqual({}, table)
+        engine.engine.refreshData()
+        self.assertEqual(required, list(engine.engine.getRequiredProperties()))
+        self.assertEqual(table, engine.property_indexes)
+        flowdata = pipeline.create_flowdata()
+        self.addCleanup(flowdata.close)
+        flowdata.evidence.add("query.client-ip", "8.8.8.8")
+        flowdata.process()
+        results = flowdata.ip.swig_results
+        for name, index in table.items():
+            self.assertEqual(name, results.getPropertyName(index).lower())
 
     def test_the_table_follows_the_order_the_names_were_given_in(self):
         names, _ = self.names_from_the_data_set()