diff --git a/cxx b/cxx index c6d4e4d9..7c4ac436 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit c6d4e4d907e8da83845c095e80374280f8db29c5 +Subproject commit 7c4ac436e47cd15beed304cad2b8151d50a1efd1 diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index 3a1a9af7..ef2be582 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -181,7 +181,8 @@ def __init__( properties_list = RequiredPropertiesConfigSwig(restricted_properties_vector) # The property names the engine is built with. A property's index is - # its place in this list, which is fixed at build. + # its place in this list, which is fixed at build. An empty list + # means the engine takes its names from the native code once built. self.build_property_names = list(restricted_properties_vector) # Assemble configuration @@ -304,9 +305,7 @@ def init_engine(self): self.engine = engine # The table from each build-time name to its index, made once here - # so every read goes by index. An engine built with no names has no - # build-time list, so the table is empty and every read goes by - # name, which is the native behaviour it has today. + # so every read goes by index. self.property_indexes = self.read_property_indexes() # Get keys and add to evidenceKey filter @@ -350,12 +349,17 @@ def read_property_indexes(self): The index of each property the engine was built with, keyed by its name in lower case. The native code gives a property the place of its name among the names the engine was built with, the first place - where a name is given twice. An engine built with no names has an - empty table, so every read of its properties goes by name. + where a name is given twice. An engine built with no names takes + the names the native code gives after the first load, which are + every property in that data set sorted by name and which a reload + does not change. """ + names = self.build_property_names + if not names: + names = list(self.engine.getRequiredProperties()) indexes = {} - for index, name in enumerate(self.build_property_names): + for index, name in enumerate(names): indexes.setdefault(name.lower(), index) return indexes diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py index a4d38f84..981b715b 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py @@ -23,7 +23,8 @@ """An engine built with a list of property names reads each one by its index in that list, with the native engine and the Lite data file. Also prints how long a read by name and a read by index take. An engine built -with no names keeps reading by name until the native change lands.""" +with no names takes the names the native code gives after the first load as +its list, and reads by index too.""" import time import unittest @@ -39,6 +40,8 @@ "/11D167 Safari/9537.53") # Reads timed for the figures in the warnings. TIMED_READS = 20000 +# Builds of the index table timed for the figures in the warnings. +TIMED_TABLE_BUILDS = 200 def shown(value): @@ -58,6 +61,15 @@ def process(engine): return pipeline, flowdata +def first_places(names): + """Each name in lower case against its first place in the list, which + is the index the native code gives.""" + indexes = {} + for index, name in enumerate(names): + indexes.setdefault(name.lower(), index) + return indexes + + def data_set_names(engine): """The names the data set holds, in the data set's own order. Only the tests use this order, to build engines whose name lists differ from it. @@ -80,19 +92,95 @@ def names_from_the_data_set(self): self.addCleanup(flowdata.close) return data_set_names(engine), flowdata - def test_an_engine_with_no_names_keeps_reading_by_name(self): - """An engine built with no names has an empty table, so nothing - about it changes until the native change lands.""" + def no_names_engine(self): engine = DeviceDetectionOnPremise( data_file_path=data_file, licence_keys="", auto_update=False) pipeline, flowdata = process(engine) self.addCleanup(pipeline.close) self.addCleanup(flowdata.close) - self.assertEqual([], engine.build_property_names) - self.assertEqual({}, engine.property_indexes) - # Reading still works, by name, exactly as before. + return engine, pipeline, flowdata + + def test_an_engine_with_no_names_reads_by_index(self): + """An engine built with no names takes the names the native code + gives after the first load as its list, so its table holds every + one of them and each index names its property.""" + engine, _, flowdata = self.no_names_engine() + required = list(engine.engine.getRequiredProperties()) + self.assertIn("IsMobile", required) + self.assertEqual(first_places(required), engine.property_indexes) + results = flowdata.device.swig_results + for name, index in engine.property_indexes.items(): + self.assertEqual(name, results.getPropertyName(index).lower()) + # Reading through the table gives what an engine built with the + # same names gives. + named = DeviceDetectionOnPremise( + data_file_path=data_file, licence_keys="", auto_update=False, + restricted_properties=list(required)) + second, named_data = process(named) + self.addCleanup(second.close) + self.addCleanup(named_data.close) + for name in ("ismobile", "browsername", "platformname"): + self.assertEqual( + shown(named_data.device.get(name)), + shown(flowdata.device.get(name)), name) self.assertTrue(flowdata.device.ismobile.value()) + def test_a_reload_keeps_the_table_of_an_engine_with_no_names(self): + """A reload leaves an engine built with no names with its table, + and each index still names the same property.""" + engine, pipeline, _ = self.no_names_engine() + required = list(engine.engine.getRequiredProperties()) + table = dict(engine.property_indexes) + self.assertEqual(first_places(required), table) + engine.engine.refreshData() + self.assertEqual(required, list(engine.engine.getRequiredProperties())) + self.assertEqual(table, engine.property_indexes) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("header.user-agent", mobile_ua) + flowdata.process() + results = flowdata.device.swig_results + for name, index in table.items(): + self.assertEqual(name, results.getPropertyName(index).lower()) + self.assertTrue(flowdata.device.ismobile.value()) + + def test_an_engine_with_no_names_times_each_read(self): + """With no names, a read by name, a read by index and the typed + read give the same value, with the time each read takes and the + time to build the index table, for the figures.""" + engine, _, flowdata = self.no_names_engine() + results = flowdata.device.swig_results + index = list(engine.engine.getRequiredProperties()).index("IsMobile") + self.assertEqual( + results.getValueAsBool("IsMobile").getValue(), + results.getValueAsBool(index).getValue()) + self.assertEqual( + results.getValueAsBool("IsMobile").getValue(), + flowdata.device.ismobile.value()) + + start = time.perf_counter() + for _ in range(TIMED_READS): + results.getValueAsBool("IsMobile") + by_name = time.perf_counter() - start + start = time.perf_counter() + for _ in range(TIMED_READS): + results.getValueAsBool(index) + by_index = time.perf_counter() - start + start = time.perf_counter() + for _ in range(TIMED_READS): + flowdata.device.get("ismobile") + typed = time.perf_counter() - start + start = time.perf_counter() + for _ in range(TIMED_TABLE_BUILDS): + engine.read_property_indexes() + table = time.perf_counter() - start + warnings.warn( + f"No names, IsMobile read {TIMED_READS} times, native by name " + f"{by_name * 1e9 / TIMED_READS:.0f} ns a read, native by index " + f"{by_index * 1e9 / TIMED_READS:.0f} ns a read, typed read " + f"{typed * 1e9 / TIMED_READS:.0f} ns a read, index table " + f"{table * 1e9 / TIMED_TABLE_BUILDS:.0f} ns a build") + def test_the_table_follows_the_order_the_names_were_given_in(self): names, _ = self.names_from_the_data_set() given = list(reversed(names[:6])) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 865506ba..054026ba 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -150,7 +150,8 @@ def __init__( properties_list = RequiredPropertiesConfigSwig(restricted_properties_vector) # The property names the engine is built with. A property's index is - # its place in this list, which is fixed at build. + # its place in this list, which is fixed at build. An empty list + # means the engine takes its names from the native code once built. self.build_property_names = list(restricted_properties_vector) # Assemble configuration @@ -245,9 +246,7 @@ def init_engine(self): self.engine = engine # The table from each build-time name to its index, made once here - # so every read goes by index. An engine built with no names has no - # build-time list, so the table is empty and every read goes by - # name, which is the native behaviour it has today. + # so every read goes by index. self.property_indexes = self.read_property_indexes() # Get keys and add to evidenceKey filter @@ -295,12 +294,17 @@ def read_property_indexes(self): The index of each property the engine was built with, keyed by its name in lower case. The native code gives a property the place of its name among the names the engine was built with, the first place - where a name is given twice. An engine built with no names has an - empty table, so every read of its properties goes by name. + where a name is given twice. An engine built with no names takes + the names the native code gives after the first load, which are + every property in that data set sorted by name and which a reload + does not change. """ + names = self.build_property_names + if not names: + names = list(self.engine.getRequiredProperties()) indexes = {} - for index, name in enumerate(self.build_property_names): + for index, name in enumerate(names): indexes.setdefault(name.lower(), index) return indexes diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py index 1bb785d3..f8ccdfdc 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py @@ -22,8 +22,8 @@ """An engine built with a list of property names reads each one by its index in that list, with the native engine and the data file the tests use. -An engine built with no names keeps reading by name until the native -change lands.""" +An engine built with no names takes the names the native code gives after +the first load as its list, and reads by index too.""" import os import unittest @@ -54,6 +54,15 @@ def process(engine): return pipeline, flowdata +def first_places(names): + """Each name in lower case against its first place in the list, which + is the index the native code gives.""" + indexes = {} + for index, name in enumerate(names): + indexes.setdefault(name.lower(), index) + return indexes + + def data_set_names(engine): """The names the data set holds, in the data set's own order. Only the tests use this order, to build engines whose name lists differ from it. @@ -79,18 +88,57 @@ def names_from_the_data_set(self): self.addCleanup(flowdata.close) return data_set_names(engine), flowdata - def test_an_engine_with_no_names_keeps_reading_by_name(self): - """An engine built with no names has an empty table, so nothing - about it changes until the native change lands.""" + def no_names_engine(self): engine = IpIntelligenceOnPremise( data_file_path=data_file, licence_keys="") pipeline, flowdata = process(engine) self.addCleanup(pipeline.close) self.addCleanup(flowdata.close) - self.assertEqual([], engine.build_property_names) - self.assertEqual({}, engine.property_indexes) - # Reading still works, by name, exactly as before. - self.assertTrue(flowdata.ip.get(data_set_names(engine)[0])) + return engine, pipeline, flowdata + + def test_an_engine_with_no_names_reads_by_index(self): + """An engine built with no names takes the names the native code + gives after the first load as its list, so its table holds every + one of them and each index names its property.""" + engine, _, flowdata = self.no_names_engine() + required = list(engine.engine.getRequiredProperties()) + self.assertNotEqual([], required) + self.assertEqual(first_places(required), engine.property_indexes) + results = flowdata.ip.swig_results + for name, index in engine.property_indexes.items(): + self.assertEqual(name, results.getPropertyName(index).lower()) + # Reading through the table gives what an engine built with the + # same names gives. + named = IpIntelligenceOnPremise( + data_file_path=data_file, licence_keys="", + restricted_properties=list(required)) + second, named_data = process(named) + self.addCleanup(second.close) + self.addCleanup(named_data.close) + for name in engine.property_indexes: + if name in engine.properties: + self.assertEqual( + shown(named_data.ip.get(name)), + shown(flowdata.ip.get(name)), name) + + def test_a_reload_keeps_the_table_of_an_engine_with_no_names(self): + """A reload leaves an engine built with no names with its table, + and each index still names the same property.""" + engine, pipeline, _ = self.no_names_engine() + required = list(engine.engine.getRequiredProperties()) + table = dict(engine.property_indexes) + self.assertEqual(first_places(required), table) + self.assertNotEqual({}, table) + engine.engine.refreshData() + self.assertEqual(required, list(engine.engine.getRequiredProperties())) + self.assertEqual(table, engine.property_indexes) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("query.client-ip", "8.8.8.8") + flowdata.process() + results = flowdata.ip.swig_results + for name, index in table.items(): + self.assertEqual(name, results.getPropertyName(index).lower()) def test_the_table_follows_the_order_the_names_were_given_in(self): names, _ = self.names_from_the_data_set()