diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index 0fd31aac..38cbafb7 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -478,7 +478,12 @@ def process_internal(self, flow_data): evidence = EvidenceDeviceDetectionSwig() # The native evidence takes text, and the Sequence Element adds a number. + # Evidence keys ignore case, but the native evidence matches a key's + # prefix with case, so the prefix is given in lower case. for key, value in flow_data.evidence.get_all().items(): + prefix, dot, field = key.partition(".") + if dot: + key = prefix.lower() + dot + field evidence.__setitem__(key, str(value)) result = self.engine.process(evidence) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_evidence_key_case.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_evidence_key_case.py new file mode 100644 index 00000000..c3ded15d --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_evidence_key_case.py @@ -0,0 +1,67 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder + +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" + +MOBILE_USER_AGENT = ( + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) AppleWebKit/537" + ".51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D167 Safari/9537.53") + + +class EvidenceKeyCaseTests(unittest.TestCase): + """! + evidence.md line 10: "Keys MUST be case-insensitive". The native + evidence matches a key's prefix with case, so a key spelled in another + case must still reach the native engine. Checked against a real data + file. + """ + + def setUp(self): + self.pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=data_file, licence_keys="", + usage_sharing=False).build() + self.addCleanup(self.pipeline.close) + + def is_mobile(self, key): + # The value is read before the flow data closes, which releases its + # native results. + with self.pipeline.create_flowdata() as flowdata: + flowdata.evidence.add(key, MOBILE_USER_AGENT) + flowdata.process() + ismobile = flowdata.device.ismobile + self.assertTrue(ismobile.has_value(), key) + return ismobile.value() + + def test_a_key_in_any_case_reaches_the_native_engine(self): + for key in ("Header.User-Agent", "HEADER.USER-AGENT", + "Query.User-Agent"): + with self.subTest(key=key): + self.assertEqual( + self.is_mobile(key.lower()), self.is_mobile(key)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py index 4ee68108..b859fecf 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py @@ -20,9 +20,30 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .evidence_keyfilter import EvidenceKeyFilter from .flowdata import FlowData from .logger import Logger + +class PipelineEvidenceKeyFilter(EvidenceKeyFilter): + + """! + The Evidence a Pipeline accepts, being every key one of its FlowElements + accepts. Each element's filter is read when a key is checked, as an + element's list can change once its data or the cloud service's answer + arrives. + + """ + + def __init__(self, pipeline): + self.pipeline = pipeline + + def filter_evidence_key(self, key): + return any( + element.get_evidence_key_filter().filter_evidence_key(key) + for element in self.pipeline.flow_elements_list.values()) + + class Pipeline: """! @@ -168,6 +189,19 @@ def get_element(self, key): return self.flow_elements_list[key] + def get_evidence_key_filter(self): + + """! + The Evidence the Pipeline accepts, being every key one of its + FlowElements accepts. + + @rtype: EvidenceKeyFilter + @return: Returns an EvidenceKeyFilter + + """ + + return PipelineEvidenceKeyFilter(self) + def element_not_found(self, element, flowdata): """! Trigger error when an element cannot be found in the pipeline diff --git a/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py new file mode 100644 index 00000000..16cb4612 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py @@ -0,0 +1,93 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + + +class Element(FlowElement): + """An element that accepts the keys it holds when asked.""" + + def __init__(self, datakey, keys): + super(Element, self).__init__() + self.datakey = datakey + self.keys = keys + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + + def process_internal(self, flowdata): + pass + + +class PipelineEvidenceKeyFilterTests(unittest.TestCase): + """! + advertize-accepted-evidence.md line 5: "All Flow Elements and Pipelines + MUST be capable of programmatically describing the Evidence keys that + they can make use of." + """ + + def build(self, *elements): + builder = PipelineBuilder(NO_DEFAULT_ELEMENTS) + for element in elements: + builder.add(element) + pipeline = builder.build() + self.addCleanup(pipeline.close) + return pipeline + + def test_the_pipeline_accepts_what_any_element_accepts(self): + accepted = self.build( + Element("one", ["header.user-agent"]), + Element("two", ["query.mark"])).get_evidence_key_filter() + self.assertTrue(accepted.filter_evidence_key("header.user-agent")) + self.assertTrue(accepted.filter_evidence_key("Header.User-Agent")) + self.assertTrue(accepted.filter_evidence_key("query.mark")) + self.assertFalse(accepted.filter_evidence_key("cookie.other")) + + def test_the_pipeline_reads_its_elements_filters_when_asked(self): + later = Element("later", []) + accepted = self.build(later).get_evidence_key_filter() + self.assertFalse(accepted.filter_evidence_key("header.x-later")) + later.keys = ["header.x-later"] + self.assertTrue(accepted.filter_evidence_key("header.x-later")) + + def test_the_pipeline_filters_evidence_to_what_it_accepts(self): + pipeline = self.build(Element("one", ["header.user-agent"])) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a browser") + flowdata.evidence.add("cookie.other", "not read") + self.assertEqual( + {"header.user-agent": "a browser"}, + pipeline.get_evidence_key_filter().filter_evidence( + flowdata.evidence.get_all())) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index f35fac78..176a7111 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -693,7 +693,11 @@ def process_internal(self, flow_data): # accepts no evidence of its own. cachekey = self.tracker_evidence_key_filter.filter_evidence( self.found_evidence(flow_data)) - cachekey = json.dumps(cachekey, sort_keys=True, default=str) + # Evidence keys ignore case, so a repeat is found whatever the + # letters of its keys. + cachekey = json.dumps( + {key.lower(): value for key, value in cachekey.items()}, + sort_keys=True, default=str) share = self.tracker.track(cachekey) if share: self.tracker.set_cache_value(cachekey) @@ -864,9 +868,11 @@ def try_add(self, key, value): category = "" field = key + # Evidence keys ignore case, so the prefix, which names the record's + # element, is in lower case whatever the letters of the key. first_separator = key.find(".") if first_separator > 0: - category = key[0:first_separator] + category = key[0:first_separator].lower() field = key[first_separator + 1:len(key)] # Add the evidence to the dictionary. diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record_fields.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record_fields.py index 9590bab4..2586e325 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record_fields.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record_fields.py @@ -259,6 +259,32 @@ def test_evidence_added_beside_the_web_request_is_shared(self): + "<", name) self.assertIn('Name="user-agent"', sent[0]) + def test_a_repeat_with_keys_in_another_case_is_shared_once(self): + # evidence.md line 10, "Keys MUST be case-insensitive", so the same + # values under keys spelled in another case are a repeat. + usage = usage_element() + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + for key in ("header.x-test", "Header.X-Test"): + flow_data = pipeline.create_flowdata() + flow_data.evidence.add("header.user-agent", "agent") + flow_data.evidence.add(key, "the value") + flow_data.process() + self.assertEqual(1, len(sent_on_close(usage))) + + def test_evidence_is_recorded_under_its_prefix_in_lower_case(self): + # The record names each value's element by the key's prefix, which + # is the same element whatever the letters of the key. + usage = usage_element() + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + flow_data = pipeline.create_flowdata() + flow_data.evidence.add("Header.X-Test", "the value") + flow_data.process() + sent = sent_on_close(usage) + self.assertEqual(1, len(sent)) + self.assertIn("
the value<", sent[0]) + if __name__ == "__main__": unittest.main()