diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index a4162609..ff598ef1 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -456,6 +456,32 @@ def process_internal(self, flowdata): return + def is_cacheable(self, data): + + """! + A response whose JSON has a top-level member named fodid, in any + case, holds a 51Did created for the one request that asked for it, + so it is neither stored in the cache nor served from it. A fodid + below the top level is not one, and a response that cannot be read + as a JSON object is cached as before. + + @type data: ElementData + @param data: A response the engine has just made, or one found in + the cache + + @rtype: bool + @return: False if the response holds a fodid + """ + + try: + answer = json.loads(data.get("cloud")) + except (AttributeError, JSONDecodeError, KeyError, TypeError, + ValueError): + return True + if not isinstance(answer, dict): + return True + return not any(str(name).lower() == "fodid" for name in answer) + def report_properties_not_covered(self, result): """! diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine_cache.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine_cache.py new file mode 100644 index 00000000..d49db793 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine_cache.py @@ -0,0 +1,146 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +A cache never stores or serves a Cloud Request Engine response that holds a +value created for one request, being a top-level member named fodid in any +case, as the caching page of the specification says. +""" + +import json + +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines.aspectdata_dictionary import \ + AspectDataDictionary +from fiftyone_pipeline_engines.basic_dictionary_cache import \ + BasicDictionaryCache + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase +from .classes.mockrequestclient import MockRequestClient + + +class CountingClient(MockRequestClient): + """Answers every data request with the same text and counts them.""" + + def __init__(self, answer): + super(CountingClient, self).__init__() + self.answer = answer + self.data_requests = 0 + + def request(self, type, url, content, originHeader): + response = super(CountingClient, self).request( + type, url, content, originHeader) + if "resource_key.json" in url: + self.data_requests += 1 + response.text = self.answer + return response + + +class AlwaysHitCache(BasicDictionaryCache): + """Answers every lookup with the one entry it holds, so that a reader + reaches an entry the engine never stored itself.""" + + def __init__(self): + super(AlwaysHitCache, self).__init__() + self.entry = None + + def get_cache_value(self, key): + return self.entry + + +class TestCloudRequestEngineCache(CloudRequestEngineTestsBase): + + def pipeline_for(self, client, cache): + """A pipeline holding one Cloud Request Engine with the cache given. + Gives the engine and the pipeline.""" + + engine = CloudRequestEngine({ + "resource_key": "resource_key", "http_client": client}) + engine.set_cache(cache) + pipeline = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}).add(engine).build() + self.addCleanup(pipeline.close) + return engine, pipeline + + def answer_for(self, pipeline): + """One request through the pipeline. Gives the response text the + request ended with.""" + + data = pipeline.create_flowdata() + data.evidence.add("query.User-Agent", "iPhone") + data.process() + return data.get("cloud").get("cloud") + + def requests_for(self, answer): + """Two requests with the same evidence, through a cache. Gives how + many reached the cloud service.""" + + client = CountingClient(answer) + _, pipeline = self.pipeline_for(client, BasicDictionaryCache()) + for _ in range(2): + self.assertEqual(answer, self.answer_for(pipeline)) + return client.data_requests + + def found_in_cache(self, entry_text): + """One request whose cache answers with an entry holding entry_text, + where the cloud service answers with something else. Gives the + response text the request ended with and how many requests reached + the service.""" + + answer = json.dumps({"device": {"ismobile": False}}) + client = CountingClient(answer) + cache = AlwaysHitCache() + engine, pipeline = self.pipeline_for(client, cache) + cache.entry = AspectDataDictionary(engine, {"cloud": entry_text}) + return self.answer_for(pipeline), client.data_requests + + def test_an_ordinary_response_is_cached(self): + answer = json.dumps({"device": {"ismobile": True}}) + self.assertEqual(1, self.requests_for(answer)) + + def test_a_response_with_a_fodid_is_neither_stored_nor_served(self): + for name in ("fodid", "FodId", "FODID"): + answer = json.dumps({"device": {"ismobile": True}, name: "abc"}) + self.assertEqual(2, self.requests_for(answer), name) + + def test_a_fodid_below_the_top_level_does_not_count(self): + answer = json.dumps({"device": {"fodid": "abc"}}) + self.assertEqual(1, self.requests_for(answer)) + + def test_an_entry_without_a_fodid_is_served(self): + """The cache really is read, so the test below says something.""" + + entry_text = json.dumps({"device": {"ismobile": True}}) + served, requests = self.found_in_cache(entry_text) + self.assertEqual(entry_text, served) + self.assertEqual(0, requests) + + def test_an_entry_holding_a_fodid_is_not_served(self): + for name in ("fodid", "FodId", "FODID"): + entry_text = json.dumps({"device": {"ismobile": True}, + name: "abc"}) + served, requests = self.found_in_cache(entry_text) + self.assertNotEqual(entry_text, served, name) + self.assertEqual(1, requests, name) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py index 8729ab87..4894bca8 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py @@ -74,11 +74,33 @@ def set_restricted_properties(self, properties_list): name.lower() for name in properties_list} + def is_cacheable(self, data): + + """! + Whether a result may be stored in the cache and served from it. A + result that holds a value created for the one request that asked for + it must not be, so an engine whose results can hold one overrides + this. + + @type data: ElementData + @param data: A result the engine has just made, or one found in the + cache + + @rtype: bool + @return: True if the result may be stored and served + + """ + + return True + def in_cache(self, flowdata): """! - A method to check if a flowdata's evidence is in the cache - + A method to check if a flowdata's evidence is in the cache. An entry + the engine may not serve, because it holds a value created for the + one request that asked for it, counts as a miss and is left where it + is, so the engine processes the flowdata as if nothing was there. + @type FlowData: FlowData @param FlowData: @@ -86,14 +108,14 @@ def in_cache(self, flowdata): @return: True or false: a flowdata's evidence is in the cache """ - + keys = self.filter_evidence(flowdata) cacheKey = json.dumps(keys) cached = self.cache.get_cache_value(cacheKey) - if cached is not None: + if cached is not None and self.is_cacheable(cached): flowdata.set_element_data(cached) return True @@ -130,8 +152,10 @@ def process(self, flowdata): return True else: self.process_internal(flowdata) - cacheKey = json.dumps(self.filter_evidence(flowdata)) - self.cache.set_cache_value(cacheKey, flowdata.get(self.datakey)) + data = flowdata.get(self.datakey) + if self.is_cacheable(data): + cacheKey = json.dumps(self.filter_evidence(flowdata)) + self.cache.set_cache_value(cacheKey, data) else: