Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,32 @@ def process_internal(self, flowdata):

return

def is_cacheable(self, data):

"""!
A response whose JSON has a top-level member named fodid, in any
case, holds a 51Did created for the one request that asked for it,
so it is neither stored in the cache nor served from it. A fodid
below the top level is not one, and a response that cannot be read
as a JSON object is cached as before.

@type data: ElementData
@param data: A response the engine has just made, or one found in
the cache

@rtype: bool
@return: False if the response holds a fodid
"""

try:
answer = json.loads(data.get("cloud"))
except (AttributeError, JSONDecodeError, KeyError, TypeError,
ValueError):
return True
if not isinstance(answer, dict):
return True
return not any(str(name).lower() == "fodid" for name in answer)

def report_properties_not_covered(self, result):

"""!
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
# *********************************************************************
# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
#
# This Original Work is licensed under the European Union Public Licence
# (EUPL) v.1.2 and is subject to its terms as set out below.
#
# If a copy of the EUPL was not distributed with this file, You can obtain
# one at https://opensource.org/licenses/EUPL-1.2.
#
# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
# amended by the European Commission) shall be deemed incompatible for
# the purposes of the Work and the provisions of the compatibility
# clause in Article 5 of the EUPL shall not apply.
#
# If using the Work as, or as part of, a network application, by
# including the attribution notice(s) required under Article 5 of the EUPL
# in the end user terms of the application under an appropriate heading,
# such notice(s) shall fulfill the requirements of that article.
# *********************************************************************

"""
A cache never stores or serves a Cloud Request Engine response that holds a
value created for one request, being a top-level member named fodid in any
case, as the caching page of the specification says.
"""

import json

from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \
CloudRequestEngine
from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
from fiftyone_pipeline_engines.aspectdata_dictionary import \
AspectDataDictionary
from fiftyone_pipeline_engines.basic_dictionary_cache import \
BasicDictionaryCache

from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase
from .classes.mockrequestclient import MockRequestClient


class CountingClient(MockRequestClient):
"""Answers every data request with the same text and counts them."""

def __init__(self, answer):
super(CountingClient, self).__init__()
self.answer = answer
self.data_requests = 0

def request(self, type, url, content, originHeader):
response = super(CountingClient, self).request(
type, url, content, originHeader)
if "resource_key.json" in url:
self.data_requests += 1
response.text = self.answer
return response


class AlwaysHitCache(BasicDictionaryCache):
"""Answers every lookup with the one entry it holds, so that a reader
reaches an entry the engine never stored itself."""

def __init__(self):
super(AlwaysHitCache, self).__init__()
self.entry = None

def get_cache_value(self, key):
return self.entry


class TestCloudRequestEngineCache(CloudRequestEngineTestsBase):

def pipeline_for(self, client, cache):
"""A pipeline holding one Cloud Request Engine with the cache given.
Gives the engine and the pipeline."""

engine = CloudRequestEngine({
"resource_key": "resource_key", "http_client": client})
engine.set_cache(cache)
pipeline = PipelineBuilder({
"add_javascript_builder": False,
"use_setheader_properties": False}).add(engine).build()
self.addCleanup(pipeline.close)
return engine, pipeline

def answer_for(self, pipeline):
"""One request through the pipeline. Gives the response text the
request ended with."""

data = pipeline.create_flowdata()
data.evidence.add("query.User-Agent", "iPhone")
data.process()
return data.get("cloud").get("cloud")

def requests_for(self, answer):
"""Two requests with the same evidence, through a cache. Gives how
many reached the cloud service."""

client = CountingClient(answer)
_, pipeline = self.pipeline_for(client, BasicDictionaryCache())
for _ in range(2):
self.assertEqual(answer, self.answer_for(pipeline))
return client.data_requests

def found_in_cache(self, entry_text):
"""One request whose cache answers with an entry holding entry_text,
where the cloud service answers with something else. Gives the
response text the request ended with and how many requests reached
the service."""

answer = json.dumps({"device": {"ismobile": False}})
client = CountingClient(answer)
cache = AlwaysHitCache()
engine, pipeline = self.pipeline_for(client, cache)
cache.entry = AspectDataDictionary(engine, {"cloud": entry_text})
return self.answer_for(pipeline), client.data_requests

def test_an_ordinary_response_is_cached(self):
answer = json.dumps({"device": {"ismobile": True}})
self.assertEqual(1, self.requests_for(answer))

def test_a_response_with_a_fodid_is_neither_stored_nor_served(self):
for name in ("fodid", "FodId", "FODID"):
answer = json.dumps({"device": {"ismobile": True}, name: "abc"})
self.assertEqual(2, self.requests_for(answer), name)

def test_a_fodid_below_the_top_level_does_not_count(self):
answer = json.dumps({"device": {"fodid": "abc"}})
self.assertEqual(1, self.requests_for(answer))

def test_an_entry_without_a_fodid_is_served(self):
"""The cache really is read, so the test below says something."""

entry_text = json.dumps({"device": {"ismobile": True}})
served, requests = self.found_in_cache(entry_text)
self.assertEqual(entry_text, served)
self.assertEqual(0, requests)

def test_an_entry_holding_a_fodid_is_not_served(self):
for name in ("fodid", "FodId", "FODID"):
entry_text = json.dumps({"device": {"ismobile": True},
name: "abc"})
served, requests = self.found_in_cache(entry_text)
self.assertNotEqual(entry_text, served, name)
self.assertEqual(1, requests, name)
Original file line number Diff line number Diff line change
Expand Up @@ -74,26 +74,48 @@ def set_restricted_properties(self, properties_list):
name.lower() for name in properties_list}


def is_cacheable(self, data):

"""!
Whether a result may be stored in the cache and served from it. A
result that holds a value created for the one request that asked for
it must not be, so an engine whose results can hold one overrides
this.

@type data: ElementData
@param data: A result the engine has just made, or one found in the
cache

@rtype: bool
@return: True if the result may be stored and served

"""

return True

def in_cache(self, flowdata):

"""!
A method to check if a flowdata's evidence is in the cache

A method to check if a flowdata's evidence is in the cache. An entry
the engine may not serve, because it holds a value created for the
one request that asked for it, counts as a miss and is left where it
is, so the engine processes the flowdata as if nothing was there.

@type FlowData: FlowData
@param FlowData:

@rtype: bool
@return: True or false: a flowdata's evidence is in the cache

"""

keys = self.filter_evidence(flowdata)

cacheKey = json.dumps(keys)

cached = self.cache.get_cache_value(cacheKey)

if cached is not None:
if cached is not None and self.is_cacheable(cached):
flowdata.set_element_data(cached)

return True
Expand Down Expand Up @@ -130,8 +152,10 @@ def process(self, flowdata):
return True
else:
self.process_internal(flowdata)
cacheKey = json.dumps(self.filter_evidence(flowdata))
self.cache.set_cache_value(cacheKey, flowdata.get(self.datakey))
data = flowdata.get(self.datakey)
if self.is_cacheable(data):
cacheKey = json.dumps(self.filter_evidence(flowdata))
self.cache.set_cache_value(cacheKey, data)

else:

Expand Down
Loading