From 9995692f32c39149409d79b45e5945597d5c2501 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 17:24:00 +0000 Subject: [PATCH 1/3] Test the Derived Property Element and its example Adds the tests for a new pipeline.derivedproperty module and a new developer example, before the code they test. The module is the Derived Property Element of the specification, which produces new properties from the properties other elements in the Pipeline produce, following rules written in a YAML or JSON script. The derived-properties repository is added as the submodule pipeline/derived-properties, at cc30638. Its scripts folder holds the built-in scripts the package ships, and its tests folder holds the case files and rejection cases every language runs. SharedCasesTests runs both, and BuiltInScriptTests fails when BuiltInScript and the scripts folder differ. The other tests cover reading YAML by the YAML 1.2 core schema, so yes stays a string, the conversion of source values, the Pipeline check, the no value message, the logging, the metadata and building from a configuration file. ExampleTests and FindOutMoreTests run the example. The two modules are added to pipeline/pom.xml, the developer examples pom and ci/packages.json, and SnakeYAML is named in the dependency management of pipeline/pom.xml. The tests do not compile until the code follows. --- .gitmodules | 3 + ci/packages.json | 2 + pipeline/derived-properties | 1 + pipeline/pipeline.derivedproperty/pom.xml | 86 +++ .../derivedproperty/BuiltInScriptTests.java | 109 ++++ ...ivedPropertyElementConfigurationTests.java | 152 +++++ .../DerivedPropertyElementTests.java | 605 ++++++++++++++++++ .../DerivedScriptParserTests.java | 216 +++++++ .../flowelements/RecordingLoggerFactory.java | 147 +++++ .../flowelements/SharedCasesTests.java | 419 ++++++++++++ .../flowelements/SourceValuesTests.java | 192 ++++++ .../flowelements/StubSourceElement.java | 126 ++++ .../StubSourceElementBuilder.java | 70 ++ .../flowelements/TestPipelineBuilder.java | 35 + .../pom.xml | 50 ++ .../derivedproperty/ExampleTests.java | 96 +++ .../derivedproperty/FindOutMoreTests.java | 103 +++ pipeline/pipeline.developer-examples/pom.xml | 1 + pipeline/pom.xml | 7 + 19 files changed, 2420 insertions(+) create mode 160000 pipeline/derived-properties create mode 100644 pipeline/pipeline.derivedproperty/pom.xml create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/BuiltInScriptTests.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementTests.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParserTests.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/RecordingLoggerFactory.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SharedCasesTests.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValuesTests.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElement.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElementBuilder.java create mode 100644 pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/TestPipelineBuilder.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/pom.xml create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/ExampleTests.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/FindOutMoreTests.java diff --git a/.gitmodules b/.gitmodules index f3fd5d50e..c3cdc8753 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "pipeline/pipeline.core/configuration-validation"] path = pipeline/pipeline.core/configuration-validation url = ../configuration-validation +[submodule "pipeline/derived-properties"] + path = pipeline/derived-properties + url = ../derived-properties diff --git a/ci/packages.json b/ci/packages.json index 153c48995..3c1d6db56 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -8,6 +8,7 @@ {"Name": "pipeline.cloudrequestengine", "Path": "pipeline/pipeline.cloudrequestengine", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.common"], "Publish": true}, {"Name": "pipeline.engines.fiftyone", "Path": "pipeline/pipeline.engines.fiftyone", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.common"], "Publish": true}, {"Name": "pipeline.translation", "Path": "pipeline/pipeline.translation", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.core"], "Publish": true}, + {"Name": "pipeline.derivedproperty", "Path": "pipeline/pipeline.derivedproperty", "ExtraPaths": ["pipeline/derived-properties"], "DependsOn": ["pipeline", "pipeline.engines", "pipeline.core"], "Publish": true}, {"Name": "pipeline.javascriptbuilder", "Path": "pipeline/pipeline.javascriptbuilder", "DependsOn": ["pipeline", "pipeline.jsonbuilder", "pipeline.engines", "pipeline.common"], "Publish": true}, {"Name": "pipeline.jsonbuilder", "Path": "pipeline/pipeline.jsonbuilder", "DependsOn": ["pipeline", "pipeline.engines.fiftyone", "pipeline.common", "pipeline.engines"], "Publish": true}, {"Name": "pipeline.web.packages", "Path": "pipeline/pipeline.web.packages/pom.xml", "DependsOn": ["pipeline"], "Publish": true}, @@ -24,6 +25,7 @@ {"Name": "pipeline.developerexamples.clientsideelement.mvc", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc", "DependsOn": ["pipeline.developer-examples", "pipeline.developerexamples.clientsideelement", "pipeline.web.mvc", "pipeline.developerexamples.web.shared"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, {"Name": "pipeline.developerexamples.usagesharing", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing", "DependsOn": ["pipeline.developer-examples", "pipeline.engines.fiftyone"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, {"Name": "pipeline.developerexamples.fodid", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid", "DependsOn": ["pipeline.developer-examples", "pipeline.did"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.derivedproperty", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty", "DependsOn": ["pipeline.developer-examples", "pipeline.derivedproperty"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, {"Name": "pipeline.device-detection", "Path": "device-detection/pom.xml", "DependsOn": ["pipeline-aggregate"], "Publish": true}, {"Name": "device-detection.shared", "Path": "device-detection/device-detection.shared", "DependsOn": ["pipeline.device-detection", "pipeline.engines.fiftyone"], "Publish": true}, {"Name": "device-detection.hash.engine.on-premise", "Path": "device-detection/device-detection.hash.engine.on-premise", "ExtraPaths": ["cxx"], "DependsOn": ["pipeline.device-detection", "device-detection.shared"], "Publish": true}, diff --git a/pipeline/derived-properties b/pipeline/derived-properties new file mode 160000 index 000000000..cc3063863 --- /dev/null +++ b/pipeline/derived-properties @@ -0,0 +1 @@ +Subproject commit cc30638632050795d9acc80d5118a290ffaba085 diff --git a/pipeline/pipeline.derivedproperty/pom.xml b/pipeline/pipeline.derivedproperty/pom.xml new file mode 100644 index 000000000..3b8b7bff6 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/pom.xml @@ -0,0 +1,86 @@ + + + + + 4.0.0 + + com.51degrees + pipeline + 4.5.127-SNAPSHOT + + pipeline.derivedproperty + jar + + 51Degrees :: Pipeline :: Derived Property + The Derived Property Element produces new property values from property values other elements in the Pipeline have already produced, following rules written in a script. + https://github.com/51Degrees/51degrees-java + + + + ${project.groupId} + pipeline.engines + ${project.version} + + + ${project.groupId} + pipeline.core + ${project.version} + + + + org.yaml + snakeyaml + + + org.slf4j + slf4j-api + + + + org.junit.jupiter + junit-jupiter + + + + + + + src/main/resources + + + + ../derived-properties/scripts + fiftyone/pipeline/derivedproperty/scripts + + *.yaml + *.yml + *.json + + + + + diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/BuiltInScriptTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/BuiltInScriptTests.java new file mode 100644 index 000000000..f8bf3014a --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/BuiltInScriptTests.java @@ -0,0 +1,109 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty; + +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyElement; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyElementBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The scripts this package ships are the scripts folder of the + * derived-properties submodule, and {@link BuiltInScript} names every one + * of them, so a script added to the submodule fails this test until it has + * a constant. + */ +public class BuiltInScriptTests { + + /** + * The scripts in the submodule, by name, with their text. + */ + private static Map submoduleScripts() throws IOException { + Path folder = Paths.get("..", "derived-properties", "scripts"); + assertTrue(Files.isDirectory(folder), "The derived-properties " + + "submodule is missing from " + folder.toAbsolutePath() + + ". Run 'git submodule update --init pipeline/derived-properties'."); + Map scripts = new TreeMap<>(); + try (DirectoryStream stream = Files.newDirectoryStream( + folder, "*.{yaml,yml,json}")) { + for (Path file : stream) { + String name = file.getFileName().toString(); + scripts.put( + name.substring(0, name.lastIndexOf('.')), + new String(Files.readAllBytes(file), + StandardCharsets.UTF_8)); + } + } + return scripts; + } + + @Test + public void enumNamesEveryScriptInTheSubmodule() throws IOException { + TreeSet names = new TreeSet<>(); + for (BuiltInScript script : BuiltInScript.values()) { + names.add(script.name()); + } + assertEquals(submoduleScripts().keySet(), names, + "BuiltInScript and the scripts folder of the derived-properties " + + "submodule differ. Add a constant for each script, named " + + "exactly as the script is."); + } + + @Test + public void packagedScriptsAreTheSubmodulesScripts() throws IOException { + Map scripts = submoduleScripts(); + for (BuiltInScript script : BuiltInScript.values()) { + String text = script.readText(); + assertNotNull(text, script + " is not in the package."); + assertEquals(scripts.get(script.name()), text, + "The packaged " + script + " differs from the submodule's."); + assertSame(script, BuiltInScript.fromName(script.name())); + } + } + + @Test + public void everyScriptBuilds() { + for (BuiltInScript script : BuiltInScript.values()) { + DerivedPropertyElement element = new DerivedPropertyElementBuilder( + LoggerFactory.getILoggerFactory()) + .addScript(script) + .build(); + assertEquals(script.name(), element.getScripts().get(0).getName()); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java new file mode 100644 index 000000000..83d43f2a3 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java @@ -0,0 +1,152 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A configuration file names the builder as DerivedPropertyElement and + * passes built-in script names as Scripts and file paths as ScriptFiles, + * each a comma separated string. + *

+ * See the + * configuration + * section of the specification. + */ +public class DerivedPropertyElementConfigurationTests { + + @TempDir + Path folder; + + /** + * Writes the Band script into the folder and gives a pattern matching + * it. + */ + private String bandFiles() throws Exception { + Files.write(folder.resolve("Band.yaml"), + DerivedPropertyElementTests.BAND_YAML.getBytes( + StandardCharsets.UTF_8)); + return folder.resolve("*.yaml").toString(); + } + + /** + * Options read from an XML configuration file that puts two source + * elements in front of a Derived Property Element with the build + * parameters given. + */ + private PipelineOptions options(String buildParameters) + throws Exception { + String xml = "" + + "StubSourceElement" + + "device" + + "BrowserReleaseAge=1,IsCrawler=False" + + "" + + "StubSourceElement" + + "ip" + + "CountryCode=FR" + + "" + + "DerivedPropertyElement" + + "" + buildParameters + "" + + "" + + ""; + Path file = folder.resolve("pipeline.xml"); + Files.write(file, xml.getBytes(StandardCharsets.UTF_8)); + return PipelineOptionsFactory.getOptionsFromFile(file.toFile()); + } + + @Test + public void scriptsAndScriptFilesFromAConfigurationFile() + throws Exception { + PipelineOptions options = options( + " IsGdpr " + + "" + bandFiles() + ""); + try (Pipeline pipeline = + new PipelineBuilder().buildFromConfiguration(options); + FlowData data = pipeline.createFlowData()) { + data.process(); + IDerivedPropertyData derived = + data.get(IDerivedPropertyData.class); + assertEquals(Boolean.TRUE, + derived.getAs("IsGdpr", Boolean.class).getValue()); + assertEquals("New", + derived.getAs("Band", String.class).getValue()); + assertEquals(2, pipeline.getElement(DerivedPropertyElement.class) + .getScripts().size()); + } + } + + @Test + public void listsAreCommaSeparated() throws Exception { + DerivedPropertyElement element = new DerivedPropertyElementBuilder( + LoggerFactory.getILoggerFactory()) + .setScripts("HumanConfidence, IsGdpr,") + .setScriptFiles(" ," + bandFiles()) + .build(); + assertEquals(3, element.getScripts().size()); + } + + @Test + public void anUnknownBuildParameterIsRefused() throws Exception { + PipelineOptions options = options(""); + assertThrows(PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration(options)); + } + + @Test + public void aPatternMatchingNoFileIsRefused() throws Exception { + PipelineOptions options = options("" + + folder.resolve("*.json") + ""); + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration(options)); + assertTrue(e.getMessage().contains("no file matches"), + e.getMessage()); + } + + @Test + public void anUnknownBuiltInScriptIsRefused() throws Exception { + PipelineOptions options = options("NoSuchScript"); + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration(options)); + assertTrue(e.getMessage().contains( + "'NoSuchScript' is not a built-in script"), e.getMessage()); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementTests.java new file mode 100644 index 000000000..6b43c7e41 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementTests.java @@ -0,0 +1,605 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.derivedproperty.BuiltInScript; +import fiftyone.pipeline.derivedproperty.DerivedScriptValidationException; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.exceptions.NoValueException; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The Derived Property Element against the specification and format 1, + * using scripts written here so the tests do not depend on the content of + * the shared scripts. + */ +public class DerivedPropertyElementTests { + + static final String BAND_YAML = "Format: 1\n" + + "Name: Band\n" + + "Version: 1.0.0\n" + + "Output:\n" + + " Name: Band\n" + + " Description: A band.\n" + + " ValueType: string\n" + + " IsList: false\n" + + " Values:\n" + + " - { Name: Old }\n" + + " - { Name: New, Description: Recent. }\n" + + "Checks:\n" + + " Recent: { Property: device.BrowserReleaseAge, Lt: 2.0 }\n" + + "Rules:\n" + + " - When: { All: [ { Check: Recent }, " + + "{ Property: device.IsCrawler, Eq: false } ] }\n" + + " Then: New\n" + + " - Else: Old\n"; + + static final String BAND_JSON = "{\"Format\": 1, \"Name\": \"Band\", " + + "\"Version\": \"1.0.0\",\n" + + " \"Output\": {\"Name\": \"Band\", \"Description\": \"A band.\", " + + "\"ValueType\": \"string\", \"IsList\": false,\n" + + " \"Values\": [{\"Name\": \"Old\"}, {\"Name\": \"New\", " + + "\"Description\": \"Recent.\"}]},\n" + + " \"Checks\": {\"Recent\": {\"Property\": " + + "\"device.BrowserReleaseAge\", \"Lt\": 2}},\n" + + " \"Rules\": [{\"When\": {\"All\": [{\"Check\": \"Recent\"}, " + + "{\"Property\": \"device.IsCrawler\", \"Eq\": false}]},\n" + + " \"Then\": \"New\"}, {\"Else\": \"Old\"}]}"; + + /** + * A script reading one property under device and one under ip. + */ + static final String LIKELY_YAML = "Format: 1\n" + + "Name: Likely\n" + + "Version: 1.0.0\n" + + "Output:\n" + + " Name: Likely\n" + + " Description: Whether a person is likely to be viewing the page.\n" + + " ValueType: bool\n" + + " IsList: false\n" + + " Category: Human Detection\n" + + "Checks:\n" + + " NotCrawler: { Property: device.IsCrawler, Eq: false }\n" + + " HumanAddress: { Property: ip.HumanProbability, Ge: 7 }\n" + + "Rules:\n" + + " - When: { Failed: Checks, Eq: 0 }\n" + + " Then: true\n" + + " - Else: false\n"; + + private static final String MOBILE_YAML = "Format: 1\n" + + "Name: Mobile\n" + + "Version: 1.0.0\n" + + "Output: { Name: Mobile, Description: Whether the device is " + + "mobile., ValueType: bool, IsList: false }\n" + + "Rules:\n" + + " - When: { Property: device.IsMobile, Eq: true }\n" + + " Then: true\n" + + " - Else: false\n"; + + private static final String KIND_YAML = "Format: 1\n" + + "Name: Kind\n" + + "Version: 1.0.0\n" + + "Output: { Name: Kind, Description: The kind of device., " + + "ValueType: string, IsList: false }\n" + + "Rules:\n" + + " - When: { Property: derived.Mobile, Eq: true }\n" + + " Then: Phone\n" + + " - Else: Other\n"; + + /** + * A request the Likely script reads as true. + */ + static Map> likelyValues() { + Map> values = new LinkedHashMap<>(); + values.put("device", map("IsCrawler", "False")); + values.put("ip", map("HumanProbability", 9)); + return values; + } + + static Map map(Object... pairs) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + map.put((String) pairs[i], pairs[i + 1]); + } + return map; + } + + static DerivedScript mustValidate(String text) { + DerivedScriptValidator.Result result = + DerivedScriptValidator.validate(text, "", "code"); + assertTrue(result.getFaults().isEmpty(), + "The script did not validate:\n" + result.getFaults()); + return result.getScript(); + } + + static DerivedPropertyElementBuilder builder() { + return new DerivedPropertyElementBuilder( + LoggerFactory.getILoggerFactory()); + } + + static DerivedPropertyElement likely() { + return builder().addScript("Likely", LIKELY_YAML).build(); + } + + /** + * A YAML script and the JSON that mirrors it give the same model, and + * the canonical form adds the inferred types and the computed + * dependencies. + */ + @Test + public void yamlAndJsonGiveTheSameModel() { + String fromYaml = CanonicalJson.print(mustValidate(BAND_YAML)); + String fromJson = CanonicalJson.print(mustValidate(BAND_JSON)); + assertEquals(fromYaml, fromJson); + for (String want : new String[]{ + "\"device.BrowserReleaseAge\": {\n \"Type\": \"int\"\n }", + "\"Dependencies\": [\n \"device.BrowserReleaseAge\",\n" + + " \"device.IsCrawler\"\n ]", + "\"Lt\": 2\n"}) { + assertTrue(fromYaml.contains(want), + "The canonical form lacks " + want + ":\n" + fromYaml); + } + } + + /** + * 2 and 2.0 are both whole numbers, so they infer the same type and do + * not conflict. + */ + @Test + public void wholeNumbersInferInt() { + DerivedScript model = mustValidate(BAND_YAML.replace("Checks:\n", + "Checks:\n Young: { Property: device.BrowserReleaseAge, " + + "Lt: 2 }\n")); + assertSame(ValueType.INT, model.getProperties().get(0).getType()); + } + + /** + * Every fault in every script is reported at once, one line each. + */ + @Test + public void buildReportsEveryFault() { + DerivedScriptValidationException e = assertThrows( + DerivedScriptValidationException.class, + () -> builder() + .addScript("First", BAND_YAML.replace( + "Name: Band\nVersion", "Name: First\nVersion: x\nV")) + .addScript("Second", BAND_YAML.replace( + "Format: 1", "Format: 2")) + .setScripts("NoSuchScript") + .build()); + String message = e.getMessage(); + for (String want : new String[]{"First (code)", "Second (code)", + "Format must be 1, found 2", + "'NoSuchScript' is not a built-in script"}) { + assertTrue(message.contains(want), + "The faults lack " + want + ":\n" + message); + } + assertEquals(e.getFaults().size(), message.split("\n").length); + } + + @Test + public void buildNeedsAScript() { + assertThrows(IllegalStateException.class, () -> builder().build()); + } + + @Test + public void twoScriptsWithOneOutputAreRefused() { + DerivedScriptValidationException e = assertThrows( + DerivedScriptValidationException.class, + () -> builder() + .addScript("Band", BAND_YAML) + .addScript("Band", BAND_JSON) + .build()); + assertTrue(e.getMessage().contains("is added more than once"), + e.getMessage()); + assertTrue(e.getMessage().contains( + "is also produced by the script 'Band'"), e.getMessage()); + } + + private static void assertPipelineRefused( + String want, + FlowElement... elements) { + TestPipelineBuilder builder = new TestPipelineBuilder(); + for (FlowElement element : elements) { + builder.addFlowElement(element); + } + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, builder::build); + assertTrue(e.getMessage().contains(want), + "The build gave '" + e.getMessage() + "', not '" + want + "'"); + } + + /** + * The Pipeline check names a key no element has, a source property no + * element lists, and a derived property two elements produce. + */ + @Test + public void pipelineCheckNamesWhatIsMissing() { + List device = Collections.singletonList("IsCrawler"); + assertPipelineRefused( + "no element in the pipeline has the element data key 'ip'", + new StubSourceElement("device", device, null), + likely()); + assertPipelineRefused( + "no element under 'ip' lists the property 'HumanProbability'", + new StubSourceElement("device", device, null), + new StubSourceElement("ip", + Collections.singletonList("CountryCode"), null), + likely()); + assertPipelineRefused( + "two Derived Property Elements in the pipeline produce the " + + "derived property 'Likely'", + new StubSourceElement("device", device, null), + new StubSourceElement("ip", + Collections.singletonList("HumanProbability"), null), + likely(), + likely()); + } + + /** + * Where an element sits is not judged, so an element added before the + * elements it reads builds, and gives no value naming each property. + */ + @Test + public void positionIsNotJudged() throws Exception { + DerivedPropertyElement element = likely(); + List> elements = new ArrayList<>(); + elements.add(element); + elements.addAll(SharedCasesTests.stubsFor(element, likelyValues())); + TestPipelineBuilder builder = new TestPipelineBuilder(); + for (FlowElement e : elements) { + builder.addFlowElement(e); + } + try (Pipeline pipeline = builder.build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + AspectPropertyValue value = data + .get(IDerivedPropertyData.class) + .getAs("Likely", Boolean.class); + SharedCasesTests.checkMissing(value, + Arrays.asList("device.IsCrawler", "ip.HumanProbability"), + "Likely"); + } + } + + /** + * Sources that run in a parallel group are listed after the elements at + * the top level, and the check still finds them. + */ + @Test + public void pipelineCheckFindsAParallelGroup() throws Exception { + DerivedPropertyElement element = likely(); + List stubs = + SharedCasesTests.stubsFor(element, likelyValues()); + try (Pipeline pipeline = new TestPipelineBuilder() + .addFlowElementsParallel( + stubs.toArray(new FlowElement[0])) + .addFlowElement(element) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + assertEquals(Boolean.TRUE, data.get(IDerivedPropertyData.class) + .getAs("Likely", Boolean.class).getValue()); + } + } + + /** + * Elements in a parallel group write into the one Element Data under + * derived at the same time. + */ + @Test + public void elementsInAParallelGroupShareTheElementData() + throws Exception { + DerivedPropertyElement mobile = + builder().addScript("Mobile", MOBILE_YAML).build(); + DerivedPropertyElement band = + builder().addScript("Band", BAND_YAML).build(); + Map> values = new LinkedHashMap<>(); + values.put("device", map("IsMobile", true, "IsCrawler", false, + "BrowserReleaseAge", 1)); + StubSourceElement device = new StubSourceElement("device", + Arrays.asList("IsMobile", "IsCrawler", "BrowserReleaseAge"), + values.get("device")); + try (Pipeline pipeline = new TestPipelineBuilder() + .addFlowElement(device) + .addFlowElementsParallel(new FlowElement[]{mobile, band}) + .setAutoCloseElements(true) + .build()) { + for (int request = 0; request < 50; request++) { + try (FlowData data = pipeline.createFlowData()) { + data.process(); + IDerivedPropertyData derived = + data.get(IDerivedPropertyData.class); + assertEquals(Boolean.TRUE, + derived.getAs("Mobile", Boolean.class).getValue()); + assertEquals("New", + derived.getAs("band", String.class).getValue()); + } + } + } + } + + /** + * A request with no Element Data under a key a script reads gives no + * value, and the error is logged once for the script. + */ + @Test + public void noElementDataIsLoggedOnce() throws Exception { + RecordingLoggerFactory logs = new RecordingLoggerFactory(Level.INFO); + DerivedPropertyElement element = + new DerivedPropertyElementBuilder(logs) + .addScript("Likely", LIKELY_YAML) + .build(); + try (Pipeline pipeline = new TestPipelineBuilder() + .addFlowElement(new StubSourceElement("device", + Collections.singletonList("IsCrawler"), + map("IsCrawler", false))) + .addFlowElement(new StubSourceElement("ip", + Collections.singletonList("HumanProbability"), + map("HumanProbability", 9), false)) + .addFlowElement(element) + .build()) { + for (int request = 0; request < 3; request++) { + try (FlowData data = pipeline.createFlowData()) { + data.process(); + AspectPropertyValue value = data + .get(IDerivedPropertyData.class) + .getAs("Likely", Boolean.class); + assertFalse(value.hasValue()); + assertTrue(value.getNoValueMessage().contains( + "'ip.HumanProbability' (element 'ip' has no value " + + "for 'HumanProbability': property not present " + + "on this request)"), + value.getNoValueMessage()); + } + } + } + List errors = logs.messages(Level.ERROR); + assertEquals(1, errors.size(), errors.toString()); + assertTrue(errors.get(0).contains("reads 'ip.HumanProbability'"), + errors.get(0)); + assertTrue(errors.get(0).contains( + "after the element that supplies 'ip'"), errors.get(0)); + assertTrue(errors.get(0).contains("reported once"), errors.get(0)); + } + + /** + * A source property absent for any other reason, such as one the + * JavaScript has not populated yet, is not logged. + */ + @Test + public void anAbsentPropertyIsNotLogged() throws Exception { + RecordingLoggerFactory logs = new RecordingLoggerFactory(Level.INFO); + DerivedPropertyElement element = + new DerivedPropertyElementBuilder(logs) + .addScript("Likely", LIKELY_YAML) + .build(); + Map> values = likelyValues(); + values.get("ip").remove("HumanProbability"); + AspectPropertyValue value = + SharedCasesTests.process(element, values, "Likely"); + assertFalse(value.hasValue()); + assertTrue(logs.messages(Level.ERROR).isEmpty(), + logs.messages(Level.ERROR).toString()); + } + + /** + * A script can read the property another script in the same element + * produces, under the key derived. + */ + @Test + public void derivedFromDerived() throws Exception { + DerivedPropertyElement element = builder() + .addScript("Mobile", MOBILE_YAML) + .addScript("Kind", KIND_YAML) + .build(); + Map> values = new LinkedHashMap<>(); + values.put("device", map("IsMobile", "True")); + AspectPropertyValue kind = + SharedCasesTests.process(element, values, "Kind"); + assertTrue(kind.hasValue(), kind.getNoValueMessage()); + assertEquals("Phone", kind.getValue()); + } + + /** + * The element accepts no evidence, lists one property per script and + * exposes each script's full metadata. + */ + @Test + public void metadata() { + DerivedPropertyElement element = builder() + .addScript("Band", BAND_YAML) + .addScript("Likely", LIKELY_YAML) + .build(); + assertFalse(element.getEvidenceKeyFilter() + .include("header.user-agent")); + assertEquals("derived", element.getElementDataKey()); + List properties = element.getProperties(); + assertEquals(2, properties.size()); + assertEquals("Band", properties.get(0).getName()); + assertEquals("", properties.get(0).getCategory()); + assertSame(String.class, properties.get(0).getType()); + assertTrue(properties.get(0).isAvailable()); + assertSame(element, properties.get(0).getElement()); + assertEquals("Likely", properties.get(1).getName()); + assertEquals("Human Detection", properties.get(1).getCategory()); + assertSame(Boolean.class, properties.get(1).getType()); + + List scripts = element.getScripts(); + assertEquals(2, scripts.size()); + DerivedPropertyMetaData band = scripts.get(0); + assertEquals("Band", band.getName()); + assertEquals("1.0.0", band.getVersion()); + assertEquals(1, band.getFormat()); + assertEquals("code", band.getSource()); + assertFalse(band.isDeprecated()); + DerivedOutputMetaData output = band.getOutput(); + assertEquals("A band.", output.getDescription()); + assertEquals("string", output.getValueType()); + assertSame(String.class, output.getValueClass()); + assertFalse(output.isList()); + assertNull(output.getDefaultValue()); + assertNull(output.getCategory()); + assertEquals( + Arrays.asList("device.BrowserReleaseAge", + "device.IsCrawler"), + output.getDependencies()); + List names = new ArrayList<>(); + List descriptions = new ArrayList<>(); + for (DerivedValueMetaData value : output.getValues()) { + names.add(value.getName()); + descriptions.add(value.getDescription()); + } + assertEquals(Arrays.asList("Old", "New"), names); + assertEquals(Arrays.asList(null, "Recent."), descriptions); + assertNull(scripts.get(1).getOutput().getValues()); + } + + /** + * A built-in script's source is its name. + */ + @Test + public void builtInScriptSourceIsItsName() { + DerivedPropertyElement element = builder() + .addScript(BuiltInScript.IsGdpr) + .build(); + assertEquals("IsGdpr", element.getScripts().get(0).getSource()); + } + + /** + * The build logs one information line per script, the compiled model + * at debug level and a warning carrying a deprecated script's note. + */ + @Test + public void buildLogs() { + RecordingLoggerFactory logs = new RecordingLoggerFactory(Level.DEBUG); + String deprecated = BAND_YAML.replace("Version: 1.0.0\n", + "Version: 1.0.0\nDeprecated: true\n" + + "DeprecationNote: Use Likely.\n"); + new DerivedPropertyElementBuilder(logs) + .addScript("Likely", LIKELY_YAML) + .addScript("Band", deprecated) + .build(); + List info = logs.messages(Level.INFO, "Derived property"); + assertEquals(2, info.size(), info.toString()); + assertEquals("Derived property script 'Likely' version 1.0.0, " + + "format 1, from code, produces 'derived.Likely'", info.get(0)); + List debug = logs.messages(Level.DEBUG, "Derived property"); + assertEquals(2, debug.size(), debug.toString()); + assertTrue(debug.get(0).contains("\"Name\": \"Likely\""), + debug.get(0)); + assertTrue(debug.get(0).contains( + "\"ip.HumanProbability\": {\n \"Type\": \"int\"\n }"), + debug.get(0)); + List warnings = logs.messages(Level.WARN, "Derived property"); + assertEquals(1, warnings.size(), warnings.toString()); + assertEquals("Derived property script 'Band' is deprecated. " + + "Use Likely.", warnings.get(0)); + } + + /** + * The compiled model is printed only where debug is on. + */ + @Test + public void canonicalJsonIsNotWrittenWithoutDebug() { + RecordingLoggerFactory logs = new RecordingLoggerFactory(Level.INFO); + new DerivedPropertyElementBuilder(logs) + .addScript("Likely", LIKELY_YAML) + .build(); + assertTrue(logs.messages(Level.DEBUG).isEmpty()); + assertEquals(1, logs.messages(Level.INFO, "Derived property").size()); + } + + /** + * The Element Data gives a value of the type the script produces, and + * refuses another type or a property no script produces. + */ + @Test + public void elementDataGetAs() throws Exception { + DerivedPropertyElement element = likely(); + try (Pipeline pipeline = buildWithStubs(element, likelyValues()); + FlowData data = pipeline.createFlowData()) { + data.process(); + IDerivedPropertyData derived = data.get(IDerivedPropertyData.class); + assertEquals(Boolean.TRUE, + derived.getAs("likely", Boolean.class).getValue()); + assertThrows(ClassCastException.class, + () -> derived.getAs("Likely", String.class)); + assertThrows(PropertyMissingException.class, + () -> derived.getAs("Unlikely", Boolean.class)); + } + } + + /** + * Reading a value with no value raises the existing no value error, + * carrying the message. + */ + @Test + public void readingNoValueThrowsWithTheMessage() throws Exception { + Map> values = likelyValues(); + values.get("device").put("IsCrawler", "N/A"); + AspectPropertyValue value = + SharedCasesTests.process(likely(), values, "Likely"); + NoValueException e = + assertThrows(NoValueException.class, value::getValue); + assertEquals("Derived property 'Likely' has no value because 1 " + + "source property was not available. 'device.IsCrawler' (held " + + "'N/A' which cannot be read as bool). " + + DerivedPropertyElement.USUAL_CAUSES, e.getMessage()); + } + + static Pipeline buildWithStubs( + DerivedPropertyElement element, + Map> values) throws Exception { + TestPipelineBuilder builder = new TestPipelineBuilder(); + for (StubSourceElement stub : + SharedCasesTests.stubsFor(element, values)) { + builder.addFlowElement(stub); + } + return builder.addFlowElement(element).build(); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParserTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParserTests.java new file mode 100644 index 000000000..5e73f77da --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParserTests.java @@ -0,0 +1,216 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.derivedproperty.DerivedScriptFault; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * How a script's text is read. Plain scalars follow the YAML 1.2 core + * schema, as the reference tools do, quoted scalars are strings, keys are + * written once without regard to case, and JSON is read by the same + * reader. + *

+ * See writing + * YAML that reads the way you meant. + */ +public class DerivedScriptParserTests { + + private static ScriptNode value(String yaml) throws Exception { + return DerivedScriptParser.parse("Value: " + yaml).get("Value"); + } + + /** + * A script comparing device.Flag with the literal given. + */ + private static String flagScript(String literal) { + return "Format: 1\nName: Flag\nVersion: \"1.0.0\"\n" + + "Output: { Name: Flag, Description: A flag., ValueType: bool, " + + "IsList: false }\n" + + "Rules:\n" + + " - When: { Property: device.Flag, Eq: " + literal + " }\n" + + " Then: true\n" + + " - Else: false\n"; + } + + private static ValueType inferred(String literal) { + return DerivedPropertyElementTests.mustValidate(flagScript(literal)) + .getProperties().get(0).getType(); + } + + @Test + public void onlyTrueAndFalseAreBooleans() throws Exception { + for (String text : new String[]{"true", "True", "TRUE"}) { + assertSame(ScriptNode.Kind.BOOL, value(text).getKind(), text); + assertTrue(value(text).getBool(), text); + } + for (String text : new String[]{"false", "False", "FALSE"}) { + assertSame(ScriptNode.Kind.BOOL, value(text).getKind(), text); + } + for (String text : new String[]{"yes", "no", "on", "off", "y", "n", + "NO", "Yes", "tRUE"}) { + assertSame(ScriptNode.Kind.STRING, value(text).getKind(), text); + assertEquals(text, value(text).getText()); + } + } + + @Test + public void yesInfersStringAndTrueInfersBool() { + assertSame(ValueType.STRING, inferred("yes")); + assertSame(ValueType.STRING, inferred("\"true\"")); + assertSame(ValueType.STRING, inferred("'false'")); + assertSame(ValueType.BOOL, inferred("true")); + } + + @Test + public void numbersFollowTheCoreSchema() throws Exception { + assertEquals(8, value("8").getNumber()); + assertEquals(-8, value("-8").getNumber()); + assertEquals(8, value("+8").getNumber()); + assertEquals(8.5, value("8.5").getNumber()); + assertEquals(800, value("8e2").getNumber()); + assertEquals(0.5, value(".5").getNumber()); + assertEquals(16, value("0x10").getNumber()); + assertEquals(15, value("0o17").getNumber()); + assertEquals(Double.POSITIVE_INFINITY, value(".inf").getNumber()); + assertEquals(Double.NEGATIVE_INFINITY, value("-.Inf").getNumber()); + assertTrue(Double.isNaN(value(".NaN").getNumber())); + for (String text : new String[]{"1_000", "0b101", "1:30", "8.5.1", + "0x", "1e", "0o8"}) { + assertSame(ScriptNode.Kind.STRING, value(text).getKind(), text); + } + assertSame(ScriptNode.Kind.STRING, value("\"8\"").getKind()); + } + + @Test + public void wholeNumbersInferIntWhateverTheirForm() { + assertSame(ValueType.INT, inferred("8")); + assertSame(ValueType.INT, inferred("8.0")); + assertSame(ValueType.INT, inferred("8e0")); + assertSame(ValueType.DOUBLE, inferred("8.5")); + // Outside the range of int a whole number infers double. + assertSame(ValueType.DOUBLE, inferred("3000000000")); + } + + @Test + public void nullForms() throws Exception { + for (String text : new String[]{"~", "null", "Null", "NULL", ""}) { + assertSame(ScriptNode.Kind.NULL, value(text).getKind(), text); + } + assertSame(ScriptNode.Kind.STRING, value("'null'").getKind()); + } + + @Test + public void aKeyWrittenTwiceIsRefused() { + ScriptParseException exact = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("Name: A\nName: B\n")); + assertTrue(exact.getMessage().contains( + "the key 'Name' is written twice in one mapping"), + exact.getMessage()); + ScriptParseException cased = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse( + "Output:\n Name: A\n name: B\n")); + assertTrue(cased.getMessage().contains( + "the keys 'Name' and 'name' differ only in case"), + cased.getMessage()); + assertEquals(2, cased.getLine()); + ScriptParseException json = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("{\"Name\": \"A\", \"NAME\": 1}")); + assertTrue(json.getMessage().contains("differ only in case"), + json.getMessage()); + } + + @Test + public void textThatIsNotAScript() { + assertEquals("the script is empty", assertThrows( + ScriptParseException.class, + () -> DerivedScriptParser.parse("")).getMessage()); + assertEquals("the script is empty", assertThrows( + ScriptParseException.class, + () -> DerivedScriptParser.parse("# only a comment\n")) + .getMessage()); + assertEquals("the script must be a mapping of keys to values at the " + + "top level", assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("- 1\n- 2\n")).getMessage()); + ScriptParseException yaml = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("Name: A\n Bad: [\n")); + assertTrue(yaml.getMessage().startsWith("the text is not valid YAML: "), + yaml.getMessage()); + assertTrue(yaml.getLine() > 0); + ScriptParseException json = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse(" {\"Name\": \"A\"")); + assertTrue(json.getMessage().startsWith("the text is not valid JSON: "), + json.getMessage()); + } + + @Test + public void anAliasThatRefersToItselfIsRefused() { + ScriptParseException e = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("Rules: &a [ *a ]\n")); + assertTrue(e.getMessage().startsWith("the text is not valid YAML"), + e.getMessage()); + } + + @Test + public void aliasesAreFollowed() throws Exception { + ScriptNode document = DerivedScriptParser.parse( + "A: &x { Property: device.X, Eq: 1 }\nB: *x\n"); + assertEquals(1, document.get("B").get("Eq").getNumber()); + } + + @Test + public void faultsCarryTheLineOfTheMapping() { + DerivedScriptValidator.Result result = DerivedScriptValidator.validate( + DerivedPropertyElementTests.BAND_YAML.replace( + "{ Property: device.IsCrawler, Eq: false }", + "{ Property: device.IsCrawler, Equals: false }"), + "Band", "code"); + assertNull(result.getScript()); + assertEquals(1, result.getFaults().size(), result.getFaults() + .toString()); + DerivedScriptFault fault = result.getFaults().get(0); + assertEquals("Rules[0].When.All[1]", fault.getPath()); + assertEquals(15, fault.getLine()); + assertEquals("Band (code) line 15 at Rules[0].When.All[1]: unknown " + + "operator 'Equals', expected one of Eq, Ne, Gt, Ge, Lt, Le, In, " + + "NotIn, StartsWith, EndsWith, Contains", fault.toString()); + } + + @Test + public void inAndNotInAreRefusedOnACount() { + DerivedScriptValidator.Result result = DerivedScriptValidator.validate( + DerivedPropertyElementTests.LIKELY_YAML.replace( + "{ Failed: Checks, Eq: 0 }", "{ Failed: Checks, In: 0 }"), + "Likely", "code"); + assertNull(result.getScript()); + assertTrue(result.getFaults().get(0).getMessage().contains( + "operator 'In' is not allowed on a count"), + result.getFaults().toString()); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/RecordingLoggerFactory.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/RecordingLoggerFactory.java new file mode 100644 index 000000000..1f60e09e9 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/RecordingLoggerFactory.java @@ -0,0 +1,147 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.Marker; +import org.slf4j.event.Level; +import org.slf4j.helpers.LegacyAbstractLogger; +import org.slf4j.helpers.MessageFormatter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Keeps every message logged at or above a level, so a test can read what + * the element logged. + */ +final class RecordingLoggerFactory implements ILoggerFactory { + + private final Level threshold; + private final List levels = new ArrayList<>(); + private final List messages = new ArrayList<>(); + + RecordingLoggerFactory(Level threshold) { + this.threshold = threshold; + } + + @Override + public Logger getLogger(String name) { + return new RecordingLogger(name, this); + } + + /** + * @param level the level + * @return every message logged at exactly the level, in order + */ + synchronized List messages(Level level) { + List found = new ArrayList<>(); + for (int i = 0; i < levels.size(); i++) { + if (levels.get(i) == level) { + found.add(messages.get(i)); + } + } + return found; + } + + /** + * @param level the level + * @param start the text the message starts with + * @return every message logged at the level that starts with the text + */ + List messages(Level level, String start) { + List found = new ArrayList<>(); + for (String message : messages(level)) { + if (message.startsWith(start)) { + found.add(message); + } + } + return found; + } + + private boolean enabled(Level level) { + return level.toInt() >= threshold.toInt(); + } + + private synchronized void record(Level level, String message) { + levels.add(level); + messages.add(message); + } + + /** + * A logger that hands every message to the factory. + */ + private static final class RecordingLogger extends LegacyAbstractLogger { + + private static final long serialVersionUID = 1L; + + private final transient RecordingLoggerFactory factory; + + RecordingLogger(String name, RecordingLoggerFactory factory) { + this.name = name; + this.factory = factory; + } + + @Override + public boolean isTraceEnabled() { + return factory.enabled(Level.TRACE); + } + + @Override + public boolean isDebugEnabled() { + return factory.enabled(Level.DEBUG); + } + + @Override + public boolean isInfoEnabled() { + return factory.enabled(Level.INFO); + } + + @Override + public boolean isWarnEnabled() { + return factory.enabled(Level.WARN); + } + + @Override + public boolean isErrorEnabled() { + return factory.enabled(Level.ERROR); + } + + @Override + protected String getFullyQualifiedCallerName() { + return null; + } + + @Override + protected void handleNormalizedLoggingCall( + Level level, + Marker marker, + String messagePattern, + Object[] arguments, + Throwable throwable) { + factory.record(level, + MessageFormatter.basicArrayFormat(messagePattern, arguments)); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SharedCasesTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SharedCasesTests.java new file mode 100644 index 000000000..858202621 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SharedCasesTests.java @@ -0,0 +1,419 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WeightedValue; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.derivedproperty.BuiltInScript; +import fiftyone.pipeline.derivedproperty.DerivedScriptFault; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Runs the case files of the derived-properties repository, which every + * language runs, so the answers and the faults are checked to match case + * for case. The repository is the submodule pipeline/derived-properties. + *

+ * See writing + * and running cases. + */ +public class SharedCasesTests { + + /** + * The largest raw weighting a weighted value carries. + */ + private static final int MAX_RAW_WEIGHTING = 65535; + + /** + * The derived-properties submodule, which a test fails without. + */ + static Path sharedFolder() { + Path folder = Paths.get("..", "derived-properties"); + if (Files.isDirectory(folder.resolve("tests")) == false) { + fail("The derived-properties submodule is missing from " + + folder.toAbsolutePath() + ". Run 'git submodule update " + + "--init pipeline/derived-properties'."); + } + return folder; + } + + /** + * The files in a folder whose names end with the text given, in name + * order. + */ + static List files(Path folder, String glob) throws IOException { + List found = new ArrayList<>(); + try (DirectoryStream stream = + Files.newDirectoryStream(folder, glob)) { + for (Path path : stream) { + found.add(path); + } + } + found.sort((a, b) -> a.getFileName().toString() + .compareTo(b.getFileName().toString())); + return found; + } + + static String read(Path file) throws IOException { + return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } + + @TestFactory + public List sharedCases() throws Exception { + List files = files( + sharedFolder().resolve("tests"), "*.cases.yaml"); + assertFalse(files.isEmpty(), + "There are no case files in the derived-properties submodule."); + List tests = new ArrayList<>(); + for (Path file : files) { + ScriptNode cases = DerivedScriptParser.parse(read(file)); + String script = cases.get("Script").getText(); + BuiltInScript builtIn = BuiltInScript.fromName(script); + if (builtIn == null) { + fail(file + " proves the script " + script + ", which has " + + "no BuiltInScript."); + } + DerivedPropertyElement element = + new DerivedPropertyElementBuilder( + LoggerFactory.getILoggerFactory()) + .addScript(builtIn) + .build(); + String output = element.getScripts().get(0).getOutput().getName(); + for (boolean wrap : new boolean[]{false, true}) { + for (ScriptNode c : cases.get("Cases").getItems()) { + String name = script + "/" + c.get("Name").getText() + + "/wrapped=" + wrap; + tests.add(DynamicTest.dynamicTest(name, () -> + runCase(element, output, c, wrap))); + } + } + } + return tests; + } + + private static void runCase( + DerivedPropertyElement element, + String output, + ScriptNode c, + boolean wrap) throws Exception { + Map> values = new LinkedHashMap<>(); + ScriptNode properties = c.get("Properties"); + for (int i = 0; i < properties.getKeys().size(); i++) { + String property = properties.getKeys().get(i); + int dot = property.indexOf('.'); + String key = property.substring(0, dot).toLowerCase(Locale.ROOT); + Map forKey = values.get(key); + if (forKey == null) { + forKey = new LinkedHashMap<>(); + values.put(key, forKey); + } + forKey.put(property.substring(dot + 1), + caseValue(properties.getValues().get(i), wrap)); + } + AspectPropertyValue got = process(element, values, output); + ScriptNode expect = c.get("Expect"); + ScriptNode value = expect.get("Value"); + if (value != null) { + assertTrue(got.hasValue(), "Expected '" + expectedText(value) + + "' but there was no value. " + got.getNoValueMessage()); + assertEquals(expectedText(value), valueText(got.getValue())); + return; + } + List missing = new ArrayList<>(); + for (ScriptNode name : expect.get("Missing").getItems()) { + missing.add(name.getText()); + } + checkMissing(got, missing, output); + } + + /** + * Run one request through stubs holding the values and the element, + * and give what the element wrote for the output property. + */ + static AspectPropertyValue process( + DerivedPropertyElement element, + Map> values, + String output) throws Exception { + TestPipelineBuilder builder = new TestPipelineBuilder(); + for (StubSourceElement stub : stubsFor(element, values)) { + builder.addFlowElement(stub); + } + try (Pipeline pipeline = builder.addFlowElement(element).build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + Object value = data.get(IDerivedPropertyData.class).get(output); + assertTrue(value instanceof AspectPropertyValue, + output + " is " + value + ", not an AspectPropertyValue"); + return (AspectPropertyValue) value; + } + } + + /** + * A stub for each key the element reads, listing every property read + * under it and holding the values given for it. + */ + static List stubsFor( + DerivedPropertyElement element, + Map> values) { + Map> names = new LinkedHashMap<>(); + for (SourceProperty property : element.sourceProperties()) { + List forKey = names.get(property.getElementKey()); + if (forKey == null) { + forKey = new ArrayList<>(); + names.put(property.getElementKey(), forKey); + } + forKey.add(property.getPropertyName()); + } + List stubs = new ArrayList<>(); + for (Map.Entry> key : names.entrySet()) { + stubs.add(new StubSourceElement( + key.getKey(), key.getValue(), values.get(key.getKey()))); + } + return stubs; + } + + /** + * A value from a case file as a source element holds it. The String + * form is the string, NoValue is a value with no value, and a list of + * Value and Weight pairs is a list of weighted values. + */ + private static Object caseValue(ScriptNode node, boolean wrap) { + switch (node.getKind()) { + case NULL: + return null; + case BOOL: + case NUMBER: + case STRING: + Object value = scalar(node); + return wrap ? new AspectPropertyValueDefault<>(value) : value; + case MAP: + ScriptNode noValue = node.get("NoValue"); + if (noValue != null) { + AspectPropertyValueDefault missing = + new AspectPropertyValueDefault<>(); + missing.setNoValueMessage(Literal.of(noValue).toString()); + return missing; + } + ScriptNode text = node.get("String"); + if (text != null) { + if (text.getKind() == ScriptNode.Kind.NULL) { + return null; + } + String string = Literal.of(text).toString(); + return wrap ? + new AspectPropertyValueDefault<>(string) : string; + } + return new LinkedHashMap<>(); + default: + return weighted(node); + } + } + + /** + * A list whose members are not all weighted values cannot be read + * where one value is needed, whatever it holds. A case gives weights + * of any size, and a raw weighting is 16 bits, so each weight is scaled + * against the largest, which keeps the order. + */ + private static List weighted(ScriptNode node) { + List values = new ArrayList<>(); + List weights = new ArrayList<>(); + double largest = 0; + for (ScriptNode item : node.getItems()) { + if (item.getKind() != ScriptNode.Kind.MAP || + item.get("Value") == null) { + List nulls = new ArrayList<>(); + for (int i = 0; i < node.getItems().size(); i++) { + nulls.add(null); + } + return nulls; + } + ScriptNode weight = item.get("Weight"); + double w = weight == null ? 0 : weight.getNumber(); + values.add(caseValue(item.get("Value"), false)); + weights.add(w); + largest = Math.max(largest, w); + } + List list = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + int raw = largest > 0 && weights.get(i) > 0 ? + (int) Math.round(weights.get(i) / largest * MAX_RAW_WEIGHTING) : + 0; + IWeightedValue member = + new WeightedValue<>(raw, values.get(i)); + list.add(member); + } + return list; + } + + /** + * A scalar with the type as written, a whole number being an Integer + * where it fits and a Long where it does not. + */ + private static Object scalar(ScriptNode node) { + switch (node.getKind()) { + case BOOL: + return node.getBool(); + case NUMBER: + double number = node.getNumber(); + if (DerivedScriptValidator.isWhole(number) && + number >= Integer.MIN_VALUE && + number <= Integer.MAX_VALUE) { + return (int) number; + } + if (DerivedScriptValidator.isWhole(number) && + Math.abs(number) < Long.MAX_VALUE) { + return (long) number; + } + return number; + default: + return node.getText(); + } + } + + /** + * The text of a value, as the case files compare values. + */ + static String valueText(Object value) { + if (value instanceof Double) { + return JsText.number((Double) value); + } + return String.valueOf(value); + } + + /** + * The text of an expected value in a case file. + */ + private static String expectedText(ScriptNode node) { + return Literal.of(node).toString(); + } + + /** + * A missing value names every property that could not be read, so the + * message gives the count and each name. + */ + static void checkMissing( + AspectPropertyValue got, + List missing, + String output) { + assertFalse(got.hasValue(), () -> "Expected no value naming " + + missing + " but got '" + got.getValue() + "'"); + String message = got.getNoValueMessage(); + String noun = missing.size() == 1 ? + "1 source property was not available" : + missing.size() + " source properties were not available"; + String prefix = "Derived property '" + output + + "' has no value because " + noun + ". "; + assertTrue(message.startsWith(prefix), + "The message '" + message + "' does not start '" + prefix + "'"); + String lower = message.toLowerCase(Locale.ROOT); + for (String name : missing) { + assertTrue( + lower.contains("'" + name.toLowerCase(Locale.ROOT) + "' ("), + "The message '" + message + "' does not name " + name); + } + assertTrue(message.endsWith(DerivedPropertyElement.USUAL_CAUSES), + "The message '" + message + "' does not end with the usual " + + "causes"); + } + + @TestFactory + public List rejectedScripts() throws Exception { + List files = files( + sharedFolder().resolve("tests").resolve("invalid"), "*.yaml"); + assertFalse(files.isEmpty(), "There are no rejection cases in the " + + "derived-properties submodule."); + List tests = new ArrayList<>(); + for (Path file : files) { + String fileName = file.getFileName().toString(); + ScriptNode c = DerivedScriptParser.parse(read(file)); + tests.add(DynamicTest.dynamicTest(fileName, () -> + checkRejected(c, fileName))); + } + return tests; + } + + private static void checkRejected(ScriptNode c, String fileName) { + ScriptNode name = c.get("Name"); + DerivedScriptValidator.Result result = DerivedScriptValidator.validate( + c.get("Script").getText(), + name == null ? "" : name.getText(), + fileName); + assertNull(result.getScript(), + "The script validated, and the case says it must not."); + assertFalse(result.getFaults().isEmpty(), + "The script validated, and the case says it must not."); + StringBuilder all = new StringBuilder(); + for (DerivedScriptFault fault : result.getFaults()) { + all.append('\n').append(fault); + } + ScriptNode expect = c.get("Expect"); + ScriptNode paths = expect.get("Paths"); + if (paths != null) { + for (ScriptNode path : paths.getItems()) { + boolean found = false; + for (DerivedScriptFault fault : result.getFaults()) { + found = found || fault.getPath().equals(path.getText()); + } + assertTrue(found, "Expected a fault at '" + path.getText() + + "'. Faults were:" + all); + } + } + ScriptNode mentions = expect.get("Mentions"); + if (mentions != null) { + for (ScriptNode mention : mentions.getItems()) { + boolean found = false; + for (DerivedScriptFault fault : result.getFaults()) { + found = found || + fault.getMessage().contains(mention.getText()); + } + assertTrue(found, "Expected a fault mentioning '" + + mention.getText() + "'. Faults were:" + all); + } + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValuesTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValuesTests.java new file mode 100644 index 000000000..352b92185 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValuesTests.java @@ -0,0 +1,192 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WeightedValue; +import fiftyone.pipeline.core.data.WktString; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * How a source value is converted to the type a script infers, with the + * Java types elements hold. + *

+ * See format + * 1, conversion. + */ +public class SourceValuesTests { + + private static Slot read(Object raw, ValueType type) { + return SourceValues.readValue( + raw, new SourceProperty("device.Value", type)); + } + + private static void assertAbsent(Object raw, ValueType type, String why) { + Slot slot = read(raw, type); + assertFalse(slot.isAvailable(), raw + " was read as " + type); + assertEquals(why, slot.getReason()); + } + + @Test + public void booleans() { + assertTrue(read(true, ValueType.BOOL).getBool()); + assertTrue(read(" TRUE\t", ValueType.BOOL).getBool()); + assertFalse(read("False", ValueType.BOOL).getBool()); + assertEquals("True", read(true, ValueType.STRING).getString()); + assertEquals("False", read(false, ValueType.STRING).getString()); + for (Object raw : new Object[]{"N/A", "Unknown", "", "1", "yes", 1}) { + assertFalse(read(raw, ValueType.BOOL).isAvailable(), + raw + " became a boolean"); + } + assertAbsent("yes", ValueType.BOOL, + "held 'yes' which cannot be read as bool"); + } + + @Test + public void wholeNumbersHaveTheRangeOfInt() { + assertEquals(5, read(5, ValueType.INT).getInt()); + assertEquals(5, read(5L, ValueType.INT).getInt()); + assertEquals(5, read((short) 5, ValueType.INT).getInt()); + assertEquals(5, read((byte) 5, ValueType.INT).getInt()); + assertEquals(5, read(BigInteger.valueOf(5), ValueType.INT).getInt()); + assertEquals(12, read(" +12 ", ValueType.INT).getInt()); + assertEquals(2, read(2.0, ValueType.INT).getInt()); + assertEquals(Integer.MIN_VALUE, + read((long) Integer.MIN_VALUE, ValueType.INT).getInt()); + assertAbsent(3000000000L, ValueType.INT, + "held '3000000000' which cannot be read as int"); + assertAbsent("3000000000", ValueType.INT, + "held '3000000000' which cannot be read as int"); + assertAbsent("99999999999999999999", ValueType.INT, + "held '99999999999999999999' which cannot be read as int"); + assertAbsent(2.5, ValueType.INT, + "held '2.5' which cannot be read as int"); + for (Object raw : new Object[]{"1.0", "Unknown", "", "1e3", "0x10"}) { + assertFalse(read(raw, ValueType.INT).isAvailable(), + raw + " became an int"); + } + } + + @Test + public void doubles() { + assertEquals(1.5, read(1.5f, ValueType.DOUBLE).getDouble()); + assertEquals(3000000000.0, + read(3000000000L, ValueType.DOUBLE).getDouble()); + assertEquals(1000, read("1e3", ValueType.DOUBLE).getDouble()); + assertEquals(0.5, read(".5", ValueType.DOUBLE).getDouble()); + assertEquals(5, read("5.", ValueType.DOUBLE).getDouble()); + assertEquals(-2.5, read(" -2.5E0 ", ValueType.DOUBLE).getDouble()); + for (Object raw : new Object[]{"Unknown", "", "1e400", "NaN", + "Infinity", "0x10", "1,5", Double.NaN, + Double.POSITIVE_INFINITY}) { + assertFalse(read(raw, ValueType.DOUBLE).isAvailable(), + raw + " became a double"); + } + } + + @Test + public void strings() { + assertEquals(" as written ", + read(" as written ", ValueType.STRING).getString()); + assertEquals("8", read(8, ValueType.STRING).getString()); + assertEquals("8", read(8.0, ValueType.STRING).getString()); + assertEquals("8.5", read(8.5, ValueType.STRING).getString()); + assertEquals("1e+21", read(1e21, ValueType.STRING).getString()); + assertEquals("POINT (1 2)", + read(new WktString("POINT (1 2)"), ValueType.STRING).getString()); + } + + @Test + public void aValueWithNoValueCarriesItsMessage() { + AspectPropertyValueDefault noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage("The JavaScript has not run yet."); + assertAbsent(noValue, ValueType.BOOL, "element 'device' has no " + + "value for 'Value': The JavaScript has not run yet."); + noValue.setNoValueMessage(""); + assertAbsent(noValue, ValueType.BOOL, "element 'device' has no " + + "value for 'Value': property not present on this request"); + assertTrue(read(new AspectPropertyValueDefault<>("True"), + ValueType.BOOL).getBool()); + } + + @Test + public void aWeightedListGivesTheHeaviestValue() { + List> weighted = + Arrays.>asList( + new WeightedValue<>(100, 2), + new WeightedValue<>(900, 9), + new WeightedValue<>(900, 7)); + assertEquals(9, read(weighted, ValueType.INT).getInt()); + assertEquals(9, read(new AspectPropertyValueDefault<>(weighted), + ValueType.INT).getInt()); + assertEquals(4, read(new WeightedValue<>(1, 4), ValueType.INT) + .getInt()); + assertAbsent(Arrays.asList("a", "b"), ValueType.STRING, + "held a list where a single value is needed"); + assertAbsent(Collections.emptyList(), ValueType.STRING, + "held a list where a single value is needed"); + } + + @Test + public void aMissingPropertyIsNotPresent() { + Slot slot = SourceValues.read(null, + new SourceProperty("Device.Value", ValueType.BOOL)); + assertFalse(slot.isAvailable()); + assertEquals("element 'Device' has no value for 'Value': property " + + "not present on this request", slot.getReason()); + } + + @Test + public void numbersPrintAsJavaScriptPrintsThem() { + assertEquals("0", JsText.number(-0.0)); + assertEquals("1", JsText.number(1.0)); + assertEquals("0.1", JsText.number(0.1)); + assertEquals("123456789012345680000", + JsText.number(1.2345678901234568E20)); + assertEquals("1e+21", JsText.number(1e21)); + assertEquals("1.5e-7", JsText.number(1.5e-7)); + assertEquals("0.000001", JsText.number(1e-6)); + assertEquals("-2.5", JsText.number(-2.5)); + assertEquals("NaN", JsText.number(Double.NaN)); + assertEquals("-Infinity", JsText.number(Double.NEGATIVE_INFINITY)); + } + + @Test + public void stringsQuoteAsJsonStringifyQuotesThem() { + assertEquals("\"a\\\"b\\\\c\\n\\u0001\u00e9\"", + JsText.quote("a\"b\\c\n\u0001\u00e9")); + assertEquals("\"\\ud800\"", JsText.quote("\ud800")); + assertEquals("\"\ud83d\ude00\"", JsText.quote("\ud83d\ude00")); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElement.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElement.java new file mode 100644 index 000000000..9f3e9bb92 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElement.java @@ -0,0 +1,126 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Stands in for an element that supplies source properties, such as device + * detection. It lists every property it is given a name for, as an engine + * lists the properties it can supply, and writes the values it holds under + * its Element Data key. A name with no value is listed but not written. + */ +public final class StubSourceElement + extends FlowElementBase { + + private final String key; + private final List properties; + private final Map values; + private final boolean addData; + + /** + * @param key the Element Data key + * @param names the properties listed + * @param values the values written, by property name + * @param addData false to write no Element Data at all + */ + StubSourceElement( + String key, + Collection names, + Map values, + boolean addData) { + super(LoggerFactory.getLogger(StubSourceElement.class), + (flowData, element) -> new ElementDataBase( + LoggerFactory.getLogger(StubSourceElement.class), + flowData) { + }); + this.key = key; + this.values = values == null ? + Collections.emptyMap() : + new LinkedHashMap<>(values); + List listed = new ArrayList<>(); + for (String name : names) { + listed.add(new ElementPropertyMetaDataDefault( + name, this, "", Object.class, true)); + } + this.properties = Collections.unmodifiableList(listed); + this.addData = addData; + } + + StubSourceElement( + String key, + Collection names, + Map values) { + this(key, names, values, true); + } + + @Override + public String getElementDataKey() { + return key; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(Collections.emptyList()); + } + + @Override + public List getProperties() { + return properties; + } + + @Override + protected void processInternal(FlowData data) { + if (addData == false) { + return; + } + ElementData elementData = + data.getOrAdd(getTypedDataKey(), getDataFactory()); + for (Map.Entry value : values.entrySet()) { + elementData.put(value.getKey(), value.getValue()); + } + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElementBuilder.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElementBuilder.java new file mode 100644 index 000000000..1acb61d28 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElementBuilder.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Builds a {@link StubSourceElement}, so that a configuration file can put + * a source element in front of a Derived Property Element. + */ +@ElementBuilder +public class StubSourceElementBuilder { + + private String key = ""; + private final Map values = new LinkedHashMap<>(); + + /** + * @param key the Element Data key the element writes under + * @return this builder + */ + public StubSourceElementBuilder setKey(String key) { + this.key = key; + return this; + } + + /** + * @param values the properties and their string values, written as + * Name=Value and separated by commas + * @return this builder + */ + public StubSourceElementBuilder setValues(String values) { + for (String pair : values.split(",")) { + int equals = pair.indexOf('='); + this.values.put( + pair.substring(0, equals).trim(), + pair.substring(equals + 1).trim()); + } + return this; + } + + /** + * @return a new element + */ + public StubSourceElement build() { + return new StubSourceElement(key, values.keySet(), values); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/TestPipelineBuilder.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/TestPipelineBuilder.java new file mode 100644 index 000000000..624e73a81 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/TestPipelineBuilder.java @@ -0,0 +1,35 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.flowelements.PipelineBuilderBase; + +/** + * Builds a Pipeline from elements made in code. PipelineBuilder searches + * the class path for element builders each time one is constructed, which + * only a configuration file needs, so the many Pipelines the cases build + * use this instead. + */ +final class TestPipelineBuilder + extends PipelineBuilderBase { +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/pom.xml b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/pom.xml new file mode 100644 index 000000000..2847c5aee --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/pom.xml @@ -0,0 +1,50 @@ + + + + + + pipeline.developer-examples + com.51degrees + 4.5.127-SNAPSHOT + + 4.0.0 + + pipeline.developerexamples.derivedproperty + 51Degrees :: Pipeline :: Developer Examples :: Derived Property + https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples-pipeline.developer-examples.derivedproperty-pom.xml&utm_term=url + + + + ${project.groupId} + ${project.version} + pipeline.derivedproperty + + + + org.junit.jupiter + junit-jupiter + + + diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/ExampleTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/ExampleTests.java new file mode 100644 index 000000000..a98f07336 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/ExampleTests.java @@ -0,0 +1,96 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Runs the example and checks it shows the four things the specification + * asks of it, being a script held as text, a script read from a file, the + * same Pipeline built from a configuration file, and a property with no + * value naming the source property that was missing. + */ +public class ExampleTests { + + private static final String NO_VALUE = + "no value, because Derived property '%s' has no value because 1 " + + "source property was not available. 'session.PointerMoved' " + + "(element 'session' has no value for 'PointerMoved': the " + + "request did not carry the evidence 'query.pointermoved')."; + + private static String run() throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (PrintStream output = new PrintStream(bytes, true, "UTF-8")) { + new Main.Example().run(output); + } + return new String(bytes.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } + + private static int count(String text, String part) { + int count = 0; + for (int at = text.indexOf(part); at >= 0; + at = text.indexOf(part, at + 1)) { + count++; + } + return count; + } + + @Test + public void Example_ShowsTheFourThings() throws Exception { + String output = run(); + int code = output.indexOf("1. Configured with the builder in code"); + int file = output.indexOf("2. The same Pipeline built from " + + "derived-property.xml"); + assertTrue(code >= 0 && file > code, output); + String fromCode = output.substring(code, file); + String fromFile = output.substring(file); + + // The script held as text and the script read from a file each + // give a value from the source properties. + assertTrue(fromCode.contains( + "derived.ReaderEngagementIsHigh true"), fromCode); + assertTrue(fromCode.contains( + "derived.ReaderEngagement High"), fromCode); + // The configuration file builds the same Pipeline with the file. + assertTrue(fromFile.contains( + "derived.ReaderEngagement High"), fromFile); + assertTrue(fromFile.contains("The definition the script " + + "'ReaderEngagement' version 1.0.0 carries."), fromFile); + // A source property that is not available leaves no value, and + // the message names it. + assertEquals(1, count(fromCode, + String.format(NO_VALUE, "ReaderEngagementIsHigh")), fromCode); + assertEquals(1, count(fromCode, + String.format(NO_VALUE, "ReaderEngagement")), fromCode); + assertEquals(1, count(fromFile, + String.format(NO_VALUE, "ReaderEngagement")), fromFile); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/FindOutMoreTests.java new file mode 100644 index 000000000..9a7b110bb --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/FindOutMoreTests.java @@ -0,0 +1,103 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the + * repositories. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String[] REPOSITORIES = { + "https://github.com/51Degrees/51degrees-java", + "https://github.com/51Degrees/derived-properties", + "https://github.com/51Degrees/specifications"}; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("derived.ReaderEngagement"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + for (String repository : REPOSITORIES) { + assertTrue(section.contains(repository), + repository + " is not listed:\n" + section); + } + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pom.xml b/pipeline/pipeline.developer-examples/pom.xml index 08197b56e..b30007c94 100644 --- a/pipeline/pipeline.developer-examples/pom.xml +++ b/pipeline/pipeline.developer-examples/pom.xml @@ -44,6 +44,7 @@ pipeline.developer-examples.clientside-element-mvc pipeline.developer-examples.usage-sharing pipeline.developer-examples.fodid + pipeline.developer-examples.derivedproperty true diff --git a/pipeline/pom.xml b/pipeline/pom.xml index 533f13e51..744b35aac 100644 --- a/pipeline/pom.xml +++ b/pipeline/pom.xml @@ -54,6 +54,7 @@ pipeline.cloudrequestengine pipeline.engines.fiftyone pipeline.translation + pipeline.derivedproperty pipeline.javascriptbuilder pipeline.jsonbuilder pipeline.web.packages @@ -74,6 +75,7 @@ 20231013 0.11.7 1.10.0 + 2.7 3.0.1 3.0.2 @@ -115,6 +117,11 @@ json ${org.json.version} + + org.yaml + snakeyaml + ${snakeyaml.version} + From dcbab24c966179454526338cc312d1db6c442169 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 17:24:15 +0000 Subject: [PATCH 2/3] Add the Derived Property Element and its example pipeline.derivedproperty is the Derived Property Element of the specification. It is a Flow Element rather than an Aspect Engine, holds no data file, makes no network request and accepts no evidence. Every instance writes into the Element Data under the key derived, as every Translation Engine shares translation. DerivedPropertyElementBuilder takes scripts shipped in the package (BuiltInScript), files on disk with wildcards in the file name, and text from code, and a configuration file names it as DerivedPropertyElement with the Scripts and ScriptFiles build parameters. Each script is parsed with the compose API of SnakeYAML, so every node keeps its line, and plain scalars are resolved by the YAML 1.2 core schema. The validator follows tools/validate.mjs in the derived-properties repository, with the same paths and messages, and reports every fault in every script at once. Per request the element reads each source property once, converts it to the type the script infers without coercing loosely, then runs the checks and the rules. Where a source property is not available the output has no value, using the existing AspectPropertyValue mechanism, and the message names every property that could not be read, in the wording of tools/evaluate.mjs. A request with no Element Data under a key a script reads is logged once per script. When the Pipeline adds the element it checks that some element lists each source property, without judging where that element sits, and that no two Derived Property Elements produce the same property. The developer example builds one Pipeline in code, with a script held as text and a script read from a file beside the example, and the same Pipeline from a configuration file, and shows the message a caller sees when a source property is not available. It ends with a Find out more section. --- AGENTS.md | 8 + pipeline/README.md | 1 + .../derivedproperty/BuiltInScript.java | 127 ++ .../derivedproperty/DerivedScriptFault.java | 112 ++ .../DerivedScriptValidationException.java | 66 + .../data/DerivedPropertyData.java | 81 ++ .../data/IDerivedPropertyData.java | 50 + .../flowelements/CanonicalJson.java | 302 +++++ .../flowelements/CompiledScript.java | 71 + .../flowelements/Condition.java | 340 +++++ .../flowelements/DerivedOutputMetaData.java | 204 +++ .../flowelements/DerivedPropertyElement.java | 566 ++++++++ .../DerivedPropertyElementBuilder.java | 341 +++++ .../flowelements/DerivedPropertyMetaData.java | 97 ++ .../flowelements/DerivedScript.java | 185 +++ .../flowelements/DerivedScriptParser.java | 287 ++++ .../flowelements/DerivedScriptValidator.java | 1168 +++++++++++++++++ .../flowelements/DerivedValueMetaData.java | 52 + .../flowelements/IDerivedPropertyElement.java | 45 + .../derivedproperty/flowelements/JsText.java | 181 +++ .../derivedproperty/flowelements/Literal.java | 93 ++ .../flowelements/Operator.java | 106 ++ .../flowelements/OutputModel.java | 230 ++++ .../flowelements/ScriptNode.java | 193 +++ .../flowelements/ScriptParseException.java | 45 + .../derivedproperty/flowelements/Slot.java | 108 ++ .../flowelements/SourceProperty.java | 93 ++ .../flowelements/SourceRead.java | 52 + .../flowelements/SourceValues.java | 365 ++++++ .../flowelements/ValueType.java | 85 ++ .../derivedproperty/Main.java | 374 ++++++ .../flowelements/SessionElement.java | 164 +++ .../flowelements/SessionElementBuilder.java | 51 + .../src/main/resources/ReaderEngagement.yaml | 73 ++ .../src/main/resources/derived-property.xml | 57 + 35 files changed, 6373 insertions(+) create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/BuiltInScript.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptFault.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptValidationException.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/DerivedPropertyData.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/IDerivedPropertyData.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CanonicalJson.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CompiledScript.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Condition.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedOutputMetaData.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElement.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementBuilder.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyMetaData.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScript.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParser.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptValidator.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedValueMetaData.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/IDerivedPropertyElement.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/JsText.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Literal.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Operator.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/OutputModel.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptNode.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptParseException.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Slot.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceProperty.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceRead.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValues.java create mode 100644 pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ValueType.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElement.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElementBuilder.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml diff --git a/AGENTS.md b/AGENTS.md index df51ec15e..1af9b389d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -294,6 +294,14 @@ The Windows pre-build scripts take a toolset and SDK version from the pom. JavaScript templates and the OWID library are submodules inside `pipeline`, where the Pipeline API expects them. +`pipeline/derived-properties` is the +[derived-properties](https://github.com/51Degrees/derived-properties) +repository. `pipeline.derivedproperty` packages the scripts in its `scripts` +folder as resources, so a script added there needs a `BuiltInScript` +constant, which `BuiltInScriptTests` checks. The package's tests run every +case file and every rejection case in its `tests` folder, so its answers and +its faults are checked against the other languages'. + ## Releases Pull requests go to `rc`, not `main`, and stay drafts until they have been diff --git a/pipeline/README.md b/pipeline/README.md index 66b7d319d..b2c7df3c1 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -28,6 +28,7 @@ but additional caution should be applied. - pipeline.engines - Shared functionality that is available to all 'engines' (specialized 'flow elements') - pipeline.engines.fiftyone - Functionality that is specific to 51Degrees engines. - pipeline.cloudrequestengine - An engine that is used to retrieve data from 51Degrees' cloud API. +- pipeline.derivedproperty - The Derived Property Element, which produces new properties from the properties other elements produce, following rules written in a script. ## Installation diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/BuiltInScript.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/BuiltInScript.java new file mode 100644 index 000000000..5990e24cd --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/BuiltInScript.java @@ -0,0 +1,127 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; + +/** + * Names every script shipped in this package. The scripts are the + * {@code scripts} folder of the derived-properties repository, and each + * constant is named exactly as its script is, so {@link #name()} is the + * script's name. + */ +public enum BuiltInScript { + + /** + * The confidence that the request came from a device being used by a + * human who is viewing the page, being High, Medium, Low or Unknown. + */ + HumanConfidence, + + /** + * Whether the General Data Protection Regulation, or the United + * Kingdom's version of it, is likely to apply to the person making the + * request, judged from ip.CountryCode. + */ + IsGdpr; + + /** + * The folder on the class path the scripts are packaged in. + */ + public static final String RESOURCE_FOLDER = + "/fiftyone/pipeline/derivedproperty/scripts/"; + + /** + * The file extensions a script is read from, in the order tried. + */ + private static final String[] EXTENSIONS = {".yaml", ".yml", ".json"}; + + /** + * The size of each read from the packaged script. + */ + private static final int BUFFER_BYTES = 8192; + + /** + * The text of the script, read from the package. + * @return the text, or null where the package was built without the + * script + */ + public String readText() { + for (String extension : EXTENSIONS) { + try (InputStream stream = BuiltInScript.class.getResourceAsStream( + RESOURCE_FOLDER + name() + extension)) { + if (stream != null) { + return readAll(stream); + } + } catch (IOException e) { + throw new UncheckedIOException( + "The built-in script '" + name() + "' cannot be read.", + e); + } + } + return null; + } + + /** + * The built-in script with the name given, matched exactly. + * @param name the script's name + * @return the script, or null where no built-in script has the name + */ + public static BuiltInScript fromName(String name) { + for (BuiltInScript script : values()) { + if (script.name().equals(name)) { + return script; + } + } + return null; + } + + /** + * Every built-in script's name, for a message. + * @return the names, separated by a comma and a space + */ + public static String names() { + StringBuilder names = new StringBuilder(); + for (BuiltInScript script : values()) { + if (names.length() > 0) { + names.append(", "); + } + names.append(script.name()); + } + return names.toString(); + } + + private static String readAll(InputStream stream) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[BUFFER_BYTES]; + int read; + while ((read = stream.read(buffer)) > 0) { + bytes.write(buffer, 0, read); + } + return new String(bytes.toByteArray(), StandardCharsets.UTF_8); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptFault.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptFault.java new file mode 100644 index 000000000..0b5bdc6a0 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptFault.java @@ -0,0 +1,112 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty; + +import java.io.Serializable; + +/** + * One thing wrong with a derived property script. + */ +public final class DerivedScriptFault implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String script; + private final String source; + private final String path; + private final int line; + private final String message; + + /** + * Construct a fault. + * @param script the name the script was given, or empty where it had + * none + * @param source the built-in script's name, the file path, or "code" + * @param path the place in the document, such as Rules[3].When.All[1], + * or empty for the document as a whole + * @param line the one based line, or 0 where the parser gave none + * @param message what is wrong + */ + public DerivedScriptFault( + String script, + String source, + String path, + int line, + String message) { + this.script = script == null ? "" : script; + this.source = source == null ? "" : source; + this.path = path == null ? "" : path; + this.line = line; + this.message = message; + } + + /** + * @return the name the script was given, or empty where it had none + */ + public String getScript() { + return script; + } + + /** + * @return the built-in script's name, the file path, or "code" + */ + public String getSource() { + return source; + } + + /** + * @return the place in the document, such as Rules[3].When.All[1], or + * empty for the document as a whole + */ + public String getPath() { + return path; + } + + /** + * @return the one based line, or 0 where the parser gave none + */ + public int getLine() { + return line; + } + + /** + * @return what is wrong + */ + public String getMessage() { + return message; + } + + /** + * The fault as one line, giving the script, the source, the line where + * known, the path and the message. + * @return the fault as one line + */ + @Override + public String toString() { + return (script.isEmpty() ? "script" : script) + + " (" + source + ")" + + (line > 0 ? " line " + line : "") + + " at " + (path.isEmpty() ? "(document)" : path) + + ": " + message; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptValidationException.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptValidationException.java new file mode 100644 index 000000000..3b0681342 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptValidationException.java @@ -0,0 +1,66 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Every fault found in the scripts given to a Derived Property Element + * builder, raised once after every script has been checked. The message + * lists every fault, one per line. + */ +public class DerivedScriptValidationException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final DerivedScriptFault[] faults; + + /** + * Construct the exception. + * @param faults every fault found + */ + public DerivedScriptValidationException(List faults) { + super(message(faults)); + this.faults = faults.toArray(new DerivedScriptFault[0]); + } + + /** + * @return every fault found, in the order found + */ + public List getFaults() { + return Collections.unmodifiableList(Arrays.asList(faults)); + } + + private static String message(List faults) { + StringBuilder message = new StringBuilder(); + for (DerivedScriptFault fault : faults) { + if (message.length() > 0) { + message.append('\n'); + } + message.append(fault); + } + return message.toString(); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/DerivedPropertyData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/DerivedPropertyData.java new file mode 100644 index 000000000..83b56eca8 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/DerivedPropertyData.java @@ -0,0 +1,81 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.data; + +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import org.slf4j.Logger; + +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * The Element Data of the Derived Property Elements. Every Derived Property + * Element in a Pipeline writes into the one instance on a request, and + * elements in a parallel group write at the same time, so the values are + * held in a concurrent map whose keys are matched without regard to case. + */ +public class DerivedPropertyData + extends ElementDataBase + implements IDerivedPropertyData { + + /** + * Construct the Element Data for one request. + * @param logger used for logging + * @param flowData the request the data belongs to + */ + public DerivedPropertyData(Logger logger, FlowData flowData) { + super( + logger, + flowData, + new ConcurrentSkipListMap( + String.CASE_INSENSITIVE_ORDER)); + } + + @Override + @SuppressWarnings("unchecked") + public AspectPropertyValue getAs( + String propertyName, + Class valueType) { + Object value = get(propertyName); + if (value instanceof AspectPropertyValue == false) { + throw new PropertyMissingException( + MissingPropertyReason.Unknown, + propertyName, + "There is no derived property '" + propertyName + "'. A " + + "derived property is named by the Output.Name of a " + + "script the Derived Property Element holds."); + } + AspectPropertyValue result = (AspectPropertyValue) value; + if (result.hasValue() && + valueType.isInstance(result.getValue()) == false) { + throw new ClassCastException( + "The derived property '" + propertyName + "' holds a " + + result.getValue().getClass().getSimpleName() + + ", not a " + valueType.getSimpleName() + "."); + } + return (AspectPropertyValue) result; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/IDerivedPropertyData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/IDerivedPropertyData.java new file mode 100644 index 000000000..561d50a7e --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/IDerivedPropertyData.java @@ -0,0 +1,50 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.data; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; + +/** + * The Element Data every Derived Property Element writes under the key + * {@code derived}. It holds one {@link AspectPropertyValue} per script, + * keyed by the script's output property name, matched without regard to + * case. + */ +public interface IDerivedPropertyData extends ElementData { + + /** + * Get a derived property's value. A value that has no value carries a + * message naming every source property that could not be read, and + * {@link AspectPropertyValue#getValue()} then throws with that message. + * @param propertyName the output property name of a script + * @param valueType the type the script gives, being {@link String}, + * {@link Boolean}, {@link Integer} or {@link Double} + * @param the value type + * @return the value + * @throws fiftyone.pipeline.engines.exceptions.PropertyMissingException + * where no script produces the property + * @throws ClassCastException where the value is not of the type given + */ + AspectPropertyValue getAs(String propertyName, Class valueType); +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CanonicalJson.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CanonicalJson.java new file mode 100644 index 000000000..721690a99 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CanonicalJson.java @@ -0,0 +1,302 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Prints a model as canonical JSON, with PascalCase keys, two space indent, + * literal types preserved, and the inferred types and computed dependencies + * included, as tools/canonical.mjs in the derived-properties repository + * does. The debug log shows this text, so anyone holding the log can + * reconstruct what was evaluated, and a YAML script and the JSON that + * mirrors it print the same. + */ +final class CanonicalJson { + + private static final String INDENT = " "; + + /** + * JSON text written as it is. + */ + private static final class Raw { + private final String text; + + Raw(String text) { + this.text = text; + } + } + + private CanonicalJson() { + } + + /** + * Print a model. + * @param script the model + * @return the canonical JSON, without a trailing new line + */ + static String print(DerivedScript script) { + StringBuilder text = new StringBuilder(); + write(text, object(script), 0); + return text.toString(); + } + + private static Map object(DerivedScript script) { + Map result = new LinkedHashMap<>(); + result.put("Format", new Raw("1")); + result.put("Name", script.getName()); + result.put("Version", script.getVersion()); + if (script.isDeprecated()) { + result.put("Deprecated", Boolean.TRUE); + result.put("DeprecationNote", script.getDeprecationNote()); + } + result.put("Output", output(script.getOutput())); + + // The inferred type of every source property, which the file does + // not carry because the literals compared with it give it. + Map properties = new LinkedHashMap<>(); + for (SourceProperty property : script.getProperties()) { + Map type = new LinkedHashMap<>(); + type.put("Type", property.getType().toString()); + properties.put(property.getName(), type); + } + result.put("Properties", properties); + + if (script.getChecks().isEmpty() == false) { + Map checks = new LinkedHashMap<>(); + for (DerivedScript.NamedCheck check : script.getChecks()) { + checks.put(check.getName(), condition(check.getCondition())); + } + result.put("Checks", checks); + } + List rules = new ArrayList<>(); + for (DerivedScript.Rule rule : script.getRules()) { + Map entry = new LinkedHashMap<>(); + if (rule.getWhen() == null) { + entry.put("Else", literal(rule.getValue())); + } else { + entry.put("When", condition(rule.getWhen())); + entry.put("Then", literal(rule.getValue())); + } + rules.add(entry); + } + result.put("Rules", rules); + return result; + } + + private static Map output(OutputModel output) { + Map result = new LinkedHashMap<>(); + result.put("Name", output.getName()); + if (output.getDescription().isEmpty() == false) { + result.put("Description", output.getDescription()); + } + if (output.getValueType() != null) { + result.put("ValueType", output.getValueType().toString()); + } + putIfGiven(result, "StoredValueType", output.getStoredValueType()); + putIfGiven(result, "DefaultValue", output.getDefaultValue()); + result.put("IsList", output.isList()); + putIfGiven(result, "IsMandatory", output.getIsMandatory()); + putIfGiven(result, "IsObsolete", output.getIsObsolete()); + putIfGiven(result, "Category", output.getCategory()); + putIfGiven(result, "IsPopular", output.getIsPopular()); + putIfGiven(result, "ExportValues", output.getExportValues()); + putIfGiven(result, "Url", output.getUrl()); + if (output.getDisplayOrder() != null) { + result.put("DisplayOrder", + new Raw(JsText.number(output.getDisplayOrder()))); + } + if (output.getPropertyId() != null) { + result.put("PropertyId", + new Raw(JsText.number(output.getPropertyId()))); + } + if (output.getVendorIds() != null) { + result.put("VendorIds", node(output.getVendorIds())); + } + if (output.getDependencies() != null) { + result.put("Dependencies", node(output.getDependencies())); + } else { + result.put("Dependencies", + new ArrayList(output.getComputed())); + } + if (output.getValues() != null) { + List values = new ArrayList<>(); + for (OutputModel.ValueEntry value : output.getValues()) { + Map entry = new LinkedHashMap<>(); + entry.put("Name", literal(value.getName())); + if (value.getDescription() != null) { + entry.put("Description", value.getDescription()); + } + values.add(entry); + } + result.put("Values", values); + } + return result; + } + + private static void putIfGiven( + Map result, + String key, + Object value) { + if (value != null) { + result.put(key, value); + } + } + + private static Map condition(Condition condition) { + Map result = new LinkedHashMap<>(); + switch (condition.getKind()) { + case COMPARE: + result.put("Property", condition.getProperty()); + result.put(condition.getOperator().toString(), + node(condition.getOperand())); + break; + case CHECK: + result.put("Check", condition.getCheck()); + break; + case AGGREGATE: + result.put( + condition.isCountPassed() ? "Passed" : "Failed", + condition.getGroupNames() == null ? + "Checks" : + new ArrayList(condition.getGroupNames())); + result.put(condition.getOperator().toString(), + new Raw(JsText.number(condition.getCount()))); + break; + case ALL: + case ANY: + List items = new ArrayList<>(); + for (int i = 0; i < condition.itemCount(); i++) { + items.add(condition(condition.item(i))); + } + result.put( + condition.getKind() == Condition.Kind.ALL ? "All" : "Any", + items); + break; + case NOT: + result.put("Not", condition(condition.item(0))); + break; + default: + break; + } + return result; + } + + private static Object literal(Literal literal) { + switch (literal.getKind()) { + case BOOL: + return literal.getBool(); + case NUMBER: + return new Raw(JsText.number(literal.getNumber())); + default: + return literal.getText(); + } + } + + /** + * A value from the script with its types as written. + */ + private static Object node(ScriptNode node) { + switch (node.getKind()) { + case NULL: + case BOOL: + case NUMBER: + return new Raw(DerivedScriptValidator.jsonOf(node)); + case STRING: + return node.getText(); + case LIST: + List items = new ArrayList<>(); + for (ScriptNode item : node.getItems()) { + items.add(node(item)); + } + return items; + default: + Map result = new LinkedHashMap<>(); + for (int i = 0; i < node.getKeys().size(); i++) { + result.put( + node.getKeys().get(i), + node(node.getValues().get(i))); + } + return result; + } + } + + /** + * Write a value as JSON.stringify does with an indent of two. + */ + private static void write(StringBuilder text, Object value, int depth) { + if (value instanceof Raw) { + text.append(((Raw) value).text); + } else if (value instanceof String) { + text.append(JsText.quote((String) value)); + } else if (value instanceof Boolean) { + text.append(((Boolean) value) ? "true" : "false"); + } else if (value instanceof List) { + List items = (List) value; + if (items.isEmpty()) { + text.append("[]"); + return; + } + text.append('['); + for (int i = 0; i < items.size(); i++) { + if (i > 0) { + text.append(','); + } + newLine(text, depth + 1); + write(text, items.get(i), depth + 1); + } + newLine(text, depth); + text.append(']'); + } else if (value instanceof Map) { + Map entries = (Map) value; + if (entries.isEmpty()) { + text.append("{}"); + return; + } + text.append('{'); + boolean first = true; + for (Map.Entry entry : entries.entrySet()) { + if (first == false) { + text.append(','); + } + first = false; + newLine(text, depth + 1); + text.append(JsText.quote((String) entry.getKey())) + .append(": "); + write(text, entry.getValue(), depth + 1); + } + newLine(text, depth); + text.append('}'); + } + } + + private static void newLine(StringBuilder text, int depth) { + text.append('\n'); + for (int i = 0; i < depth; i++) { + text.append(INDENT); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CompiledScript.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CompiledScript.java new file mode 100644 index 000000000..31e0d7f23 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CompiledScript.java @@ -0,0 +1,71 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A script with each of its source properties resolved to one of the + * element's reads. + */ +final class CompiledScript { + + private final DerivedScript model; + private final int[] reads; + + /** + * Records, without a lock, that the error for a request with no Element + * Data under a key the script reads has been logged. It is the only + * state shared between requests. + */ + private final AtomicBoolean logged = new AtomicBoolean(false); + + /** + * @param model the validated script + * @param reads for each of the script's source properties, the index of + * the element's read that supplies it + */ + CompiledScript(DerivedScript model, int[] reads) { + this.model = model; + this.reads = reads.clone(); + } + + DerivedScript getModel() { + return model; + } + + int readCount() { + return reads.length; + } + + int read(int i) { + return reads[i]; + } + + /** + * @return true the first time it is called, and false after that + */ + boolean firstMissingData() { + return logged.compareAndSet(false, true); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Condition.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Condition.java new file mode 100644 index 000000000..a3edd1ee0 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Condition.java @@ -0,0 +1,340 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Collections; +import java.util.List; + +/** + * One condition of a script, which is always true or false, because a + * script only runs once every source property it names has been read. + */ +final class Condition { + + /** + * The forms a condition takes. + */ + enum Kind { COMPARE, CHECK, AGGREGATE, ALL, ANY, NOT } + + private final Kind kind; + + // A comparison of a source property with a literal. + private final int slot; + private final String property; + private final Operator operator; + private final ScriptNode operand; + private final ValueType type; + private final Literal literal; + private final Literal[] literals; + + // A reference to a check by name. + private final String check; + private final int index; + + // A count of the checks in a group that passed or failed. A null group + // is every check. + private final boolean countPassed; + private final int[] group; + private final List groupNames; + private final long count; + + // The members of All and Any, or the one condition of Not. + private final Condition[] items; + + private Condition( + Kind kind, + int slot, + String property, + Operator operator, + ScriptNode operand, + ValueType type, + Literal literal, + Literal[] literals, + String check, + int index, + boolean countPassed, + int[] group, + List groupNames, + long count, + Condition[] items) { + this.kind = kind; + this.slot = slot; + this.property = property; + this.operator = operator; + this.operand = operand; + this.type = type; + this.literal = literal; + this.literals = literals; + this.check = check; + this.index = index; + this.countPassed = countPassed; + this.group = group; + this.groupNames = groupNames; + this.count = count; + this.items = items; + } + + /** + * What a condition that could not be read becomes. A fault has been + * recorded by then, so the model is discarded unevaluated. + */ + static Condition faulted() { + return compound(Kind.ALL, new Condition[0]); + } + + static Condition comparison( + int slot, + String property, + Operator operator, + ScriptNode operand, + ValueType type) { + Literal literal = Literal.NONE; + Literal[] literals = new Literal[0]; + if (operand.getKind() == ScriptNode.Kind.LIST) { + literals = new Literal[operand.getItems().size()]; + for (int i = 0; i < literals.length; i++) { + literals[i] = Literal.of(operand.getItems().get(i)); + } + } else { + literal = Literal.of(operand); + } + return new Condition(Kind.COMPARE, slot, property, operator, operand, + type, literal, literals, null, -1, false, null, null, 0, null); + } + + static Condition checkReference(String check, int index) { + return new Condition(Kind.CHECK, -1, null, null, null, null, + Literal.NONE, null, check, index, false, null, null, 0, null); + } + + /** + * @param countPassed true for Passed, false for Failed + * @param group the indexes of the checks counted, or null for every + * check + * @param groupNames the names of the checks counted, or null for every + * check + * @param operator the comparison + * @param operand the whole number compared with, as written + */ + static Condition aggregate( + boolean countPassed, + int[] group, + List groupNames, + Operator operator, + ScriptNode operand) { + return new Condition(Kind.AGGREGATE, -1, null, operator, operand, + ValueType.INT, Literal.NONE, null, null, -1, countPassed, group, + groupNames == null ? + null : Collections.unmodifiableList(groupNames), + (long) operand.getNumber(), null); + } + + /** + * @param kind ALL, ANY or NOT + * @param items the members, or the one condition of Not + */ + static Condition compound(Kind kind, Condition[] items) { + return new Condition(kind, -1, null, null, null, null, Literal.NONE, + null, null, -1, false, null, null, 0, items); + } + + Kind getKind() { + return kind; + } + + String getProperty() { + return property; + } + + Operator getOperator() { + return operator; + } + + /** + * @return the literal compared with, as written in the script + */ + ScriptNode getOperand() { + return operand; + } + + String getCheck() { + return check; + } + + boolean isCountPassed() { + return countPassed; + } + + /** + * @return the names of the checks counted, or null for every check + */ + List getGroupNames() { + return groupNames; + } + + long getCount() { + return count; + } + + /** + * @return the number of members of All or Any, or 1 for Not + */ + int itemCount() { + return items.length; + } + + Condition item(int i) { + return items[i]; + } + + /** + * Evaluate the condition. Every slot is available. + * @param slots the source properties, read for this script + * @param checks the result of every check, in the order defined + * @return true or false + */ + boolean test(Slot[] slots, boolean[] checks) { + switch (kind) { + case COMPARE: + return compare(slots[slot]); + case CHECK: + return checks[index]; + case AGGREGATE: + return compareCount(countOf(checks)); + case ALL: + for (Condition item : items) { + if (item.test(slots, checks) == false) { + return false; + } + } + return true; + case ANY: + for (Condition item : items) { + if (item.test(slots, checks)) { + return true; + } + } + return false; + case NOT: + return items[0].test(slots, checks) == false; + default: + return false; + } + } + + private long countOf(boolean[] checks) { + long counted = 0; + if (group == null) { + for (boolean passed : checks) { + if (passed == countPassed) { + counted++; + } + } + } else { + for (int member : group) { + if (checks[member] == countPassed) { + counted++; + } + } + } + return counted; + } + + private boolean compareCount(long counted) { + switch (operator) { + case EQ: + return counted == count; + case NE: + return counted != count; + case GT: + return counted > count; + case GE: + return counted >= count; + case LT: + return counted < count; + case LE: + return counted <= count; + default: + return false; + } + } + + /** + * Apply the operator to the value read for the property. + */ + private boolean compare(Slot value) { + switch (operator) { + case IN: + for (Literal member : literals) { + if (equal(value, member)) { + return true; + } + } + return false; + case NOT_IN: + for (Literal member : literals) { + if (equal(value, member)) { + return false; + } + } + return true; + case EQ: + return equal(value, literal); + case NE: + return equal(value, literal) == false; + case STARTS_WITH: + return value.getString().startsWith(literal.getText()); + case ENDS_WITH: + return value.getString().endsWith(literal.getText()); + case CONTAINS: + return value.getString().contains(literal.getText()); + case GT: + return value.number(type) > literal.getNumber(); + case GE: + return value.number(type) >= literal.getNumber(); + case LT: + return value.number(type) < literal.getNumber(); + case LE: + return value.number(type) <= literal.getNumber(); + default: + return false; + } + } + + /** + * Whether the value read equals a literal, ordinally for strings. + */ + private boolean equal(Slot value, Literal other) { + switch (type) { + case BOOL: + return other.getKind() == ScriptNode.Kind.BOOL && + value.getBool() == other.getBool(); + case STRING: + return other.getKind() == ScriptNode.Kind.STRING && + value.getString().equals(other.getText()); + default: + return other.getKind() == ScriptNode.Kind.NUMBER && + value.number(type) == other.getNumber(); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedOutputMetaData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedOutputMetaData.java new file mode 100644 index 000000000..73a03b80b --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedOutputMetaData.java @@ -0,0 +1,204 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The Output block of a script, with the field names 51Degrees uses for the + * metadata of every property. It is the whole definition of the derived + * property, so a JSON builder, the cloud or a documentation generator can + * read every field. A field the script does not give is null. + */ +public final class DerivedOutputMetaData { + + private final OutputModel output; + private final List vendorIds; + private final List dependencies; + private final List values; + + DerivedOutputMetaData(OutputModel output) { + this.output = output; + this.vendorIds = output.getVendorIds() == null ? + null : plainList(output.getVendorIds()); + this.dependencies = output.getDependencies() == null ? + Collections.unmodifiableList( + new ArrayList(output.getComputed())) : + plainList(output.getDependencies()); + if (output.getValues() == null) { + this.values = null; + } else { + List entries = new ArrayList<>(); + for (OutputModel.ValueEntry entry : output.getValues()) { + entries.add(new DerivedValueMetaData( + entry.getName().toString(), entry.getDescription())); + } + this.values = Collections.unmodifiableList(entries); + } + } + + /** + * @return the property name published under the derived Element Data + * key + */ + public String getName() { + return output.getName(); + } + + /** + * @return what the property asserts + */ + public String getDescription() { + return output.getDescription(); + } + + /** + * @return string, bool, int or double + */ + public String getValueType() { + return output.getValueType().toString(); + } + + /** + * @return the Java type of the value, being {@link String}, + * {@link Boolean}, {@link Integer} or {@link Double} + */ + public Class getValueClass() { + return output.getValueType().getValueClass(); + } + + public String getStoredValueType() { + return output.getStoredValueType(); + } + + /** + * @return the string form of the default recorded for the property. + * This is metadata only, and nothing reads it while a request is + * processed. + */ + public String getDefaultValue() { + return output.getDefaultValue(); + } + + /** + * @return false, as format 1 has no list outputs + */ + public boolean isList() { + return output.isList(); + } + + public Boolean getIsMandatory() { + return output.getIsMandatory(); + } + + public Boolean getIsObsolete() { + return output.getIsObsolete(); + } + + public String getCategory() { + return output.getCategory(); + } + + public Boolean getIsPopular() { + return output.getIsPopular(); + } + + public Boolean getExportValues() { + return output.getExportValues(); + } + + public String getUrl() { + return output.getUrl(); + } + + public Long getDisplayOrder() { + return output.getDisplayOrder(); + } + + public Long getPropertyId() { + return output.getPropertyId(); + } + + /** + * @return the list as written, with numbers as {@link Double} + */ + public List getVendorIds() { + return vendorIds; + } + + /** + * @return the list the script gives, or every source property its + * checks and rules name, in elementKey.PropertyName form, where it gives + * none + */ + public List getDependencies() { + return dependencies; + } + + /** + * @return the values the property can have, or null where the script + * gives no list + */ + public List getValues() { + return values; + } + + /** + * A list from the script as plain Java values, being null, a + * {@link Boolean}, a {@link Double}, a {@link String}, a list or a map. + */ + private static List plainList(ScriptNode node) { + List items = new ArrayList<>(); + for (ScriptNode item : node.getItems()) { + items.add(plain(item)); + } + return Collections.unmodifiableList(items); + } + + private static Object plain(ScriptNode node) { + switch (node.getKind()) { + case BOOL: + return node.getBool(); + case NUMBER: + return node.getNumber(); + case STRING: + return node.getText(); + case LIST: + return plainList(node); + case MAP: + Map values = new LinkedHashMap<>(); + for (int i = 0; i < node.getKeys().size(); i++) { + values.put( + node.getKeys().get(i), + plain(node.getValues().get(i))); + } + return Collections.unmodifiableMap(values); + default: + return null; + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElement.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElement.java new file mode 100644 index 000000000..1dde7e040 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElement.java @@ -0,0 +1,566 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.data.factories.ElementDataFactory; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.core.typed.TypedKeyDefault; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * The Derived Property Element, which produces new property values from + * property values other Flow Elements have already produced, following + * rules written in a script, and writes them into the Element Data under + * the key {@code derived}, which every instance in a Pipeline shares. + *

+ * The element holds no data file, makes no request over the network and + * reads no evidence. Everything it holds is fixed when it is built, so + * concurrent requests need nothing further, and per request it reads each + * source property once, then runs each script. + *

+ * See the + * Derived + * Property Element specification. + */ +public final class DerivedPropertyElement + extends FlowElementBase + implements IDerivedPropertyElement { + + /** + * The Element Data key every Derived Property Element in a Pipeline + * writes under. + */ + public static final String ELEMENT_DATA_KEY = "derived"; + + /** + * The sentence that closes every message for a derived property with no + * value, being USUAL_CAUSES in tools/evaluate.mjs. + */ + static final String USUAL_CAUSES = + "Usual causes are the element that supplies the property not being " + + "in the pipeline, or being added after this element rather than " + + "before it, the property being excluded in the engine " + + "configuration, the property not being included in the resource " + + "key, or JavaScript that populates the property not having run yet."; + + private static final TypedKey DATA_KEY = + new TypedKeyDefault<>(ELEMENT_DATA_KEY, IDerivedPropertyData.class); + + private final CompiledScript[] scripts; + private final SourceRead[] reads; + private final List> dataKeys; + private final List properties; + private final List metadata; + private final EvidenceKeyFilter evidenceKeyFilter = + new EvidenceKeyFilterWhitelist(Collections.emptyList()); + + /** + * Construct the element from validated scripts. Use + * {@link DerivedPropertyElementBuilder} to build one. + * @param logger used for logging + * @param dataFactory makes the Element Data + * @param models the validated scripts + */ + DerivedPropertyElement( + Logger logger, + ElementDataFactory dataFactory, + List models) { + super(logger, dataFactory); + Map keyIndex = new HashMap<>(); + Map readIndex = new HashMap<>(); + List> keys = new ArrayList<>(); + List allReads = new ArrayList<>(); + List compiled = new ArrayList<>(); + List outputs = new ArrayList<>(); + List scriptData = new ArrayList<>(); + for (DerivedScript model : models) { + int[] scriptReads = new int[model.getProperties().size()]; + for (int i = 0; i < scriptReads.length; i++) { + SourceProperty property = model.getProperties().get(i); + String id = property.getKey() + "|" + property.getType(); + Integer index = readIndex.get(id); + if (index == null) { + int key = -1; + if (property.getElementKey().equals(ELEMENT_DATA_KEY) == + false) { + Integer known = keyIndex.get(property.getElementKey()); + if (known == null) { + known = keys.size(); + keyIndex.put(property.getElementKey(), known); + keys.add(new TypedKeyDefault( + property.getElementKey(), ElementData.class)); + } + key = known; + } + index = allReads.size(); + readIndex.put(id, index); + allReads.add(new SourceRead(property, key)); + } + scriptReads[i] = index; + } + compiled.add(new CompiledScript(model, scriptReads)); + scriptData.add(new DerivedPropertyMetaData(model)); + OutputModel output = model.getOutput(); + outputs.add(new ElementPropertyMetaDataDefault( + output.getName(), + this, + output.getCategory() == null ? "" : output.getCategory(), + output.getValueType().getValueClass(), + true)); + } + this.scripts = compiled.toArray(new CompiledScript[0]); + this.reads = allReads.toArray(new SourceRead[0]); + this.dataKeys = Collections.unmodifiableList(keys); + this.properties = Collections.unmodifiableList(outputs); + this.metadata = Collections.unmodifiableList(scriptData); + } + + @Override + public String getElementDataKey() { + return ELEMENT_DATA_KEY; + } + + @Override + public TypedKey getTypedDataKey() { + return DATA_KEY; + } + + /** + * The element reads properties other elements produce and nothing from + * the request itself, so it accepts no evidence. + * @return a filter that includes no key + */ + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return evidenceKeyFilter; + } + + /** + * @return one entry per script, with the output property's name, the + * Java type of its value, the category from the script's output block, + * and available true + */ + @Override + public List getProperties() { + return properties; + } + + @Override + public List getScripts() { + return metadata; + } + + /** + * Every source property the element reads from another element's + * Element Data, once each. + */ + List sourceProperties() { + List result = new ArrayList<>(); + for (SourceRead read : reads) { + if (read.getKey() >= 0) { + result.add(read.getProperty()); + } + } + return result; + } + + /** + * Read each source property once, then run each script, writing one + * value, or one value with no value carrying a message, per script. + */ + @Override + protected void processInternal(FlowData data) { + IDerivedPropertyData out = data.getOrAdd(DATA_KEY, getDataFactory()); + ElementData[] sources = new ElementData[dataKeys.size()]; + for (int i = 0; i < sources.length; i++) { + TryGetResult source = + data.tryGetValue(dataKeys.get(i)); + if (source.hasValue()) { + sources[i] = source.getValue(); + } + } + Slot[] slots = new Slot[reads.length]; + for (int i = 0; i < reads.length; i++) { + if (reads[i].getKey() >= 0) { + slots[i] = SourceValues.read( + sources[reads[i].getKey()], reads[i].getProperty()); + } + } + for (CompiledScript script : scripts) { + processScript(script, slots, sources, out); + } + } + + private void processScript( + CompiledScript script, + Slot[] slots, + ElementData[] sources, + IDerivedPropertyData out) { + DerivedScript model = script.getModel(); + Slot[] scriptSlots = new Slot[script.readCount()]; + List missing = null; + List reasons = null; + SourceRead noData = null; + for (int i = 0; i < scriptSlots.length; i++) { + SourceRead read = reads[script.read(i)]; + if (read.getKey() < 0) { + scriptSlots[i] = SourceValues.read(out, read.getProperty()); + } else { + scriptSlots[i] = slots[script.read(i)]; + if (sources[read.getKey()] == null && noData == null) { + noData = read; + } + } + if (scriptSlots[i].isAvailable() == false) { + if (missing == null) { + missing = new ArrayList<>(); + reasons = new ArrayList<>(); + } + missing.add(model.getProperties().get(i).getName()); + reasons.add(scriptSlots[i].getReason()); + } + } + OutputModel output = model.getOutput(); + if (missing != null) { + if (noData != null && script.firstMissingData()) { + String key = noData.getProperty().getElementKey(); + logger.error("Derived property script '" + model.getName() + + "' reads '" + noData.getProperty().getName() + "', and a " + + "request carried no element data under '" + key + "'. " + + "Add this element to the pipeline after the element " + + "that supplies '" + key + "'. This error is reported " + + "once for this script."); + } + AspectPropertyValueDefault value = + new AspectPropertyValueDefault<>(); + value.setNoValueMessage( + missingMessage(output.getName(), missing, reasons)); + out.put(output.getName(), value); + return; + } + out.put(output.getName(), + valueOf(output.getValueType(), model.evaluate(scriptSlots))); + } + + private static AspectPropertyValue valueOf( + ValueType type, + Literal literal) { + switch (type) { + case BOOL: + return new AspectPropertyValueDefault<>(literal.getBool()); + case INT: + return new AspectPropertyValueDefault<>( + (int) literal.getNumber()); + case DOUBLE: + return new AspectPropertyValueDefault<>(literal.getNumber()); + default: + return new AspectPropertyValueDefault<>(literal.getText()); + } + } + + /** + * Say why a script has no value, naming every source property that + * could not be read, in the wording of missingMessage in + * tools/evaluate.mjs. + */ + static String missingMessage( + String outputName, + List missing, + List reasons) { + StringBuilder message = new StringBuilder() + .append("Derived property '").append(outputName) + .append("' has no value because "); + if (missing.size() == 1) { + message.append("1 source property was not available."); + } else { + message.append(missing.size()) + .append(" source properties were not available."); + } + for (int i = 0; i < missing.size(); i++) { + message.append(" '").append(missing.get(i)).append("' (") + .append(reasons.get(i)).append(")."); + } + return message.append(' ').append(USUAL_CAUSES).toString(); + } + + /** + * Check that some element in the Pipeline supplies every source + * property a script names, and that no two Derived Property Elements + * produce the same property. Where an element sits is not judged, + * because a Pipeline lists the members of a parallel group after the + * elements at the top level, so a position does not say what ran + * first. + *

+ * See the + * Pipeline + * check. + * @param pipeline the Pipeline the element is being added to + * @throws PipelineConfigurationException where the check fails + */ + @Override + public void addPipeline(Pipeline pipeline) { + if (pipeline != null) { + checkPipeline(pipeline); + } + super.addPipeline(pipeline); + } + + private void checkPipeline(Pipeline pipeline) { + Map> available = + availableProperties(pipeline); + List> elements = elementsOf(pipeline, available); + Set keys = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + keys.addAll(available.keySet()); + boolean allLoaded = allLoaded(pipeline); + List problems = new ArrayList<>(); + for (CompiledScript script : scripts) { + for (SourceProperty property : + script.getModel().getProperties()) { + String problem = checkSupplied(elements, keys, allLoaded, + script.getModel().getName(), property); + if (problem != null) { + problems.add(problem); + } + } + } + // Each name counts the distinct Derived Property Elements that + // produce it. + Map producers = new HashMap<>(); + for (FlowElement element : elements) { + if (element instanceof DerivedPropertyElement) { + for (CompiledScript script : + ((DerivedPropertyElement) element).scripts) { + String name = script.getModel().getOutput().getName() + .toLowerCase(Locale.ROOT); + Integer count = producers.get(name); + producers.put(name, count == null ? 1 : count + 1); + } + } + } + for (CompiledScript script : scripts) { + String name = script.getModel().getOutput().getName(); + Integer count = producers.get(name.toLowerCase(Locale.ROOT)); + if (count != null && count > 1) { + problems.add("two Derived Property Elements in the pipeline " + + "produce the derived property '" + name + "'"); + } + } + if (problems.isEmpty() == false) { + throw new PipelineConfigurationException( + String.join(". ", problems)); + } + } + + /** + * Why no element supplies a source property, or null where one does. + * An element whose properties are not known yet, such as a cloud engine + * that could not reach the service, is taken to supply the properties + * under its key, with a warning, so that a passing failure does not + * stop the Pipeline being built. + */ + private String checkSupplied( + List> elements, + Set keys, + boolean allLoaded, + String script, + SourceProperty property) { + String key = property.getElementKey(); + String name = property.getPropertyName(); + boolean inspected = false; + for (FlowElement element : elements) { + if (element.getElementDataKey().equalsIgnoreCase(key) == false) { + continue; + } + inspected = true; + if (lists(element, name)) { + return null; + } + if (element.hasLoadedProperties() == false) { + warnNotLoaded(script, property); + return null; + } + } + if (inspected == false && keys.contains(key) == false) { + return "derived property script '" + script + "' reads '" + + property.getName() + "', and no element in the pipeline has " + + "the element data key '" + key + "'"; + } + if (inspected == false && allLoaded == false) { + warnNotLoaded(script, property); + return null; + } + return "derived property script '" + script + "' reads '" + + property.getName() + "', and no element under '" + key + + "' lists the property '" + name + "'"; + } + + private void warnNotLoaded(String script, SourceProperty property) { + logger.warn("Derived property script '" + script + "' reads '" + + property.getName() + "', and the element under '" + + property.getElementKey() + "' has not loaded its properties " + + "yet, so whether it supplies '" + property.getPropertyName() + + "' cannot be checked."); + } + + /** + * Whether an element lists a property in its metadata, matched without + * regard to case. + */ + private static boolean lists(FlowElement element, String name) { + try { + for (ElementPropertyMetaData property : element.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + } catch (RuntimeException e) { + return false; + } + return false; + } + + /** + * The Pipeline's available properties by Element Data key, which covers + * the members of every parallel group as well as the elements at the + * top level. + */ + private static Map> + availableProperties(Pipeline pipeline) { + try { + return pipeline.getElementAvailableProperties(); + } catch (RuntimeException e) { + return Collections.emptyMap(); + } + } + + /** + * Every element of the Pipeline that has an Element Data key, with the + * members of a parallel group found through the available properties, + * whose metadata names the element that owns each property. + */ + private static List> elementsOf( + Pipeline pipeline, + Map> available) { + Map, Boolean> seen = new IdentityHashMap<>(); + List> elements = new ArrayList<>(); + for (Object top : pipeline.getFlowElements()) { + addElement((FlowElement) top, seen, elements); + } + for (Map byName : + available.values()) { + for (ElementPropertyMetaData property : byName.values()) { + FlowElement owner = property.getElement(); + if (owner != null) { + addElement(owner, seen, elements); + } + } + } + return elements; + } + + /** + * Add an element that has an Element Data key. A parallel group has + * none, and is passed over. + */ + private static void addElement( + FlowElement element, + Map, Boolean> seen, + List> elements) { + if (seen.containsKey(element)) { + return; + } + seen.put(element, Boolean.TRUE); + try { + element.getElementDataKey(); + } catch (UnsupportedOperationException e) { + return; + } + elements.add(element); + } + + private static boolean allLoaded(Pipeline pipeline) { + for (Object element : pipeline.getFlowElements()) { + if (((FlowElement) element).hasLoadedProperties() == false) { + return false; + } + } + return true; + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + + /** + * Write the build log for each script, being one information line + * giving the name, the version, the format, the source and the output + * property, the compiled model as canonical JSON at debug level, and a + * warning carrying the note of a deprecated script. + */ + void logScripts() { + for (CompiledScript script : scripts) { + DerivedScript model = script.getModel(); + logger.info("Derived property script '" + model.getName() + + "' version " + model.getVersion() + ", format " + + model.getFormat() + ", from " + model.getSource() + + ", produces '" + ELEMENT_DATA_KEY + "." + + model.getOutput().getName() + "'"); + if (logger.isDebugEnabled()) { + logger.debug("Derived property script '" + model.getName() + + "' compiled to:\n" + CanonicalJson.print(model)); + } + if (model.isDeprecated()) { + logger.warn("Derived property script '" + model.getName() + + "' is deprecated. " + model.getDeprecationNote()); + } + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementBuilder.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementBuilder.java new file mode 100644 index 000000000..492f753ac --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementBuilder.java @@ -0,0 +1,341 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; +import fiftyone.pipeline.derivedproperty.BuiltInScript; +import fiftyone.pipeline.derivedproperty.DerivedScriptFault; +import fiftyone.pipeline.derivedproperty.DerivedScriptValidationException; +import fiftyone.pipeline.derivedproperty.data.DerivedPropertyData; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Builds a {@link DerivedPropertyElement} from scripts given in three ways, + * which can be mixed, being scripts shipped in this package, files on disk + * and text in code. There is no way to fetch a script over the network. + *

+ * Every script is parsed and validated when {@link #build()} is called, + * and every fault in every script is reported at once, in one + * {@link DerivedScriptValidationException}. + *

+ * A configuration file names this builder as + * {@code DerivedPropertyElement}, with {@code Scripts} naming built-in + * scripts and {@code ScriptFiles} naming files, each a comma separated + * list. Script text from code is not available from configuration. + */ +@ElementBuilder +public class DerivedPropertyElementBuilder { + + /** + * The characters that make a file name a pattern. + */ + private static final String WILDCARDS = "*?[{"; + + /** + * A script as given, before it is validated. + */ + private static final class ScriptText { + private final String name; + private final String text; + private final String source; + + ScriptText(String name, String text, String source) { + this.name = name; + this.text = text; + this.source = source; + } + } + + private final ILoggerFactory loggerFactory; + private final List scripts = new ArrayList<>(); + private final List faults = new ArrayList<>(); + + /** + * Construct a new builder. + * @param loggerFactory the logger factory for the element and the + * Element Data it creates + */ + public DerivedPropertyElementBuilder(ILoggerFactory loggerFactory) { + if (loggerFactory == null) { + throw new IllegalArgumentException("loggerFactory"); + } + this.loggerFactory = loggerFactory; + } + + /** + * Add a script shipped in this package. + * @param script the script + * @return this builder + */ + public DerivedPropertyElementBuilder addScript(BuiltInScript script) { + String text = script.readText(); + if (text == null) { + faults.add(new DerivedScriptFault(script.name(), script.name(), + "", 0, "the built-in script '" + script.name() + "' is not " + + "in this package, which was built without the " + + "derived-properties scripts")); + return this; + } + scripts.add(new ScriptText(script.name(), text, script.name())); + return this; + } + + /** + * Add every file the path matches. The file name may hold the wildcards + * {@code *} and {@code ?}, as in {@code derived/*.yaml}. A script's Name + * must equal its file name without the extension. A path matching no + * file is a fault. + * @param path the path, relative to the working directory or absolute + * @return this builder + */ + public DerivedPropertyElementBuilder addScriptFile(String path) { + List matches = matches(path); + if (matches.isEmpty()) { + faults.add(new DerivedScriptFault("", path, "", 0, + "no file matches '" + path + "'")); + return this; + } + for (Path match : matches) { + String fileName = match.getFileName().toString(); + int dot = fileName.lastIndexOf('.'); + String name = dot > 0 ? fileName.substring(0, dot) : fileName; + try { + String text = new String( + Files.readAllBytes(match), StandardCharsets.UTF_8); + scripts.add(new ScriptText(name, text, match.toString())); + } catch (IOException e) { + faults.add(new DerivedScriptFault(name, match.toString(), "", + 0, "the file cannot be read: " + e.getMessage())); + } + } + return this; + } + + /** + * Add a script from text in code, read as YAML or JSON as the text + * shows. + * @param name the name the script must have + * @param content the text of the script + * @return this builder + */ + public DerivedPropertyElementBuilder addScript( + String name, + String content) { + scripts.add(new ScriptText(name, content, "code")); + return this; + } + + /** + * Add built-in scripts by name, which is how the {@code Scripts} build + * parameter of a configuration file reaches the builder. + * @param names the names, separated by commas + * @return this builder + */ + public DerivedPropertyElementBuilder setScripts(String names) { + for (String name : split(names)) { + BuiltInScript script = BuiltInScript.fromName(name); + if (script == null) { + faults.add(new DerivedScriptFault(name, name, "", 0, "'" + + name + "' is not a built-in script. The built-in " + + "scripts are " + BuiltInScript.names())); + } else { + addScript(script); + } + } + return this; + } + + /** + * Add script files by path, which is how the {@code ScriptFiles} build + * parameter of a configuration file reaches the builder. + * @param paths the paths, separated by commas, each of which may hold + * wildcards in its file name + * @return this builder + */ + public DerivedPropertyElementBuilder setScriptFiles(String paths) { + for (String path : split(paths)) { + addScriptFile(path); + } + return this; + } + + /** + * Validate every script and build the element. + * @return a new element + * @throws DerivedScriptValidationException listing every fault in every + * script, where any script has one + * @throws IllegalStateException where no script was added + */ + public DerivedPropertyElement build() { + if (scripts.isEmpty() && faults.isEmpty()) { + throw new IllegalStateException( + "A Derived Property Element needs at least one script. Add " + + "one with addScript or addScriptFile, or name one with the " + + "Scripts or ScriptFiles build parameter."); + } + List all = new ArrayList<>(faults); + List models = new ArrayList<>(); + for (ScriptText script : scripts) { + DerivedScriptValidator.Result result = DerivedScriptValidator + .validate(script.text, script.name, script.source); + all.addAll(result.getFaults()); + if (result.getScript() != null) { + models.add(result.getScript()); + } + } + all.addAll(duplicates(models)); + if (all.isEmpty() == false) { + throw new DerivedScriptValidationException(all); + } + Logger dataLogger = + loggerFactory.getLogger(DerivedPropertyData.class.getName()); + DerivedPropertyElement element = new DerivedPropertyElement( + loggerFactory.getLogger(DerivedPropertyElement.class.getName()), + (flowData, flowElement) -> + new DerivedPropertyData(dataLogger, flowData), + models); + element.logScripts(); + return element; + } + + /** + * Two scripts with the same name, or producing the same output + * property, as there is one script per name and one value per output + * property. + */ + private static List duplicates( + List models) { + List found = new ArrayList<>(); + Map names = new HashMap<>(); + Map outputs = new HashMap<>(); + for (DerivedScript model : models) { + String name = model.getName().toLowerCase(Locale.ROOT); + DerivedScript first = names.get(name); + if (first != null) { + found.add(new DerivedScriptFault(model.getName(), + model.getSource(), "Name", 0, "the script '" + + model.getName() + "' is added more than once, from " + + first.getSource() + " and from " + model.getSource())); + } else { + names.put(name, model); + } + String output = + model.getOutput().getName().toLowerCase(Locale.ROOT); + first = outputs.get(output); + if (first != null) { + found.add(new DerivedScriptFault(model.getName(), + model.getSource(), "Output.Name", 0, "the output " + + "property '" + model.getOutput().getName() + "' is " + + "also produced by the script '" + first.getName() + + "' from " + first.getSource())); + } else { + outputs.put(output, model); + } + } + return found; + } + + /** + * The entries of a comma separated list, with the white space around + * each removed and empty entries left out. + */ + private static List split(String list) { + List entries = new ArrayList<>(); + if (list == null) { + return entries; + } + for (String entry : list.split(",")) { + String trimmed = entry.trim(); + if (trimmed.isEmpty() == false) { + entries.add(trimmed); + } + } + return entries; + } + + /** + * The files a path matches, in name order. Only the file name may hold + * wildcards, which are read as a glob. + */ + private static List matches(String pattern) { + List found = new ArrayList<>(); + if (pattern == null || pattern.isEmpty()) { + return found; + } + File file = new File(pattern); + String glob = file.getName(); + try { + if (hasWildcard(glob) == false) { + Path path = file.toPath(); + if (Files.isRegularFile(path)) { + found.add(path); + } + return found; + } + File parent = file.getParentFile(); + Path directory = parent == null ? + new File(".").toPath() : parent.toPath(); + try (DirectoryStream stream = + Files.newDirectoryStream(directory, glob)) { + for (Path match : stream) { + if (Files.isRegularFile(match)) { + found.add(parent == null ? + match.getFileName() : match); + } + } + } + } catch (IOException | DirectoryIteratorException | + IllegalArgumentException e) { + // An invalid path or pattern matches no file. + return Collections.emptyList(); + } + found.sort(Comparator.comparing(Path::toString)); + return found; + } + + private static boolean hasWildcard(String name) { + for (int i = 0; i < name.length(); i++) { + if (WILDCARDS.indexOf(name.charAt(i)) >= 0) { + return true; + } + } + return false; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyMetaData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyMetaData.java new file mode 100644 index 000000000..614774c7d --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyMetaData.java @@ -0,0 +1,97 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * The full Property metadata block of one script a Derived Property Element + * holds. Its {@link #getOutput()} is the whole definition of the derived + * property. + */ +public final class DerivedPropertyMetaData { + + private final String name; + private final String version; + private final int format; + private final boolean deprecated; + private final String deprecationNote; + private final String source; + private final DerivedOutputMetaData output; + + DerivedPropertyMetaData(DerivedScript script) { + this.name = script.getName(); + this.version = script.getVersion(); + this.format = script.getFormat(); + this.deprecated = script.isDeprecated(); + this.deprecationNote = script.getDeprecationNote(); + this.source = script.getSource(); + this.output = new DerivedOutputMetaData(script.getOutput()); + } + + /** + * @return the script's name, which is what selects it + */ + public String getName() { + return name; + } + + /** + * @return the author's semantic version, which plays no part in + * selecting the script + */ + public String getVersion() { + return version; + } + + /** + * @return the version of the script language, which is 1 + */ + public int getFormat() { + return format; + } + + public boolean isDeprecated() { + return deprecated; + } + + /** + * @return what to use instead, or empty where the script is not + * deprecated + */ + public String getDeprecationNote() { + return deprecationNote; + } + + /** + * @return the built-in script's name, the file path, or "code" + */ + public String getSource() { + return source; + } + + /** + * @return the definition of the property the script produces + */ + public DerivedOutputMetaData getOutput() { + return output; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScript.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScript.java new file mode 100644 index 000000000..6c253198c --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScript.java @@ -0,0 +1,185 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A validated script, the model every language builds from the same text. + * Nothing in it changes once it is built. + */ +final class DerivedScript { + + /** + * A check, which is a named condition. + */ + static final class NamedCheck { + private final String name; + private final Condition condition; + + NamedCheck(String name, Condition condition) { + this.name = name; + this.condition = condition; + } + + String getName() { + return name; + } + + Condition getCondition() { + return condition; + } + } + + /** + * A When with a Then, or, where the condition is null, an Else. + */ + static final class Rule { + private final Condition when; + private final Literal value; + + Rule(Condition when, Literal value) { + this.when = when; + this.value = value; + } + + /** + * @return the condition, or null for an Else + */ + Condition getWhen() { + return when; + } + + Literal getValue() { + return value; + } + } + + private final int format; + private final String name; + private final String version; + private final boolean deprecated; + private final String deprecationNote; + private final String source; + private final OutputModel output; + private final List properties; + private final List checks; + private final List rules; + + /** + * @param properties the source properties in the order the script first + * names them, checks before rules. A comparison reads + * its property from the slot with the same index. + */ + DerivedScript( + int format, + String name, + String version, + boolean deprecated, + String deprecationNote, + String source, + OutputModel output, + List properties, + List checks, + List rules) { + this.format = format; + this.name = name; + this.version = version; + this.deprecated = deprecated; + this.deprecationNote = deprecationNote; + this.source = source; + this.output = output; + this.properties = Collections.unmodifiableList( + new ArrayList<>(properties)); + this.checks = Collections.unmodifiableList(new ArrayList<>(checks)); + this.rules = Collections.unmodifiableList(new ArrayList<>(rules)); + } + + int getFormat() { + return format; + } + + String getName() { + return name; + } + + String getVersion() { + return version; + } + + boolean isDeprecated() { + return deprecated; + } + + String getDeprecationNote() { + return deprecationNote; + } + + /** + * @return the built-in script's name, the file path, or "code" + */ + String getSource() { + return source; + } + + OutputModel getOutput() { + return output; + } + + List getProperties() { + return properties; + } + + List getChecks() { + return checks; + } + + List getRules() { + return rules; + } + + /** + * Run the script on the source properties read for it. Every slot is + * available, so every condition is true or false, and the Else every + * script ends in always matches. + * @param slots the source properties, in the order of + * {@link #getProperties()} + * @return the value of the first rule whose condition is true + */ + Literal evaluate(Slot[] slots) { + boolean[] results = new boolean[checks.size()]; + for (int i = 0; i < results.length; i++) { + results[i] = checks.get(i).getCondition().test(slots, results); + } + for (Rule rule : rules) { + if (rule.getWhen() == null || + rule.getWhen().test(slots, results)) { + return rule.getValue(); + } + } + throw new IllegalStateException( + "The rules of '" + output.getName() + "' do not end in an Else."); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParser.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParser.java new file mode 100644 index 000000000..58cc00e82 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParser.java @@ -0,0 +1,287 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.error.Mark; +import org.yaml.snakeyaml.error.MarkedYAMLException; +import org.yaml.snakeyaml.error.YAMLException; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; + +import java.io.StringReader; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Reads the text of a script, as YAML or as JSON, into {@link ScriptNode}s. + *

+ * JSON is read as the JSON subset of YAML 1.2 by the same reader. The + * nodes are composed rather than constructed, so every node keeps its line, + * and a plain scalar is resolved here by the YAML 1.2 core schema, under + * which only {@code true} and {@code false} in their three spellings are + * booleans, so {@code yes} and {@code NO} stay strings. A quoted scalar is + * always a string. + *

+ * See + * YAML 1.2.2, + * 10.3.2 Tag Resolution and + * format + * 1, two places where the languages are allowed to differ. + */ +final class DerivedScriptParser { + + /** + * How deep a value may nest, which only aliases that refer to + * themselves can reach, because the reader limits nesting written out + * in the text to less than this. + */ + private static final int MAX_DEPTH = 100; + + private static final Pattern DECIMAL_INT = Pattern.compile("[-+]?[0-9]+"); + private static final Pattern OCTAL_INT = Pattern.compile("0o[0-7]+"); + private static final Pattern HEX_INT = Pattern.compile("0x[0-9a-fA-F]+"); + private static final Pattern FLOAT = Pattern.compile( + "[-+]?(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?"); + private static final Pattern INFINITY = + Pattern.compile("[-+]?\\.(inf|Inf|INF)"); + private static final Pattern NOT_A_NUMBER = + Pattern.compile("\\.(nan|NaN|NAN)"); + + private static final int OCTAL_RADIX = 8; + private static final int HEX_RADIX = 16; + + /** + * The length of the 0o or 0x prefix. + */ + private static final int RADIX_PREFIX_LENGTH = 2; + + private DerivedScriptParser() { + } + + /** + * Parse the text of a script. + * @param text the script, as YAML or JSON + * @return the document, which is always a mapping + * @throws ScriptParseException where the text is not YAML or JSON, is + * empty, is not a mapping, or writes a key twice + */ + static ScriptNode parse(String text) throws ScriptParseException { + if (text == null) { + throw new ScriptParseException("the script is empty", 0); + } + String what = startsLikeJson(text) ? "JSON" : "YAML"; + Node root; + try { + root = new Yaml(new LoaderOptions()).compose( + new StringReader(text)); + } catch (MarkedYAMLException e) { + Mark mark = e.getProblemMark() != null ? + e.getProblemMark() : e.getContextMark(); + String problem = e.getProblem() != null ? + e.getProblem() : e.getMessage(); + throw new ScriptParseException( + "the text is not valid " + what + ": " + problem, + mark == null ? 0 : mark.getLine() + 1); + } catch (YAMLException e) { + throw new ScriptParseException( + "the text is not valid " + what + ": " + e.getMessage(), 0); + } + if (root == null) { + throw new ScriptParseException("the script is empty", 0); + } + ScriptNode document = convert(root, 0); + if (document.getKind() == ScriptNode.Kind.NULL) { + throw new ScriptParseException("the script is empty", 0); + } + if (document.getKind() != ScriptNode.Kind.MAP) { + throw new ScriptParseException( + "the script must be a mapping of keys to values at the top " + + "level", + 0); + } + requireDistinctKeys(document); + return document; + } + + /** + * Whether the text opens with a brace once leading white space is + * passed over, which is how the reference tools tell JSON from YAML. + */ + private static boolean startsLikeJson(String text) { + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (JsText.isWhitespace(c) == false) { + return c == '{'; + } + } + return false; + } + + private static ScriptNode convert(Node node, int depth) + throws ScriptParseException { + int line = node.getStartMark() == null ? + 0 : node.getStartMark().getLine() + 1; + if (depth > MAX_DEPTH) { + throw new ScriptParseException( + "the text is not valid YAML: aliases nest too deeply", line); + } + if (node instanceof MappingNode) { + List keys = new ArrayList<>(); + List values = new ArrayList<>(); + for (NodeTuple tuple : ((MappingNode) node).getValue()) { + keys.add(keyText(tuple.getKeyNode())); + values.add(convert(tuple.getValueNode(), depth + 1)); + } + return ScriptNode.ofMap(keys, values, line); + } + if (node instanceof SequenceNode) { + List items = new ArrayList<>(); + for (Node item : ((SequenceNode) node).getValue()) { + items.add(convert(item, depth + 1)); + } + return ScriptNode.ofList(items, line); + } + if (node instanceof ScalarNode) { + return scalar((ScalarNode) node, line); + } + return ScriptNode.ofNull(line); + } + + /** + * A key is the text it is written as, as JavaScript makes every key of + * an object a string. + */ + private static String keyText(Node key) { + return key instanceof ScalarNode ? ((ScalarNode) key).getValue() : ""; + } + + /** + * Resolve a scalar. A plain scalar takes the tag the YAML 1.2 core + * schema gives it, and any other scalar is a string. + */ + static ScriptNode scalar(ScalarNode node, int line) { + String value = node.getValue(); + if (node.isPlain() == false) { + return ScriptNode.ofString(value, line); + } + switch (value) { + case "": + case "~": + case "null": + case "Null": + case "NULL": + return ScriptNode.ofNull(line); + case "true": + case "True": + case "TRUE": + return ScriptNode.ofBool(true, line); + case "false": + case "False": + case "FALSE": + return ScriptNode.ofBool(false, line); + default: + break; + } + Double number = coreNumber(value); + return number == null ? + ScriptNode.ofString(value, line) : + ScriptNode.ofNumber(number, line); + } + + /** + * The number a plain scalar is under the core schema, or null where it + * is not one. + */ + private static Double coreNumber(String value) { + if (DECIMAL_INT.matcher(value).matches() || + FLOAT.matcher(value).matches()) { + return Double.parseDouble(value); + } + if (OCTAL_INT.matcher(value).matches()) { + return new BigInteger( + value.substring(RADIX_PREFIX_LENGTH), OCTAL_RADIX) + .doubleValue(); + } + if (HEX_INT.matcher(value).matches()) { + return new BigInteger( + value.substring(RADIX_PREFIX_LENGTH), HEX_RADIX) + .doubleValue(); + } + if (INFINITY.matcher(value).matches()) { + return value.charAt(0) == '-' ? + Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; + } + if (NOT_A_NUMBER.matcher(value).matches()) { + return Double.NaN; + } + return null; + } + + /** + * Refuse two keys of one mapping that differ only in case, because keys + * are matched without regard to case and one of the two would be + * dropped, and refuse a key written twice for the same reason. + */ + private static void requireDistinctKeys(ScriptNode node) + throws ScriptParseException { + if (node.getKind() == ScriptNode.Kind.LIST) { + for (ScriptNode item : node.getItems()) { + requireDistinctKeys(item); + } + return; + } + if (node.getKind() != ScriptNode.Kind.MAP) { + return; + } + Map seen = new HashMap<>(); + for (int i = 0; i < node.getKeys().size(); i++) { + String key = node.getKeys().get(i); + String first = seen.get(key.toLowerCase(Locale.ROOT)); + if (first != null) { + if (first.equals(key)) { + throw new ScriptParseException( + "the key '" + key + "' is written twice in one " + + "mapping, so one of the two would be dropped", + node.getLine()); + } + throw new ScriptParseException( + "the keys '" + first + "' and '" + key + "' differ only " + + "in case, and keys are matched without regard to " + + "case, so one of the two would be dropped", + node.getLine()); + } + seen.put(key.toLowerCase(Locale.ROOT), key); + requireDistinctKeys(node.getValues().get(i)); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptValidator.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptValidator.java new file mode 100644 index 000000000..a801ba562 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptValidator.java @@ -0,0 +1,1168 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.derivedproperty.DerivedScriptFault; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Checks a parsed script and builds its model, collecting every fault + * rather than stopping at the first. The paths and the messages are those + * of the reference implementation, tools/validate.mjs in the + * derived-properties repository, so a fault reads the same in every + * language. + *

+ * See format + * 1. + */ +final class DerivedScriptValidator { + + /** + * The result of validating one script. + */ + static final class Result { + private final DerivedScript script; + private final List faults; + + Result(DerivedScript script, List faults) { + this.script = script; + this.faults = Collections.unmodifiableList(faults); + } + + /** + * @return the model, or null where the script has a fault + */ + DerivedScript getScript() { + return script; + } + + List getFaults() { + return faults; + } + } + + static final List VALUE_TYPES = + Arrays.asList("string", "bool", "int", "double"); + static final List AGGREGATE_NAMES = + Arrays.asList("Passed", "Failed"); + static final List TOP_LEVEL_KEYS = Arrays.asList( + "Format", "Name", "Version", "Deprecated", "DeprecationNote", + "Output", "Checks", "Rules"); + /** + * The keys of Output, in the order the canonical form prints them. + */ + static final List OUTPUT_KEYS = Arrays.asList( + "Name", "Description", "ValueType", "StoredValueType", + "DefaultValue", "IsList", "IsMandatory", "IsObsolete", "Category", + "IsPopular", "ExportValues", "Url", "DisplayOrder", "PropertyId", + "VendorIds", "Dependencies", "Values"); + private static final List COUNT_OPERATORS = + Arrays.asList("Eq", "Ne", "Gt", "Ge", "Lt", "Le"); + + private static final Pattern IDENTIFIER = + Pattern.compile("^[A-Za-z][A-Za-z0-9]*$"); + private static final Pattern SOURCE_PATTERN = + Pattern.compile("^[A-Za-z][A-Za-z0-9]*\\.[A-Za-z][A-Za-z0-9]*$"); + private static final Pattern SEMANTIC_VERSION = Pattern.compile( + "^\\d+\\.\\d+\\.\\d+(?:-[0-9A-Za-z.-]+(?:\\.[0-9A-Za-z.-]+)*)?" + + "(?:\\+[0-9A-Za-z.-]+)?$"); + + /** + * Where a source property was first seen and the type inferred there, + * so a conflict can name both places. + */ + private static final class PropertyUse { + private final int slot; + private final ValueType type; + private final String typePath; + + PropertyUse(int slot, ValueType type, String typePath) { + this.slot = slot; + this.type = type; + this.typePath = typePath; + } + } + + private final ScriptNode document; + private final String script; + private final String source; + private final List faults = new ArrayList<>(); + private final Map uses = new HashMap<>(); + private final List properties = new ArrayList<>(); + + private DerivedScriptValidator( + ScriptNode document, + String script, + String source) { + this.document = document; + this.script = script; + this.source = source; + } + + /** + * Parse and validate the text of a script. + * @param text the script, as YAML or JSON + * @param name the file name without its extension, which the script's + * Name must equal, or empty where there is none + * @param source the built-in script's name, the file path, or "code" + * @return the model, or every fault found + */ + static Result validate(String text, String name, String source) { + String expected = name == null ? "" : name; + ScriptNode document; + try { + document = DerivedScriptParser.parse(text); + } catch (ScriptParseException e) { + List faults = new ArrayList<>(); + faults.add(new DerivedScriptFault( + expected, source, "", e.getLine(), e.getMessage())); + return new Result(null, faults); + } + DerivedScriptValidator validator = + new DerivedScriptValidator(document, expected, source); + DerivedScript model = validator.buildModel(); + if (validator.faults.isEmpty() == false) { + return new Result(null, validator.faults); + } + return new Result(model, validator.faults); + } + + private void fault(String path, ScriptNode at, String message) { + int line = at != null && at.isCollection() ? at.getLine() : 0; + faults.add(new DerivedScriptFault( + script, source, path, line, message)); + } + + private DerivedScript buildModel() { + for (String key : unknownKeys(document, TOP_LEVEL_KEYS)) { + fault(key, document, "unknown key '" + key + "' at the top " + + "level. Expected one of " + String.join(", ", TOP_LEVEL_KEYS)); + } + int format = readFormat(); + String name = readName(); + String version = readVersion(); + boolean deprecated = readDeprecated(); + String deprecationNote = readDeprecationNote(deprecated); + OutputModel.Draft output = readOutput(); + // Checks are read before rules, so that a rule can refer to one. + List checkNames = readCheckNames(); + List checks = readChecks(checkNames); + List rules = readRules(output, checkNames); + List computed = new ArrayList<>(); + if (output.dependencies == null) { + for (SourceProperty property : properties) { + computed.add(property.getName()); + } + } + return new DerivedScript(format, name, version, deprecated, + deprecationNote, source, new OutputModel(output, computed), + properties, checks, rules); + } + + private int readFormat() { + ScriptNode value = document.get("Format"); + if (value == null) { + fault("Format", document, "required key 'Format' is missing"); + return 0; + } + if (value.getKind() != ScriptNode.Kind.NUMBER || + value.getNumber() != 1) { + fault("Format", document, + "Format must be 1, found " + jsonOf(value)); + return 0; + } + return 1; + } + + private String readName() { + ScriptNode value = document.get("Name"); + if (value == null) { + fault("Name", document, "required key 'Name' is missing"); + return ""; + } + if (value.getKind() != ScriptNode.Kind.STRING) { + fault("Name", document, + "Name expected a string, found " + typeOf(value)); + return ""; + } + if (IDENTIFIER.matcher(value.getText()).matches() == false) { + fault("Name", document, "script name '" + value.getText() + + "' does not match the pattern " + IDENTIFIER.pattern()); + return value.getText(); + } + if (script.isEmpty() == false && + script.equals(value.getText()) == false) { + fault("Name", document, "script name '" + value.getText() + + "' must equal the file name '" + script + "'"); + } + return value.getText(); + } + + private String readVersion() { + ScriptNode value = document.get("Version"); + if (value == null) { + fault("Version", document, "required key 'Version' is missing"); + return ""; + } + if (value.getKind() != ScriptNode.Kind.STRING || + SEMANTIC_VERSION.matcher(value.getText()).matches() == false) { + fault("Version", document, "Version expected a semantic " + + "version such as 1.0.0, found " + jsonOf(value)); + return ""; + } + return value.getText(); + } + + private boolean readDeprecated() { + ScriptNode deprecated = document.get("Deprecated"); + if (deprecated == null) { + return false; + } + if (deprecated.getKind() != ScriptNode.Kind.BOOL) { + fault("Deprecated", document, + "Deprecated expected a boolean, found " + typeOf(deprecated)); + return false; + } + return deprecated.getBool(); + } + + private String readDeprecationNote(boolean deprecated) { + ScriptNode note = document.get("DeprecationNote"); + String text = ""; + if (note != null && note.getKind() != ScriptNode.Kind.STRING) { + fault("DeprecationNote", document, + "DeprecationNote expected a string, found " + typeOf(note)); + } else if (note != null) { + text = note.getText(); + } + if (deprecated && text.isEmpty()) { + fault("DeprecationNote", document, "a deprecated script must " + + "say what to use instead in DeprecationNote"); + } + if (deprecated == false && text.isEmpty() == false) { + fault("DeprecationNote", document, + "DeprecationNote is only allowed when Deprecated is true"); + } + return text; + } + + private OutputModel.Draft readOutput() { + OutputModel.Draft output = new OutputModel.Draft(); + String fallbackName = script.isEmpty() ? "unknown" : script; + ScriptNode raw = document.get("Output"); + if (raw == null) { + fault("Output", document, "required key 'Output' is missing"); + output.name = fallbackName; + return output; + } + if (raw.getKind() != ScriptNode.Kind.MAP) { + fault("Output", document, + "Output expected a mapping, found " + typeOf(raw)); + output.name = fallbackName; + return output; + } + for (String key : unknownKeys(raw, OUTPUT_KEYS)) { + fault("Output." + key, raw, "unknown key '" + key + "' under " + + "Output. A typo in a metadata field is a fault rather than " + + "a value that is quietly dropped"); + } + + ScriptNode name = raw.get("Name"); + if (name == null) { + fault("Output.Name", raw, "required key 'Name' is missing"); + } else if (name.getKind() != ScriptNode.Kind.STRING || + IDENTIFIER.matcher(name.getText()).matches() == false) { + fault("Output.Name", raw, "Output.Name " + jsonOf(name) + + " does not match the pattern " + IDENTIFIER.pattern()); + } else { + output.name = name.getText(); + } + if (output.name.isEmpty()) { + output.name = fallbackName; + } + + ScriptNode description = raw.get("Description"); + if (description == null) { + fault("Output.Description", raw, "required key 'Description' " + + "is missing. Say what the property asserts, not how far to " + + "trust it"); + } else if (description.getKind() != ScriptNode.Kind.STRING || + JsText.trim(description.getText()).isEmpty()) { + fault("Output.Description", raw, "Output.Description expected " + + "a non empty string, found " + typeOf(description)); + } else { + output.description = description.getText(); + } + + ScriptNode valueType = raw.get("ValueType"); + if (valueType == null) { + fault("Output.ValueType", raw, + "required key 'ValueType' is missing"); + } else if (valueType.getKind() != ScriptNode.Kind.STRING || + ValueType.fromText(valueType.getText()) == null) { + fault("Output.ValueType", raw, "Output.ValueType '" + + displayOf(valueType) + "' is not allowed in format 1. " + + "Expected one of " + String.join(", ", VALUE_TYPES)); + } else { + output.valueType = ValueType.fromText(valueType.getText()); + } + + ScriptNode isList = raw.get("IsList"); + if (isList == null) { + fault("Output.IsList", raw, "required key 'IsList' is missing"); + } else if (isList.getKind() != ScriptNode.Kind.BOOL) { + fault("Output.IsList", raw, + "Output.IsList expected a boolean, found " + typeOf(isList)); + } else if (isList.getBool()) { + fault("Output.IsList", raw, "Output.IsList must be false in " + + "format 1. List outputs are deferred"); + } + + readValues(raw, output); + + output.isMandatory = readBoolean(raw, "IsMandatory"); + output.isObsolete = readBoolean(raw, "IsObsolete"); + output.isPopular = readBoolean(raw, "IsPopular"); + output.exportValues = readBoolean(raw, "ExportValues"); + output.category = readString(raw, "Category"); + output.url = readString(raw, "Url"); + output.storedValueType = readString(raw, "StoredValueType"); + output.displayOrder = readInteger(raw, "DisplayOrder"); + output.propertyId = readInteger(raw, "PropertyId"); + output.vendorIds = readList(raw, "VendorIds"); + output.dependencies = readList(raw, "Dependencies"); + + // DefaultValue is the string form of the value, so it is checked + // against the value type and the value list. + ScriptNode defaultValue = raw.get("DefaultValue"); + if (defaultValue != null) { + if (defaultValue.getKind() != ScriptNode.Kind.STRING) { + fault("Output.DefaultValue", raw, "Output.DefaultValue " + + "expected a string holding the string form of the " + + "value, found " + typeOf(defaultValue)); + } else { + String text = defaultValue.getText(); + output.defaultValue = text; + if (output.valueType != null && + SourceValues.convertString(text, output.valueType) == + null) { + fault("Output.DefaultValue", raw, "Output.DefaultValue '" + + text + "' cannot be read as " + output.valueType); + } + if (output.hasValues && output.listsValue(text) == false) { + fault("Output.DefaultValue", raw, "Output.DefaultValue '" + + text + "' is not one of the values listed under " + + "Output.Values"); + } + } + } + return output; + } + + private Boolean readBoolean(ScriptNode raw, String key) { + ScriptNode value = raw.get(key); + if (value == null) { + return null; + } + if (value.getKind() != ScriptNode.Kind.BOOL) { + fault("Output." + key, raw, "Output." + key + + " expected a boolean, found " + typeOf(value)); + return null; + } + return value.getBool(); + } + + private String readString(ScriptNode raw, String key) { + ScriptNode value = raw.get(key); + if (value == null) { + return null; + } + if (value.getKind() != ScriptNode.Kind.STRING) { + fault("Output." + key, raw, "Output." + key + + " expected a string, found " + typeOf(value)); + return null; + } + return value.getText(); + } + + private Long readInteger(ScriptNode raw, String key) { + ScriptNode value = raw.get(key); + if (value == null) { + return null; + } + if (value.getKind() != ScriptNode.Kind.NUMBER || + isWhole(value.getNumber()) == false) { + fault("Output." + key, raw, "Output." + key + + " expected an integer, found " + typeOf(value)); + return null; + } + return (long) value.getNumber(); + } + + private ScriptNode readList(ScriptNode raw, String key) { + ScriptNode value = raw.get(key); + if (value == null) { + return null; + } + if (value.getKind() != ScriptNode.Kind.LIST) { + fault("Output." + key, raw, "Output." + key + + " expected a list, found " + typeOf(value)); + return null; + } + return value; + } + + private void readValues(ScriptNode raw, OutputModel.Draft output) { + ScriptNode values = raw.get("Values"); + if (values == null) { + return; + } + if (values.getKind() != ScriptNode.Kind.LIST) { + fault("Output.Values", raw, + "Output.Values expected a list, found " + typeOf(values)); + return; + } + if (output.valueType != null && + output.valueType != ValueType.STRING && + output.valueType != ValueType.INT) { + fault("Output.Values", raw, "Output.Values is only allowed " + + "where ValueType is string or int, not " + output.valueType); + return; + } + for (int index = 0; index < values.getItems().size(); index++) { + ScriptNode entry = values.getItems().get(index); + String path = "Output.Values[" + index + "]"; + if (entry.getKind() != ScriptNode.Kind.MAP) { + fault(path, values, "a value expected a mapping of Name " + + "and Description, found " + typeOf(entry)); + continue; + } + for (String key : unknownKeys( + entry, Arrays.asList("Name", "Description"))) { + fault(path + "." + key, entry, "unknown key '" + key + + "' in a value. Expected Name and Description"); + } + ScriptNode name = entry.get("Name"); + if (name == null || (name.getKind() != ScriptNode.Kind.STRING && + name.getKind() != ScriptNode.Kind.NUMBER)) { + fault(path + ".Name", entry, "a value must have a Name"); + continue; + } + String text = null; + ScriptNode description = entry.get("Description"); + if (description != null) { + if (description.getKind() != ScriptNode.Kind.STRING) { + fault(path + ".Description", entry, "a value " + + "Description expected a string, found " + + typeOf(description)); + } + text = description.getText(); + } + output.values.add( + new OutputModel.ValueEntry(Literal.of(name), text)); + } + List seen = new ArrayList<>(); + for (OutputModel.ValueEntry entry : output.values) { + String key = entry.getName().toString(); + if (seen.contains(key)) { + fault("Output.Values", values, + "the value '" + key + "' is listed more than once"); + } + seen.add(key); + } + output.hasValues = true; + } + + private List readCheckNames() { + ScriptNode raw = document.get("Checks"); + if (raw == null) { + return Collections.emptyList(); + } + if (raw.getKind() != ScriptNode.Kind.MAP) { + fault("Checks", document, "Checks expected a mapping of names " + + "to conditions, found " + typeOf(raw)); + return Collections.emptyList(); + } + List names = new ArrayList<>(); + for (String name : raw.getKeys()) { + if (IDENTIFIER.matcher(name).matches() == false) { + fault("Checks." + name, raw, "check name '" + name + + "' does not match the pattern " + IDENTIFIER.pattern()); + continue; + } + names.add(name); + } + return names; + } + + private List readChecks( + List checkNames) { + List checks = new ArrayList<>(); + ScriptNode raw = document.get("Checks"); + if (raw == null || raw.getKind() != ScriptNode.Kind.MAP) { + return checks; + } + for (String name : checkNames) { + Condition condition = readCondition( + raw.valueOf(name), "Checks." + name, raw, checkNames); + checks.add(new DerivedScript.NamedCheck(name, condition)); + } + return checks; + } + + private List readRules( + OutputModel.Draft output, + List checkNames) { + List rules = new ArrayList<>(); + ScriptNode raw = document.get("Rules"); + if (raw == null) { + fault("Rules", document, "required key 'Rules' is missing"); + return rules; + } + if (raw.getKind() != ScriptNode.Kind.LIST) { + fault("Rules", document, + "Rules expected a list, found " + typeOf(raw)); + return rules; + } + List items = raw.getItems(); + if (items.isEmpty()) { + fault("Rules", raw, "Rules must hold at least one rule"); + return rules; + } + // Every script ends in an Else, so a script always chooses a value + // once its source properties have been read. + if (items.get(items.size() - 1).get("Else") == null) { + fault("Rules", raw, "the last rule must be an Else, which is " + + "what a script falls back to when no earlier rule matched"); + } + for (int index = 0; index < items.size(); index++) { + ScriptNode entry = items.get(index); + String path = "Rules[" + index + "]"; + if (entry.getKind() != ScriptNode.Kind.MAP) { + fault(path, raw, "a rule expected a mapping, found " + + typeOf(entry)); + continue; + } + for (String key : unknownKeys( + entry, Arrays.asList("When", "Then", "Else"))) { + fault(path + "." + key, entry, "unknown key '" + key + + "' in a rule. Expected When and Then, or Else"); + } + ScriptNode when = entry.get("When"); + ScriptNode then = entry.get("Then"); + ScriptNode otherwise = entry.get("Else"); + boolean isLast = index == items.size() - 1; + if (otherwise != null && when != null) { + fault(path, entry, "a rule has both When and Else. A rule " + + "is either When with Then, or Else on its own"); + continue; + } + if (otherwise != null && isLast == false) { + fault(path, entry, "Else is only allowed on the last rule"); + continue; + } + if (otherwise == null && when == null) { + fault(path, entry, + "a rule needs a When, or an Else on the last rule"); + continue; + } + if (when != null && then == null) { + fault(path, entry, "a rule with When needs a Then"); + continue; + } + Literal value = otherwise != null ? + readRuleValue(output, otherwise, path + ".Else", entry) : + readRuleValue(output, then, path + ".Then", entry); + Condition condition = when == null ? null : + readCondition(when, path + ".When", entry, checkNames); + rules.add(new DerivedScript.Rule(condition, value)); + } + return rules; + } + + /** + * Read a Then or an Else, which is a literal of Output.ValueType and + * one of Output.Values where that list is given. + */ + private Literal readRuleValue( + OutputModel.Draft output, + ScriptNode raw, + String path, + ScriptNode at) { + if (raw.getKind() == ScriptNode.Kind.NULL) { + fault(path, at, "a rule value is a null literal, which format 1 " + + "does not allow"); + return Literal.NONE; + } + if (raw.isCollection()) { + fault(path, at, "a rule value is a literal of the output value " + + "type, found " + typeOf(raw)); + return Literal.NONE; + } + if (output.valueType != null && + literalMatches(inferType(raw), output.valueType) == false) { + fault(path, at, "expected a " + output.valueType + + " to match Output.ValueType, found " + typeOf(raw)); + return Literal.NONE; + } + Literal value = Literal.of(raw); + if (output.hasValues && output.listsValue(value.toString()) == false) { + List names = new ArrayList<>(); + for (OutputModel.ValueEntry entry : output.values) { + names.add(entry.getName().toString()); + } + fault(path, at, "'" + value + "' is not one of the values " + + "listed under Output.Values (" + String.join(", ", names) + + ")"); + return Literal.NONE; + } + return value; + } + + private Condition readCondition( + ScriptNode raw, + String path, + ScriptNode parent, + List checkNames) { + if (raw == null || raw.getKind() != ScriptNode.Kind.MAP) { + fault(path, parent, + "a condition expected a mapping, found " + typeOf(raw)); + return Condition.faulted(); + } + if (raw.getKeys().isEmpty()) { + fault(path, raw, "a condition is empty"); + return Condition.faulted(); + } + if (raw.get("Property") != null) { + return readComparison(raw, path); + } + if (raw.get("Check") != null) { + return readCheckReference(raw, path, checkNames); + } + if (raw.get("Passed") != null || raw.get("Failed") != null) { + return readAggregate(raw, path, checkNames); + } + if (raw.get("All") != null || raw.get("Any") != null) { + String which = raw.get("All") != null ? "All" : "Any"; + if (raw.getKeys().size() != 1) { + fault(path, raw, which + " must be the only key of its " + + "condition, found " + String.join(", ", raw.getKeys())); + } + ScriptNode items = raw.get(which); + if (items.getKind() != ScriptNode.Kind.LIST) { + fault(path + "." + which, raw, which + " expected a list of " + + "conditions, found " + typeOf(items)); + return Condition.faulted(); + } + if (items.getItems().isEmpty()) { + fault(path + "." + which, raw, + which + " must list at least one condition"); + return Condition.faulted(); + } + Condition[] members = new Condition[items.getItems().size()]; + for (int index = 0; index < members.length; index++) { + members[index] = readCondition( + items.getItems().get(index), + path + "." + which + "[" + index + "]", + items, + checkNames); + } + return Condition.compound( + which.equals("All") ? + Condition.Kind.ALL : Condition.Kind.ANY, + members); + } + if (raw.get("Not") != null) { + if (raw.getKeys().size() != 1) { + fault(path, raw, "Not must be the only key of its " + + "condition, found " + String.join(", ", raw.getKeys())); + } + Condition item = readCondition( + raw.get("Not"), path + ".Not", raw, checkNames); + return Condition.compound( + Condition.Kind.NOT, new Condition[]{item}); + } + fault(path, raw, "a condition must be a comparison, a Check " + + "reference, an aggregate, All, Any or Not. Found the keys " + + String.join(", ", raw.getKeys())); + return Condition.faulted(); + } + + private Condition readComparison(ScriptNode raw, String path) { + String propertyKey = null; + List operatorKeys = new ArrayList<>(); + for (String key : raw.getKeys()) { + if (propertyKey == null && key.equalsIgnoreCase("Property")) { + propertyKey = key; + continue; + } + operatorKeys.add(key); + } + ScriptNode property = raw.valueOf(propertyKey); + if (property.getKind() != ScriptNode.Kind.STRING || + SOURCE_PATTERN.matcher(property.getText()).matches() == false) { + fault(path + ".Property", raw, "'" + displayOf(property) + + "' is not a source property. Write it as " + + "elementKey.PropertyName, for example device.IsCrawler"); + return Condition.faulted(); + } + if (operatorKeys.isEmpty()) { + fault(path, raw, "a comparison on '" + property.getText() + + "' has no operator. Expected exactly one of " + + Operator.names()); + return Condition.faulted(); + } + List known = new ArrayList<>(); + for (String key : operatorKeys) { + if (Operator.fromText(key) != null) { + known.add(key); + } else { + fault(path, raw, "unknown operator '" + key + + "', expected one of " + Operator.names()); + } + } + if (known.isEmpty()) { + return Condition.faulted(); + } + if (operatorKeys.size() > 1) { + fault(path, raw, "a condition takes exactly one operator, found " + + String.join(", ", operatorKeys)); + return Condition.faulted(); + } + Operator operator = Operator.fromText(known.get(0)); + ScriptNode operand = raw.valueOf(known.get(0)); + if (operand.getKind() == ScriptNode.Kind.NULL) { + fault(path + "." + operator, raw, "a null literal is not " + + "allowed. Give the value to compare against"); + return Condition.faulted(); + } + + ValueType type; + if (operator.takesList()) { + if (operand.getKind() != ScriptNode.Kind.LIST) { + fault(path + "." + operator, raw, operator + " expects a " + + "list of values, found " + typeOf(operand)); + return Condition.faulted(); + } + if (operand.getItems().isEmpty()) { + fault(path + "." + operator, raw, + operator + " expects a non empty list"); + return Condition.faulted(); + } + // A member with no type, such as a mapping, joins as an empty + // string, as JavaScript joins a null. + List memberTypes = new ArrayList<>(); + boolean numeric = true; + boolean allSame = true; + for (ScriptNode member : operand.getItems()) { + if (member.getKind() == ScriptNode.Kind.NULL) { + fault(path + "." + operator, raw, + "a null literal is not allowed in a list"); + return Condition.faulted(); + } + ValueType memberType = inferType(member); + String text = memberType == null ? "" : memberType.toString(); + memberTypes.add(text); + if (memberType != ValueType.INT && + memberType != ValueType.DOUBLE) { + numeric = false; + } + if (text.equals(memberTypes.get(0)) == false) { + allSame = false; + } + } + if (numeric == false && allSame == false) { + fault(path + "." + operator, raw, "every member of a list " + + "must be of the same type, found " + + String.join(", ", memberTypes)); + return Condition.faulted(); + } + if (numeric) { + type = memberTypes.contains(ValueType.DOUBLE.toString()) ? + ValueType.DOUBLE : ValueType.INT; + } else { + type = inferType(operand.getItems().get(0)); + } + } else { + type = inferType(operand); + if (type == null) { + fault(path + "." + operator, raw, "the literal " + + jsonOf(operand) + " has no type format 1 knows"); + return Condition.faulted(); + } + } + + if (type == null || operator.getTypes().contains(type) == false) { + List allowed = new ArrayList<>(); + for (ValueType t : operator.getTypes()) { + allowed.add(t.toString()); + } + fault(path, raw, "operator '" + operator + "' is not allowed on " + + "type " + (type == null ? "null" : type.toString()) + + ". It is allowed on " + String.join(", ", allowed)); + return Condition.faulted(); + } + + int slot = useProperty(property.getText(), type, path, raw); + return Condition.comparison( + slot, property.getText(), operator, operand, type); + } + + private Condition readCheckReference( + ScriptNode raw, + String path, + List checkNames) { + if (raw.getKeys().size() != 1) { + fault(path, raw, "Check must be the only key of its condition, " + + "found " + String.join(", ", raw.getKeys())); + } + ScriptNode name = raw.get("Check"); + if (name.getKind() != ScriptNode.Kind.STRING) { + fault(path + ".Check", raw, + "Check expected the name of a check, found " + typeOf(name)); + return Condition.faulted(); + } + int index = checkNames.indexOf(name.getText()); + if (index < 0) { + fault(path + ".Check", raw, "check '" + name.getText() + + "' is not defined. The checks are " + listOrNone(checkNames)); + return Condition.faulted(); + } + return Condition.checkReference(name.getText(), index); + } + + private Condition readAggregate( + ScriptNode raw, + String path, + List checkNames) { + List aggregateKeys = new ArrayList<>(); + for (String key : raw.getKeys()) { + if (canonicalName(AGGREGATE_NAMES, key) != null) { + aggregateKeys.add(key); + } + } + if (aggregateKeys.size() > 1) { + fault(path, raw, "an aggregate condition takes one of " + + String.join(", ", AGGREGATE_NAMES) + ", found " + + String.join(", ", aggregateKeys)); + return Condition.faulted(); + } + String aggregateKey = aggregateKeys.get(0); + String aggregate = canonicalName(AGGREGATE_NAMES, aggregateKey); + List groupNames = new ArrayList<>(); + int[] group = readGroup(raw.valueOf(aggregateKey), + path + "." + aggregate, raw, checkNames, groupNames); + + List operatorKeys = new ArrayList<>(); + for (String key : raw.getKeys()) { + if (key.equals(aggregateKey) == false) { + operatorKeys.add(key); + } + } + if (operatorKeys.isEmpty()) { + fault(path, raw, "an aggregate condition has no operator. " + + "Expected exactly one of " + + String.join(", ", COUNT_OPERATORS)); + return Condition.faulted(); + } + if (operatorKeys.size() > 1) { + fault(path, raw, "a condition takes exactly one operator, found " + + String.join(", ", operatorKeys)); + return Condition.faulted(); + } + Operator operator = Operator.fromText(operatorKeys.get(0)); + if (operator == null) { + fault(path, raw, "unknown operator '" + operatorKeys.get(0) + + "', expected one of " + String.join(", ", COUNT_OPERATORS)); + return Condition.faulted(); + } + // In and NotIn are refused as well, because a count is compared + // with one whole number, never with a list. + if (COUNT_OPERATORS.contains(operator.toString()) == false) { + fault(path, raw, "operator '" + operator + "' is not allowed " + + "on a count, which is an int"); + return Condition.faulted(); + } + // A count is compared with a whole number and with nothing else. + ScriptNode operand = raw.valueOf(operatorKeys.get(0)); + if (operand.getKind() != ScriptNode.Kind.NUMBER || + isWhole(operand.getNumber()) == false) { + fault(path + "." + operator, raw, "an aggregate is compared " + + "with a whole number, found " + typeOf(operand)); + return Condition.faulted(); + } + return Condition.aggregate( + aggregate.equals("Passed"), + group, + group == null ? null : groupNames, + operator, + operand); + } + + /** + * Read the word Checks, meaning every check, which gives a null group, + * or a list of check names, which gives their indexes. + */ + private int[] readGroup( + ScriptNode raw, + String path, + ScriptNode at, + List checkNames, + List groupNames) { + if (raw.getKind() == ScriptNode.Kind.STRING) { + if (raw.getText().equalsIgnoreCase("Checks")) { + return null; + } + fault(path, at, "a group is the word Checks, meaning every " + + "check, or a list of check names. Found '" + raw.getText() + + "'"); + return new int[0]; + } + if (raw.getKind() == ScriptNode.Kind.LIST) { + List indexes = new ArrayList<>(); + for (ScriptNode name : raw.getItems()) { + if (name.getKind() != ScriptNode.Kind.STRING) { + fault(path, at, "a group lists check names, found " + + typeOf(name)); + continue; + } + int index = checkNames.indexOf(name.getText()); + if (index < 0) { + fault(path, at, "check '" + name.getText() + "' is not " + + "defined. The checks are " + listOrNone(checkNames)); + continue; + } + indexes.add(index); + groupNames.add(name.getText()); + } + int[] group = new int[indexes.size()]; + for (int i = 0; i < group.length; i++) { + group[i] = indexes.get(i); + } + return group; + } + fault(path, at, "a group is the word Checks or a list of check " + + "names, found " + typeOf(raw)); + return new int[0]; + } + + /** + * Record that a condition names a source property with the type its + * literal infers, and give the slot the property is read from. + */ + private int useProperty( + String name, + ValueType type, + String path, + ScriptNode at) { + String key = name.toLowerCase(Locale.ROOT); + PropertyUse use = uses.get(key); + if (use == null) { + use = new PropertyUse(properties.size(), type, path); + uses.put(key, use); + properties.add(new SourceProperty(name, type)); + return use.slot; + } + if (use.type != type) { + fault(path, at, "'" + name + "' is inferred as " + type + + " here but was already inferred as " + use.type + " at " + + use.typePath + ". Every use of a property must infer the " + + "same type"); + } + return use.slot; + } + + /** + * The type a literal infers. A number infers its type from its value + * rather than the way it is written, so 8, 8.0 and 8e0 all infer int, + * and a whole number outside the range of int infers double. + * @return the type, or null for a literal with no type + */ + static ValueType inferType(ScriptNode node) { + switch (node.getKind()) { + case BOOL: + return ValueType.BOOL; + case NUMBER: + if (isWhole(node.getNumber()) == false) { + return ValueType.DOUBLE; + } + return node.getNumber() >= ValueType.INT_MIN && + node.getNumber() <= ValueType.INT_MAX ? + ValueType.INT : ValueType.DOUBLE; + case STRING: + return ValueType.STRING; + default: + return null; + } + } + + /** + * Whether a literal of one type may stand for a value of another. A + * whole number reads as a double. + */ + private static boolean literalMatches( + ValueType literalType, + ValueType outputType) { + return literalType == outputType || + (literalType == ValueType.INT && outputType == ValueType.DOUBLE); + } + + static boolean isWhole(double value) { + return Double.isInfinite(value) == false && + Double.isNaN(value) == false && + value == Math.floor(value); + } + + /** + * Describe a value in a message. + */ + static String typeOf(ScriptNode node) { + if (node == null) { + return "nothing"; + } + switch (node.getKind()) { + case NULL: + return "a null literal"; + case LIST: + return "a list"; + case BOOL: + return "a boolean"; + case NUMBER: + return isWhole(node.getNumber()) ? "an integer" : "a number"; + case STRING: + return "a string (" + jsonOf(node) + ")"; + default: + return "a mapping"; + } + } + + /** + * A value as JavaScript's template strings print it. + */ + static String displayOf(ScriptNode node) { + switch (node.getKind()) { + case NULL: + return "null"; + case BOOL: + case NUMBER: + return Literal.of(node).toString(); + case STRING: + return node.getText(); + case LIST: + List parts = new ArrayList<>(); + for (ScriptNode item : node.getItems()) { + parts.add(displayOf(item)); + } + return String.join(",", parts); + default: + return "[object Object]"; + } + } + + /** + * A value as compact JSON, as JSON.stringify gives it. + */ + static String jsonOf(ScriptNode node) { + StringBuilder text = new StringBuilder(); + appendJson(text, node); + return text.toString(); + } + + private static void appendJson(StringBuilder text, ScriptNode node) { + switch (node.getKind()) { + case NULL: + text.append("null"); + break; + case BOOL: + text.append(node.getBool() ? "true" : "false"); + break; + case NUMBER: + text.append(Double.isInfinite(node.getNumber()) || + Double.isNaN(node.getNumber()) ? + "null" : JsText.number(node.getNumber())); + break; + case STRING: + text.append(JsText.quote(node.getText())); + break; + case LIST: + text.append('['); + for (int i = 0; i < node.getItems().size(); i++) { + if (i > 0) { + text.append(','); + } + appendJson(text, node.getItems().get(i)); + } + text.append(']'); + break; + default: + text.append('{'); + for (int i = 0; i < node.getKeys().size(); i++) { + if (i > 0) { + text.append(','); + } + text.append(JsText.quote(node.getKeys().get(i))) + .append(':'); + appendJson(text, node.getValues().get(i)); + } + text.append('}'); + break; + } + } + + private static List unknownKeys( + ScriptNode node, + List allowed) { + List unknown = new ArrayList<>(); + for (String key : node.getKeys()) { + if (canonicalName(allowed, key) == null) { + unknown.add(key); + } + } + return unknown; + } + + /** + * The name in names that matches the key without regard to case. + * @return the name, or null where none matches + */ + private static String canonicalName(List names, String key) { + for (String name : names) { + if (name.equalsIgnoreCase(key)) { + return name; + } + } + return null; + } + + private static String listOrNone(List names) { + return names.isEmpty() ? "(none)" : String.join(", ", names); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedValueMetaData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedValueMetaData.java new file mode 100644 index 000000000..509daa525 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedValueMetaData.java @@ -0,0 +1,52 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * One entry of a script's Output.Values, being a value the derived property + * can have. + */ +public final class DerivedValueMetaData { + + private final String name; + private final String description; + + DerivedValueMetaData(String name, String description) { + this.name = name; + this.description = description; + } + + /** + * @return the string form of the value, as a rule's value is compared + */ + public String getName() { + return name; + } + + /** + * @return the description, or null where the script gives none + */ + public String getDescription() { + return description; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/IDerivedPropertyElement.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/IDerivedPropertyElement.java new file mode 100644 index 000000000..fbbaff234 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/IDerivedPropertyElement.java @@ -0,0 +1,45 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; + +import java.util.List; + +/** + * A Flow Element that produces new property values from property values + * other elements in the Pipeline have already produced, following rules + * written in a script, and writes them under the Element Data key + * {@code derived}. + */ +public interface IDerivedPropertyElement + extends FlowElement { + + /** + * @return the metadata of every script the element holds, in the order + * the scripts were added + */ + List getScripts(); +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/JsText.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/JsText.java new file mode 100644 index 000000000..33c602fa7 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/JsText.java @@ -0,0 +1,181 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.math.BigDecimal; + +/** + * Text written the way the JavaScript reference tools write it, so that a + * message, a number read as a string and the canonical form read the same + * in every language. + */ +final class JsText { + + /** + * From this size up, and below {@link #SMALLEST_PLAIN}, JavaScript + * writes a number with an exponent. + */ + private static final double LARGEST_PLAIN = 1e21; + + private static final double SMALLEST_PLAIN = 1e-6; + + private static final char LAST_CONTROL = 0x1f; + private static final char VERTICAL_TAB = 0x0b; + private static final String HEX_DIGITS = "0123456789abcdef"; + private static final int NIBBLE_BITS = 4; + private static final int NIBBLE_MASK = 0xf; + private static final int UNICODE_ESCAPE_DIGITS = 4; + + private JsText() { + } + + /** + * Write a number as JavaScript's String does. + * @param value the number + * @return the text + */ + static String number(double value) { + if (Double.isNaN(value)) { + return "NaN"; + } + if (Double.isInfinite(value)) { + return value > 0 ? "Infinity" : "-Infinity"; + } + if (value == 0) { + return "0"; + } + BigDecimal decimal = + new BigDecimal(Double.toString(value)).stripTrailingZeros(); + double size = Math.abs(value); + if (size >= SMALLEST_PLAIN && size < LARGEST_PLAIN) { + return decimal.toPlainString(); + } + String digits = decimal.unscaledValue().abs().toString(); + int exponent = digits.length() - 1 - decimal.scale(); + StringBuilder text = new StringBuilder(); + if (value < 0) { + text.append('-'); + } + text.append(digits.charAt(0)); + if (digits.length() > 1) { + text.append('.').append(digits, 1, digits.length()); + } + text.append('e').append(exponent < 0 ? '-' : '+') + .append(Math.abs(exponent)); + return text.toString(); + } + + /** + * Quote a string as JSON.stringify does. + * @param value the string + * @return the quoted string + */ + static String quote(String value) { + StringBuilder text = new StringBuilder(value.length() + 2); + text.append('"'); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': + text.append("\\\""); + break; + case '\\': + text.append("\\\\"); + break; + case '\b': + text.append("\\b"); + break; + case '\f': + text.append("\\f"); + break; + case '\n': + text.append("\\n"); + break; + case '\r': + text.append("\\r"); + break; + case '\t': + text.append("\\t"); + break; + default: + if (c <= LAST_CONTROL || isLoneSurrogate(value, i)) { + unicodeEscape(text, c); + } else { + text.append(c); + } + break; + } + } + return text.append('"').toString(); + } + + /** + * Whether a character is white space as JavaScript's trim reads it. + * @param c the character + * @return true for white space or a line terminator + */ + static boolean isWhitespace(char c) { + return c == '\t' || c == '\n' || c == VERTICAL_TAB || c == '\f' || + c == '\r' || c == '
' || c == '
' || c == '' || + Character.getType(c) == Character.SPACE_SEPARATOR; + } + + /** + * Remove leading and trailing white space as JavaScript's trim does. + * @param value the text + * @return the text without the white space around it + */ + static String trim(String value) { + int start = 0; + int end = value.length(); + while (start < end && isWhitespace(value.charAt(start))) { + start++; + } + while (end > start && isWhitespace(value.charAt(end - 1))) { + end--; + } + return value.substring(start, end); + } + + private static boolean isLoneSurrogate(String value, int i) { + char c = value.charAt(i); + if (Character.isHighSurrogate(c)) { + return i + 1 >= value.length() || + Character.isLowSurrogate(value.charAt(i + 1)) == false; + } + if (Character.isLowSurrogate(c)) { + return i == 0 || + Character.isHighSurrogate(value.charAt(i - 1)) == false; + } + return false; + } + + private static void unicodeEscape(StringBuilder text, char c) { + text.append("\\u"); + for (int shift = (UNICODE_ESCAPE_DIGITS - 1) * NIBBLE_BITS; + shift >= 0; + shift -= NIBBLE_BITS) { + text.append(HEX_DIGITS.charAt((c >> shift) & NIBBLE_MASK)); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Literal.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Literal.java new file mode 100644 index 000000000..3682ccfe4 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Literal.java @@ -0,0 +1,93 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * A scalar from a script, being a boolean, a number or a string. + */ +final class Literal { + + /** + * What a value the validator refused becomes. The model holding it is + * discarded before anything reads it. + */ + static final Literal NONE = + new Literal(ScriptNode.Kind.NULL, false, 0, ""); + + private final ScriptNode.Kind kind; + private final boolean bool; + private final double number; + private final String text; + + private Literal( + ScriptNode.Kind kind, + boolean bool, + double number, + String text) { + this.kind = kind; + this.bool = bool; + this.number = number; + this.text = text; + } + + static Literal of(ScriptNode node) { + return new Literal( + node.getKind(), + node.getBool(), + node.getNumber(), + node.getText()); + } + + ScriptNode.Kind getKind() { + return kind; + } + + boolean getBool() { + return bool; + } + + double getNumber() { + return number; + } + + String getText() { + return text; + } + + /** + * The string form of the literal, as JavaScript's String gives it, + * which is how a rule's value is compared with Output.Values. + * @return the string form + */ + @Override + public String toString() { + switch (kind) { + case BOOL: + return bool ? "true" : "false"; + case NUMBER: + return JsText.number(number); + default: + return text; + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Operator.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Operator.java new file mode 100644 index 000000000..6deb51f67 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Operator.java @@ -0,0 +1,106 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * The operators of format 1, each with the types it is allowed on. + */ +enum Operator { + EQ("Eq", ValueType.BOOL, ValueType.INT, ValueType.DOUBLE, + ValueType.STRING), + NE("Ne", ValueType.BOOL, ValueType.INT, ValueType.DOUBLE, + ValueType.STRING), + GT("Gt", ValueType.INT, ValueType.DOUBLE), + GE("Ge", ValueType.INT, ValueType.DOUBLE), + LT("Lt", ValueType.INT, ValueType.DOUBLE), + LE("Le", ValueType.INT, ValueType.DOUBLE), + IN("In", ValueType.BOOL, ValueType.INT, ValueType.DOUBLE, + ValueType.STRING), + NOT_IN("NotIn", ValueType.BOOL, ValueType.INT, ValueType.DOUBLE, + ValueType.STRING), + STARTS_WITH("StartsWith", ValueType.STRING), + ENDS_WITH("EndsWith", ValueType.STRING), + CONTAINS("Contains", ValueType.STRING); + + private final String text; + private final List types; + + Operator(String text, ValueType... types) { + this.text = text; + this.types = Collections.unmodifiableList(Arrays.asList(types)); + } + + /** + * @return the types the operator is allowed on + */ + List getTypes() { + return types; + } + + /** + * @return true for In and NotIn, which take a list + */ + boolean takesList() { + return this == IN || this == NOT_IN; + } + + /** + * The operator a key names, matched without regard to case. + * @param key the key in the script + * @return the operator, or null where format 1 has no such operator + */ + static Operator fromText(String key) { + for (Operator operator : values()) { + if (operator.text.equalsIgnoreCase(key)) { + return operator; + } + } + return null; + } + + /** + * @return every operator's name, separated by a comma and a space + */ + static String names() { + StringBuilder names = new StringBuilder(); + for (Operator operator : values()) { + if (names.length() > 0) { + names.append(", "); + } + names.append(operator.text); + } + return names.toString(); + } + + /** + * @return the name format 1 gives the operator, such as Eq + */ + @Override + public String toString() { + return text; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/OutputModel.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/OutputModel.java new file mode 100644 index 000000000..795eb3e88 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/OutputModel.java @@ -0,0 +1,230 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The Output block of a script, which is the whole definition of the + * derived property. A field the script does not give is null. + */ +final class OutputModel { + + /** + * One entry of Output.Values. + */ + static final class ValueEntry { + private final Literal name; + private final String description; + + ValueEntry(Literal name, String description) { + this.name = name; + this.description = description; + } + + Literal getName() { + return name; + } + + /** + * @return the description, or null where the script gives none + */ + String getDescription() { + return description; + } + } + + /** + * The Output block while the validator reads it. + */ + static final class Draft { + String name = ""; + String description = ""; + ValueType valueType; + String storedValueType; + String defaultValue; + boolean isList; + Boolean isMandatory; + Boolean isObsolete; + String category; + Boolean isPopular; + Boolean exportValues; + String url; + Long displayOrder; + Long propertyId; + ScriptNode vendorIds; + ScriptNode dependencies; + final List values = new ArrayList<>(); + boolean hasValues; + + /** + * Whether the string form of a value is one of the names under + * Output.Values. + */ + boolean listsValue(String value) { + for (ValueEntry entry : values) { + if (entry.getName().toString().equals(value)) { + return true; + } + } + return false; + } + } + + private final String name; + private final String description; + private final ValueType valueType; + private final String storedValueType; + private final String defaultValue; + private final boolean isList; + private final Boolean isMandatory; + private final Boolean isObsolete; + private final String category; + private final Boolean isPopular; + private final Boolean exportValues; + private final String url; + private final Long displayOrder; + private final Long propertyId; + private final ScriptNode vendorIds; + private final ScriptNode dependencies; + private final List computed; + private final List values; + + /** + * @param draft the block as read + * @param computed every source property the script names, used where + * the script gives no Dependencies + */ + OutputModel(Draft draft, List computed) { + this.name = draft.name; + this.description = draft.description; + this.valueType = draft.valueType; + this.storedValueType = draft.storedValueType; + this.defaultValue = draft.defaultValue; + this.isList = draft.isList; + this.isMandatory = draft.isMandatory; + this.isObsolete = draft.isObsolete; + this.category = draft.category; + this.isPopular = draft.isPopular; + this.exportValues = draft.exportValues; + this.url = draft.url; + this.displayOrder = draft.displayOrder; + this.propertyId = draft.propertyId; + this.vendorIds = draft.vendorIds; + this.dependencies = draft.dependencies; + this.computed = Collections.unmodifiableList( + new ArrayList<>(computed)); + this.values = draft.hasValues ? + Collections.unmodifiableList(new ArrayList<>(draft.values)) : + null; + } + + String getName() { + return name; + } + + String getDescription() { + return description; + } + + ValueType getValueType() { + return valueType; + } + + String getStoredValueType() { + return storedValueType; + } + + String getDefaultValue() { + return defaultValue; + } + + boolean isList() { + return isList; + } + + Boolean getIsMandatory() { + return isMandatory; + } + + Boolean getIsObsolete() { + return isObsolete; + } + + String getCategory() { + return category; + } + + Boolean getIsPopular() { + return isPopular; + } + + Boolean getExportValues() { + return exportValues; + } + + String getUrl() { + return url; + } + + Long getDisplayOrder() { + return displayOrder; + } + + Long getPropertyId() { + return propertyId; + } + + /** + * @return the VendorIds list as written, or null + */ + ScriptNode getVendorIds() { + return vendorIds; + } + + /** + * @return the Dependencies list as written, or null where the script + * gives none and {@link #getComputed()} holds them + */ + ScriptNode getDependencies() { + return dependencies; + } + + /** + * @return every source property the checks and the rules name, in + * elementKey.PropertyName form + */ + List getComputed() { + return computed; + } + + /** + * @return the entries of Output.Values, or null where the script gives + * no list + */ + List getValues() { + return values; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptNode.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptNode.java new file mode 100644 index 000000000..800d506e0 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptNode.java @@ -0,0 +1,193 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Collections; +import java.util.List; + +/** + * One value of a parsed script. A mapping keeps its keys in the order they + * are written, because checks are numbered and faults reported in that + * order. Numbers are held as doubles, because format 1 infers a number's + * type from its value rather than from how it was written. + */ +final class ScriptNode { + + /** + * The kinds of value a node holds. + */ + enum Kind { NULL, BOOL, NUMBER, STRING, LIST, MAP } + + private final Kind kind; + private final boolean bool; + private final double number; + private final String text; + private final List keys; + private final List values; + private final List items; + private final int line; + + private ScriptNode( + Kind kind, + boolean bool, + double number, + String text, + List keys, + List values, + List items, + int line) { + this.kind = kind; + this.bool = bool; + this.number = number; + this.text = text; + this.keys = keys; + this.values = values; + this.items = items; + this.line = line; + } + + static ScriptNode ofNull(int line) { + return new ScriptNode(Kind.NULL, false, 0, "", + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), line); + } + + static ScriptNode ofBool(boolean value, int line) { + return new ScriptNode(Kind.BOOL, value, 0, "", + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), line); + } + + static ScriptNode ofNumber(double value, int line) { + return new ScriptNode(Kind.NUMBER, false, value, "", + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), line); + } + + static ScriptNode ofString(String value, int line) { + return new ScriptNode(Kind.STRING, false, 0, value, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), line); + } + + static ScriptNode ofList(List items, int line) { + return new ScriptNode(Kind.LIST, false, 0, "", + Collections.emptyList(), + Collections.emptyList(), + Collections.unmodifiableList(items), line); + } + + static ScriptNode ofMap( + List keys, + List values, + int line) { + return new ScriptNode(Kind.MAP, false, 0, "", + Collections.unmodifiableList(keys), + Collections.unmodifiableList(values), + Collections.emptyList(), line); + } + + Kind getKind() { + return kind; + } + + boolean getBool() { + return bool; + } + + double getNumber() { + return number; + } + + /** + * @return the text of a string, or empty for any other kind + */ + String getText() { + return text; + } + + /** + * @return the keys of a mapping, in the order written + */ + List getKeys() { + return keys; + } + + /** + * @return the values of a mapping, in the order of its keys + */ + List getValues() { + return values; + } + + /** + * @return the members of a list + */ + List getItems() { + return items; + } + + /** + * @return the one based line the value starts on, or 0 where unknown + */ + int getLine() { + return line; + } + + boolean isCollection() { + return kind == Kind.MAP || kind == Kind.LIST; + } + + /** + * The value of a mapping's key, matched without regard to case. + * @param name the key + * @return the value, or null where this is not a mapping or has no + * such key + */ + ScriptNode get(String name) { + for (int i = 0; i < keys.size(); i++) { + if (keys.get(i).equalsIgnoreCase(name)) { + return values.get(i); + } + } + return null; + } + + /** + * The value of a mapping's key, matched exactly as written. + * @param key the key + * @return the value, or null where there is no such key + */ + ScriptNode valueOf(String key) { + for (int i = 0; i < keys.size(); i++) { + if (keys.get(i).equals(key)) { + return values.get(i); + } + } + return null; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptParseException.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptParseException.java new file mode 100644 index 000000000..12f245323 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptParseException.java @@ -0,0 +1,45 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * Text that is not YAML or JSON, or is not a mapping at the top level. + */ +final class ScriptParseException extends Exception { + + private static final long serialVersionUID = 1L; + + private final int line; + + ScriptParseException(String message, int line) { + super(message); + this.line = line; + } + + /** + * @return the one based line, or 0 where the parser gave none + */ + int getLine() { + return line; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Slot.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Slot.java new file mode 100644 index 000000000..ffae41a17 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Slot.java @@ -0,0 +1,108 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * One source property as read for one request, converted to the type the + * script infers for it, or the reason it could not be read. + */ +final class Slot { + + private final boolean available; + private final boolean bool; + private final long integer; + private final double real; + private final String text; + private final String reason; + + private Slot( + boolean available, + boolean bool, + long integer, + double real, + String text, + String reason) { + this.available = available; + this.bool = bool; + this.integer = integer; + this.real = real; + this.text = text; + this.reason = reason; + } + + static Slot ofBool(boolean value) { + return new Slot(true, value, 0, 0, null, null); + } + + static Slot ofInt(long value) { + return new Slot(true, false, value, 0, null, null); + } + + static Slot ofDouble(double value) { + return new Slot(true, false, 0, value, null, null); + } + + static Slot ofString(String value) { + return new Slot(true, false, 0, 0, value, null); + } + + static Slot absent(String reason) { + return new Slot(false, false, 0, 0, null, reason); + } + + boolean isAvailable() { + return available; + } + + boolean getBool() { + return bool; + } + + long getInt() { + return integer; + } + + double getDouble() { + return real; + } + + String getString() { + return text; + } + + /** + * @return why the property could not be read, or null where it was + */ + String getReason() { + return reason; + } + + /** + * The value as a number, for a property read as int or double. + * @param type the type the property is read as + * @return the value + */ + double number(ValueType type) { + return type == ValueType.INT ? integer : real; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceProperty.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceProperty.java new file mode 100644 index 000000000..30ff72d30 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceProperty.java @@ -0,0 +1,93 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Locale; + +/** + * A property another element produces, which a script reads as the type it + * infers from the literals compared with it. + */ +final class SourceProperty { + + private final String key; + private final String name; + private final ValueType type; + private final String elementKey; + private final String writtenKey; + private final String propertyName; + + /** + * @param name the name as the script first writes it, being + * elementKey.PropertyName + * @param type the type inferred for it + */ + SourceProperty(String name, ValueType type) { + int dot = name.indexOf('.'); + this.key = name.toLowerCase(Locale.ROOT); + this.name = name; + this.type = type; + this.writtenKey = name.substring(0, dot); + this.elementKey = writtenKey.toLowerCase(Locale.ROOT); + this.propertyName = name.substring(dot + 1); + } + + /** + * @return the name in lower case, which identifies the property + */ + String getKey() { + return key; + } + + /** + * @return the name as the script first writes it + */ + String getName() { + return name; + } + + ValueType getType() { + return type; + } + + /** + * @return the Element Data key, in lower case + */ + String getElementKey() { + return elementKey; + } + + /** + * @return the Element Data key as the script writes it + */ + String getWrittenKey() { + return writtenKey; + } + + /** + * @return the property half of the name, as written + */ + String getPropertyName() { + return propertyName; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceRead.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceRead.java new file mode 100644 index 000000000..67fcd0786 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceRead.java @@ -0,0 +1,52 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * One source property, read once per request as one type, however many + * scripts name it. + */ +final class SourceRead { + + private final SourceProperty property; + private final int key; + + /** + * @param property the property and the type it is read as + * @param key the index of the Element Data key among the element's + * keys, or -1 for a derived property, which is read when + * the script that names it runs + */ + SourceRead(SourceProperty property, int key) { + this.property = property; + this.key = key; + } + + SourceProperty getProperty() { + return property; + } + + int getKey() { + return key; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValues.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValues.java new file mode 100644 index 000000000..f434ea5a0 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValues.java @@ -0,0 +1,365 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WktString; +import fiftyone.pipeline.engines.data.AspectPropertyValue; + +import java.math.BigInteger; +import java.util.List; +import java.util.Locale; + +/** + * Reads a source property from the Element Data of the element that + * supplies it, and converts the value to the type the script infers. A + * value arrives as its native type or as a string, and both are accepted, + * but values are never coerced loosely, so N/A, Unknown and the empty + * string never become false or zero. + *

+ * The wording of every reason is that of readSlot in tools/evaluate.mjs in + * the derived-properties repository. See + * format + * 1, absent and invalid source properties. + */ +final class SourceValues { + + /** + * The reason given where the property is not in the Element Data, or + * its value is null. + */ + static final String NOT_PRESENT = "property not present on this request"; + + /** + * Stands for a list whose members are not all weighted values. + */ + private static final Object NOT_WEIGHTED = new Object(); + + /** + * The bits in a long, which a BigInteger must fit inside to be read as + * a whole number. + */ + private static final int LONG_BITS = 63; + + private SourceValues() { + } + + /** + * Read a property from the Element Data of its source element. + * @param data the Element Data, or null where the request carries none + * under the key + * @param property the property + * @return the value, converted, or the reason it could not be read + */ + static Slot read(ElementData data, SourceProperty property) { + if (data == null) { + return notAvailable(property, NOT_PRESENT); + } + Object raw; + try { + raw = data.get(property.getPropertyName()); + } catch (RuntimeException e) { + return notAvailable(property, NOT_PRESENT); + } + if (raw == null) { + return notAvailable(property, NOT_PRESENT); + } + return readValue(raw, property); + } + + /** + * Convert a value an element holds. A value with no value carries its + * own message, a list of weighted values gives the value with the + * highest weight, and any other list cannot be read where one value is + * needed. + */ + static Slot readValue(Object value, SourceProperty property) { + Object raw = value; + if (raw instanceof AspectPropertyValue) { + AspectPropertyValue wrapped = (AspectPropertyValue) raw; + if (wrapped.hasValue() == false) { + String message = wrapped.getNoValueMessage(); + return notAvailable(property, + message == null || message.isEmpty() ? + NOT_PRESENT : message); + } + raw = wrapped.getValue(); + if (raw == null) { + return notAvailable(property, NOT_PRESENT); + } + } + if (raw instanceof List) { + raw = heaviest((List) raw); + if (raw == NOT_WEIGHTED) { + return Slot.absent( + "held a list where a single value is needed"); + } + if (raw == null) { + return notAvailable(property, NOT_PRESENT); + } + } else if (raw instanceof IWeightedValue) { + raw = ((IWeightedValue) raw).getValue(); + if (raw == null) { + return notAvailable(property, NOT_PRESENT); + } + } + Slot slot = convert(raw, property.getType()); + if (slot != null) { + return slot; + } + return Slot.absent("held '" + display(raw) + + "' which cannot be read as " + property.getType()); + } + + private static Slot notAvailable(SourceProperty property, String detail) { + return Slot.absent("element '" + property.getWrittenKey() + + "' has no value for '" + property.getPropertyName() + "': " + + detail); + } + + /** + * The value with the highest weight from a list of weighted values, the + * first where two weigh the same. + * @return the value, or {@link #NOT_WEIGHTED} where the list is empty or + * any member is not a weighted value + */ + private static Object heaviest(List list) { + if (list.isEmpty()) { + return NOT_WEIGHTED; + } + IWeightedValue best = null; + for (Object member : list) { + if (member instanceof IWeightedValue == false) { + return NOT_WEIGHTED; + } + IWeightedValue weighted = (IWeightedValue) member; + if (best == null || + weighted.getRawWeighting() > best.getRawWeighting()) { + best = weighted; + } + } + return best.getValue(); + } + + /** + * Convert a source value, which arrives as its native type or as a + * string. + * @return the value, or null where it cannot be read as the type + */ + static Slot convert(Object raw, ValueType type) { + if (raw instanceof String) { + return convertString((String) raw, type); + } + if (raw instanceof WktString) { + String text = raw.toString(); + return text == null ? null : convertString(text, type); + } + if (raw instanceof Boolean) { + boolean value = (Boolean) raw; + switch (type) { + case BOOL: + return Slot.ofBool(value); + case STRING: + return Slot.ofString(value ? "True" : "False"); + default: + return null; + } + } + Long whole = wholeNumber(raw); + if (whole != null) { + long value = whole; + switch (type) { + case INT: + return value < ValueType.INT_MIN || + value > ValueType.INT_MAX ? + null : Slot.ofInt(value); + case DOUBLE: + return Slot.ofDouble(value); + case STRING: + return Slot.ofString(Long.toString(value)); + default: + return null; + } + } + if (raw instanceof Double || raw instanceof Float) { + double value = ((Number) raw).doubleValue(); + switch (type) { + case INT: + return DerivedScriptValidator.isWhole(value) == false || + value < ValueType.INT_MIN || + value > ValueType.INT_MAX ? + null : Slot.ofInt((long) value); + case DOUBLE: + return Double.isInfinite(value) || Double.isNaN(value) ? + null : Slot.ofDouble(value); + case STRING: + return Slot.ofString(JsText.number(value)); + default: + return null; + } + } + return null; + } + + /** + * A value of one of Java's whole number types, or null for any other. + */ + private static Long wholeNumber(Object raw) { + if (raw instanceof Integer || raw instanceof Long || + raw instanceof Short || raw instanceof Byte) { + return ((Number) raw).longValue(); + } + if (raw instanceof BigInteger && + ((BigInteger) raw).bitLength() <= LONG_BITS) { + return ((BigInteger) raw).longValue(); + } + return null; + } + + /** + * Convert the string form of a value. + * @return the value, or null where it cannot be read as the type + */ + static Slot convertString(String raw, ValueType type) { + switch (type) { + case BOOL: { + String trimmed = JsText.trim(raw).toLowerCase(Locale.ROOT); + if (trimmed.equals("true")) { + return Slot.ofBool(true); + } + if (trimmed.equals("false")) { + return Slot.ofBool(false); + } + return null; + } + case INT: { + String trimmed = JsText.trim(raw); + if (isIntegerText(trimmed) == false) { + return null; + } + try { + long value = Long.parseLong(trimmed); + return value < ValueType.INT_MIN || + value > ValueType.INT_MAX ? + null : Slot.ofInt(value); + } catch (NumberFormatException e) { + return null; + } + } + case DOUBLE: { + String trimmed = JsText.trim(raw); + if (isDecimalText(trimmed) == false) { + return null; + } + double value = Double.parseDouble(trimmed); + return Double.isInfinite(value) || Double.isNaN(value) ? + null : Slot.ofDouble(value); + } + case STRING: + return Slot.ofString(raw); + default: + return null; + } + } + + /** + * Whether text is an optional sign and then ASCII digits. + */ + private static boolean isIntegerText(String text) { + int start = startAfterSign(text); + return start < text.length() && + digits(text, start) == text.length() - start; + } + + /** + * Whether text is an optional sign, digits with a full stop as the + * decimal separator, and an optional exponent. + */ + private static boolean isDecimalText(String text) { + int position = startAfterSign(text); + int whole = digits(text, position); + position += whole; + int fraction = 0; + if (position < text.length() && text.charAt(position) == '.') { + position++; + fraction = digits(text, position); + position += fraction; + } + if (whole == 0 && fraction == 0) { + return false; + } + if (position < text.length() && + (text.charAt(position) == 'e' || text.charAt(position) == 'E')) { + position++; + if (position < text.length() && + (text.charAt(position) == '+' || + text.charAt(position) == '-')) { + position++; + } + int exponent = digits(text, position); + if (exponent == 0) { + return false; + } + position += exponent; + } + return position == text.length(); + } + + private static int startAfterSign(String text) { + return text.length() > 0 && + (text.charAt(0) == '+' || text.charAt(0) == '-') ? 1 : 0; + } + + /** + * The number of ASCII digits in text from start. + */ + private static int digits(String text, int start) { + int position = start; + while (position < text.length() && + text.charAt(position) >= '0' && text.charAt(position) <= '9') { + position++; + } + return position - start; + } + + /** + * A source value as a message prints it. + */ + private static String display(Object raw) { + if (raw instanceof String) { + return (String) raw; + } + if (raw instanceof Boolean) { + return (Boolean) raw ? "True" : "False"; + } + Long whole = wholeNumber(raw); + if (whole != null) { + return whole.toString(); + } + if (raw instanceof Double || raw instanceof Float) { + return JsText.number(((Number) raw).doubleValue()); + } + return String.valueOf(raw); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ValueType.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ValueType.java new file mode 100644 index 000000000..6de338937 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ValueType.java @@ -0,0 +1,85 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Locale; + +/** + * The four value types format 1 has, each with the Java type a value of it + * is held as. + */ +enum ValueType { + STRING("string", String.class), + BOOL("bool", Boolean.class), + INT("int", Integer.class), + DOUBLE("double", Double.class); + + /** + * The smallest value of int, which format 1 fixes at a signed 32 bit + * whole number so that one script gives one answer in every language. + */ + static final long INT_MIN = Integer.MIN_VALUE; + + /** + * The largest value of int. + */ + static final long INT_MAX = Integer.MAX_VALUE; + + private final String text; + private final Class valueClass; + + ValueType(String text, Class valueClass) { + this.text = text; + this.valueClass = valueClass; + } + + /** + * @return the Java type a value of this type is held as + */ + Class getValueClass() { + return valueClass; + } + + /** + * The value type a script names, matched without regard to case. + * @param text the name in the script + * @return the type, or null where format 1 has no such type + */ + static ValueType fromText(String text) { + String lower = text.toLowerCase(Locale.ROOT); + for (ValueType type : values()) { + if (type.text.equals(lower)) { + return type; + } + } + return null; + } + + /** + * @return the name format 1 gives the type, such as int + */ + @Override + public String toString() { + return text; + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java new file mode 100644 index 000000000..8246fc9cb --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java @@ -0,0 +1,374 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty; + +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedOutputMetaData; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyElement; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyElementBuilder; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyMetaData; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedValueMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; +import pipeline.developerexamples.derivedproperty.flowelements.SessionElement; +import pipeline.developerexamples.derivedproperty.flowelements.SessionElementBuilder; + +import java.io.File; +import java.io.PrintStream; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Arrays; +import java.util.List; + +/** + * @example derivedproperty/Main.java + * + * The Derived Property Element computes new properties from properties + * that other elements in the Pipeline have already produced, following a + * script written in YAML or in JSON. The element holds no data file, sends + * no request over the network and needs no resource key, so this example + * runs offline. + * + * The example shows four things. + * + * 1. A script held in the example as text, passed to the builder. + * 2. A script read from a file beside the example, + * src/main/resources/ReaderEngagement.yaml. + * 3. The same Pipeline built from a configuration file, + * src/main/resources/derived-property.xml, naming the builder and + * passing the script through the ScriptFiles build parameter. + * 4. What a caller sees when a source property a script names is not + * available, which is a property with no value and a message naming + * what was missing. + * + * The source properties come from a small element written in this example, + * SessionElement, so that nothing outside is needed. A real Pipeline puts + * device detection or IP intelligence in that position instead. + * + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java). + * + * The script file the example reads is + * @include pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml + * + * and the configuration file is + * @include pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml + */ +public class Main { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * The script file beside the example, which both halves of the example + * use, one by naming it on the builder and one by naming it in the + * configuration file. + */ + static final String SCRIPT_FILE = "ReaderEngagement.yaml"; + + /** + * The configuration file beside the example. + */ + static final String CONFIGURATION_FILE = "derived-property.xml"; + + /** + * The property the script in {@link #SCRIPT_FILE} produces. + */ + static final String ENGAGEMENT = "ReaderEngagement"; + + /** + * The property the script held below produces. + */ + static final String EVERY_TEST = "ReaderEngagementIsHigh"; + + /** + * A second script, held as text rather than in a file, to show that a + * script can come straight from your own code. The text is YAML here, + * and JSON is read just as well. It reads the same three source + * properties and says whether every one of the tests passed, so one + * element runs more than one script over one set of source properties. + */ + static final String EVERY_TEST_SCRIPT = String.join("\n", + "Format: 1", + "Name: ReaderEngagementIsHigh", + "Version: \"1.0.0\"", + "Output:", + " Name: ReaderEngagementIsHigh", + " Description: Whether every one of the ReaderEngagement tests " + + "passed on this request.", + " ValueType: bool", + " IsList: false", + " Category: Example", + "Checks:", + " SeveralPages: { Property: session.PagesViewed, Ge: 3 }", + " LongEnough: { Property: session.SecondsSincePageLoad, Ge: 30 }", + " PointerMoved: { Property: session.PointerMoved, Eq: true }", + "Rules:", + " - When: { Failed: Checks, Eq: 0 }", + " Then: true", + " - Else: false", + ""); + + /** + * The rule above and below each part of the output. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + public static class Example { + + /** + * Run the example. + * @param output where to write what the example shows + * @throws Exception where the example cannot run + */ + public void run(PrintStream output) throws Exception { + runFromCode(output); + runFromConfiguration(output); + } + + /** + * The first way to configure the element, being the builder in + * code. One element runs two scripts, one held as text and one read + * from a file. + */ + private void runFromCode(PrintStream output) throws Exception { + output.println(RULE); + output.println("1. Configured with the builder in code"); + output.println(); +//! [usage] + SessionElement session = + new SessionElementBuilder(loggerFactory).build(); + DerivedPropertyElement derived = + new DerivedPropertyElementBuilder(loggerFactory) + // A script from your own environment. A path may hold a + // wildcard, so "scripts/*.yaml" adds a whole folder. + .addScriptFile(resource(SCRIPT_FILE).getPath()) + // A script held as text in your own code. The name + // given here must equal the Name the script carries. + .addScript(EVERY_TEST, EVERY_TEST_SCRIPT) + // addScript(BuiltInScript.HumanConfidence) adds a + // script that ships inside the package. Those scripts + // come from the derived-properties repository, and + // this example uses none of them so that it stands on + // its own. + .build(); + + // The Derived Property Element goes after every element that + // supplies a source property. The Pipeline closes both elements + // when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(session) + .addFlowElement(derived) + .setAutoCloseElements(true) + .build()) { + List properties = Arrays.asList(ENGAGEMENT, EVERY_TEST); + process(output, pipeline, + "Request 1, every source property present.", + 5, 120, true, properties); + process(output, pipeline, + "Request 2, the pointer has not been seen to move.", + 5, 120, null, properties); + } +//! [usage] + output.println("One element runs both scripts, so both " + + "properties are written under the"); + output.println("key derived, and both say the same thing about " + + "the second request because"); + output.println("they read the same source properties. The " + + "second request has no value at"); + output.println("all, because a property the scripts name was " + + "not there, and the message"); + output.println("says which property was missing and what the " + + "element that supplies it said."); + output.println(); + } + + /** + * The second way to configure the element, being a configuration + * file. The file names the builders and the script file, so the + * Pipeline is described entirely outside the code. + */ + private void runFromConfiguration(PrintStream output) + throws Exception { + output.println(RULE); + output.println("2. The same Pipeline built from " + + CONFIGURATION_FILE); + output.println(); + PipelineOptions options = PipelineOptionsFactory + .getOptionsFromFile(resource(CONFIGURATION_FILE)); + // The Pipeline builder makes each element with the builder the + // file names. A script text held in code cannot be given in a + // configuration file, so this Pipeline runs the file's script. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .buildFromConfiguration(options)) { + List properties = Arrays.asList(ENGAGEMENT); + process(output, pipeline, + "Request 1, every source property present.", + 5, 120, true, properties); + process(output, pipeline, + "Request 2, the pointer has not been seen to move.", + 5, 120, null, properties); + printDefinitions(output, + pipeline.getElement(DerivedPropertyElement.class)); + } + } + } + + /** + * A file beside the example, which the build copies onto the class + * path. + */ + static File resource(String name) throws URISyntaxException { + URL url = Main.class.getClassLoader().getResource(name); + if (url == null) { + throw new IllegalStateException( + "The file " + name + " is not on the class path."); + } + return new File(url.toURI()); + } + + /** + * Process one request and print the source properties and the derived + * properties it produced. + * @param pointerMoved evidence for session.PointerMoved, or null to + * leave the property with no value on this request + */ + private static void process( + PrintStream output, + Pipeline pipeline, + String title, + int pagesViewed, + int seconds, + Boolean pointerMoved, + List derivedProperties) throws Exception { + output.println(title); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(SessionElement.PAGES_VIEWED_EVIDENCE, pagesViewed); + data.addEvidence(SessionElement.SECONDS_EVIDENCE, seconds); + if (pointerMoved != null) { + data.addEvidence( + SessionElement.POINTER_MOVED_EVIDENCE, pointerMoved); + } + data.process(); + + // The source properties, read the way any element's + // properties are read. + ElementData session = data.get(SessionElement.ELEMENT_DATA_KEY); + for (String name : new String[]{SessionElement.PAGES_VIEWED, + SessionElement.SECONDS_SINCE_PAGE_LOAD, + SessionElement.POINTER_MOVED}) { + print(output, "session." + name, + (AspectPropertyValue) session.get(name)); + } + + // The derived properties, read from the Element Data the + // Derived Property Element writes under the key derived. + IDerivedPropertyData derived = data.get(IDerivedPropertyData.class); + for (String name : derivedProperties) { + print(output, "derived." + name, + (AspectPropertyValue) derived.get(name)); + } + } + output.println(); + } + + /** + * Print a value, or, where it has no value, the reason there is none. + */ + private static void print( + PrintStream output, + String name, + AspectPropertyValue value) { + output.println(String.format(" %-33s %s", name, value.hasValue() ? + value.getValue() : + "no value, because " + value.getNoValueMessage())); + } + + /** + * Print the full definition each script carries, read from the element, + * so that everything a script says about its property is available to + * a caller. + */ + private static void printDefinitions( + PrintStream output, + DerivedPropertyElement element) { + for (DerivedPropertyMetaData script : element.getScripts()) { + DerivedOutputMetaData definition = script.getOutput(); + output.println("The definition the script '" + script.getName() + + "' version " + script.getVersion() + " carries."); + output.println(String.format(" %-13s %s", "Name", + definition.getName())); + output.println(String.format(" %-13s %s", "Description", + definition.getDescription())); + output.println(String.format(" %-13s %s", "ValueType", + definition.getValueType())); + output.println(String.format(" %-13s %s", "DefaultValue", + definition.getDefaultValue())); + output.println(String.format(" %-13s %s", "Category", + definition.getCategory())); + output.println(String.format(" %-13s %s", "Dependencies", + definition.getDependencies())); + if (definition.getValues() != null) { + output.println(" Values"); + for (DerivedValueMetaData value : definition.getValues()) { + output.println(String.format(" %-8s %s", + value.getName(), value.getDescription())); + } + } + output.println(); + } + } + + /** + * End the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThe script format, the shared scripts and how to " + + "test a script: https://github.com/51Degrees/derived-properties"); + output.println("\tA page that runs a script in your browser: " + + "https://51degrees.github.io/derived-properties/"); + output.println("\tThe Pipeline API: https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.derivedproperty-main.java&utm_term=find-out-more"); + output.println("\tCustom flow elements, for work a script cannot do: https://51degrees.com/documentation/_pipeline_api__features__custom_elements.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.derivedproperty-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + + public static void main(String[] args) throws Exception { + new Example().run(System.out); + writeFindOutMore(System.out); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElement.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElement.java new file mode 100644 index 000000000..3820431fc --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElement.java @@ -0,0 +1,164 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.slf4j.ILoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Supplies the source properties the example's scripts read. A real + * Pipeline puts device detection or IP intelligence in this position, and + * this small element stands in for one of those so that the example runs + * with no resource key, no data file and no network connection. + *

+ * The element reads three items of evidence and publishes each one under + * the Element Data key {@code session}. Evidence that is not supplied + * leaves the property with no value and a message saying why, which is how + * the example shows a script producing no value and naming what was + * missing. + */ +public final class SessionElement + extends FlowElementBase { + + /** + * The Element Data key, being the part of session.PagesViewed before + * the dot. + */ + public static final String ELEMENT_DATA_KEY = "session"; + + /** + * The evidence holding how many pages the visitor has seen. + */ + public static final String PAGES_VIEWED_EVIDENCE = "query.pagesviewed"; + + /** + * The evidence holding how long the page has been open, in seconds. + */ + public static final String SECONDS_EVIDENCE = "query.secondssincepageload"; + + /** + * The evidence holding whether the pointer has moved. + */ + public static final String POINTER_MOVED_EVIDENCE = "query.pointermoved"; + + public static final String PAGES_VIEWED = "PagesViewed"; + public static final String SECONDS_SINCE_PAGE_LOAD = "SecondsSincePageLoad"; + public static final String POINTER_MOVED = "PointerMoved"; + + private final EvidenceKeyFilter evidenceKeyFilter = + new EvidenceKeyFilterWhitelist(Arrays.asList( + PAGES_VIEWED_EVIDENCE, SECONDS_EVIDENCE, POINTER_MOVED_EVIDENCE)); + + private final List properties; + + /** + * Construct the element. Use {@link SessionElementBuilder} to build one. + * @param loggerFactory the logger factory for the element and its data + */ + SessionElement(ILoggerFactory loggerFactory) { + super(loggerFactory.getLogger(SessionElement.class.getName()), + (flowData, element) -> new ElementDataBase( + loggerFactory.getLogger(ElementDataBase.class.getName()), + flowData) { + }); + // The Derived Property Element checks, when the Pipeline is built, + // that some element lists each source property a script names, so + // these names match the names the scripts use after the dot. + List listed = new ArrayList<>(); + listed.add(new ElementPropertyMetaDataDefault( + PAGES_VIEWED, this, "Example", Integer.class, true)); + listed.add(new ElementPropertyMetaDataDefault( + SECONDS_SINCE_PAGE_LOAD, this, "Example", Integer.class, true)); + listed.add(new ElementPropertyMetaDataDefault( + POINTER_MOVED, this, "Example", Boolean.class, true)); + this.properties = Collections.unmodifiableList(listed); + } + + @Override + public String getElementDataKey() { + return ELEMENT_DATA_KEY; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return evidenceKeyFilter; + } + + @Override + public List getProperties() { + return properties; + } + + @Override + protected void processInternal(FlowData data) { + ElementData session = + data.getOrAdd(getTypedDataKey(), getDataFactory()); + session.put(PAGES_VIEWED, + valueOf(data, PAGES_VIEWED_EVIDENCE, Integer.class)); + session.put(SECONDS_SINCE_PAGE_LOAD, + valueOf(data, SECONDS_EVIDENCE, Integer.class)); + session.put(POINTER_MOVED, + valueOf(data, POINTER_MOVED_EVIDENCE, Boolean.class)); + } + + /** + * The evidence as a property value, or a value with no value saying + * which evidence the request did not carry. + */ + private static AspectPropertyValueDefault valueOf( + FlowData data, + String evidence, + Class type) { + TryGetResult result = data.tryGetEvidence(evidence, type); + if (result.hasValue()) { + return new AspectPropertyValueDefault<>(result.getValue()); + } + AspectPropertyValueDefault missing = + new AspectPropertyValueDefault<>(); + missing.setNoValueMessage( + "the request did not carry the evidence '" + evidence + "'"); + return missing; + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElementBuilder.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElementBuilder.java new file mode 100644 index 000000000..6aebf9a6f --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElementBuilder.java @@ -0,0 +1,51 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; +import org.slf4j.ILoggerFactory; + +/** + * Builds a {@link SessionElement}. The annotation lets a configuration file + * name the builder as {@code SessionElement}. + */ +@ElementBuilder +public class SessionElementBuilder { + + private final ILoggerFactory loggerFactory; + + /** + * Construct a new builder. + * @param loggerFactory the logger factory for the element + */ + public SessionElementBuilder(ILoggerFactory loggerFactory) { + this.loggerFactory = loggerFactory; + } + + /** + * @return a new element + */ + public SessionElement build() { + return new SessionElement(loggerFactory); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml new file mode 100644 index 000000000..2468a6cd3 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml @@ -0,0 +1,73 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# A derived property script written for this example alone. +# +# The scripts 51Degrees ships inside the pipeline.derivedproperty package +# come from https://github.com/51Degrees/derived-properties and are named +# by BuiltInScript in code, or by the Scripts build parameter in a +# configuration file. This file is not one of them, so the example runs the +# same whatever that repository holds. +# +# The reference for everything below is docs/format-1.md in that +# repository. + +Format: 1 +Name: ReaderEngagement +Version: "1.0.0" +Output: + Name: ReaderEngagement + Description: How far the visitor appears to have engaged with the page, judged only from properties another element has already produced. Returns one of High, Medium or Low. + ValueType: string + DefaultValue: Low + IsList: false + IsMandatory: true + IsObsolete: false + Category: Example + IsPopular: false + ExportValues: true + Values: + - Name: High + Description: Every test went in favour of engagement. + - Name: Medium + Description: One test went against engagement. + - Name: Low + Description: More than one test went against engagement. + +# A property named here is a property the script needs. Where any one of +# the three is missing from a request, the script produces no value and +# says which property was missing and what the element that supplies it +# said, rather than answering on part of the evidence. +Checks: + SeveralPages: { Property: session.PagesViewed, Ge: 3 } + LongEnough: { Property: session.SecondsSincePageLoad, Ge: 30 } + PointerMoved: { Property: session.PointerMoved, Eq: true } + +# Rules are read in order and the first one whose When is true wins. The +# last rule is an Else, which every script ends in, so a script always +# chooses a value once its source properties have been read. +Rules: + - When: { Failed: Checks, Eq: 0 } + Then: High + - When: { Failed: Checks, Eq: 1 } + Then: Medium + - Else: Low diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml new file mode 100644 index 000000000..4f1a11883 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml @@ -0,0 +1,57 @@ + + + + + + + + SessionElement + + + DerivedPropertyElement + + + + + ${filepath:ReaderEngagement.yaml} + + + + From c9693a959ba215668d9dbb7a3bc3ceb00c55c07b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 03:47:41 +0000 Subject: [PATCH 3/3] Build the configuration tests' file patterns as text, which Windows accepts The tests made each pattern with Path.resolve("*.yaml"), and Windows refuses a path that holds a wildcard with InvalidPathException, so three DerivedPropertyElementConfigurationTests failed in every Windows job of full build 43667464820. The element itself splits a pattern with File.getName and File.getParentFile and was not affected. --- .../DerivedPropertyElementConfigurationTests.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java index 83d43f2a3..6e5e73e59 100644 --- a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java @@ -33,6 +33,7 @@ import org.junit.jupiter.api.io.TempDir; import org.slf4j.LoggerFactory; +import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -55,6 +56,14 @@ public class DerivedPropertyElementConfigurationTests { @TempDir Path folder; + /** + * A file name pattern in the folder, built as text, because Windows + * refuses a Path that holds a wildcard. + */ + private String pattern(String fileNamePattern) { + return folder.toString() + File.separator + fileNamePattern; + } + /** * Writes the Band script into the folder and gives a pattern matching * it. @@ -63,7 +72,7 @@ private String bandFiles() throws Exception { Files.write(folder.resolve("Band.yaml"), DerivedPropertyElementTests.BAND_YAML.getBytes( StandardCharsets.UTF_8)); - return folder.resolve("*.yaml").toString(); + return pattern("*.yaml"); } /** @@ -132,7 +141,7 @@ public void anUnknownBuildParameterIsRefused() throws Exception { @Test public void aPatternMatchingNoFileIsRefused() throws Exception { PipelineOptions options = options("" + - folder.resolve("*.json") + ""); + pattern("*.json") + ""); PipelineConfigurationException e = assertThrows( PipelineConfigurationException.class, () -> new PipelineBuilder().buildFromConfiguration(options));