From 8c9fe532762fb50d547f0ba402abbef953af5979 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 23:52:28 +0000 Subject: [PATCH 1/2] Add the derived-properties submodule and the tests of a derived property element The tests run every case file and every rejection case in the submodule's tests folder through a pipeline, and test the element's settings, logging, metadata and the checks it makes when a pipeline is built. The package is listed in ci/packages.json as not published, and the pipeline build and unit test steps include it. The element itself follows in the next commit. --- .gitmodules | 3 + ci/packages.json | 8 + pipeline/ci/build-project.ps1 | 2 +- pipeline/ci/run-unit-tests.ps1 | 2 +- .../.pylintrc | 628 ++++++++++++++++++ .../MANIFEST.in | 3 + .../pyproject.toml | 3 + .../setup.py | 76 +++ .../derived-properties | 1 + .../tests/__init__.py | 0 .../tests/country_element.py | 33 + .../tests/helpers.py | 138 ++++ .../tests/test_built_in_scripts.py | 90 +++ .../tests/test_element.py | 333 ++++++++++ .../tests/test_pipeline_check.py | 166 +++++ .../tests/test_scripts.py | 222 +++++++ .../tests/test_shared_cases.py | 146 ++++ .../fiftyone_pipeline_derivedproperty/tox.ini | 21 + 18 files changed, 1873 insertions(+), 2 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/.pylintrc create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/MANIFEST.in create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/pyproject.toml create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/setup.py create mode 160000 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/__init__.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/country_element.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/helpers.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/test_built_in_scripts.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/test_element.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/test_pipeline_check.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/test_scripts.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/test_shared_cases.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tox.ini diff --git a/.gitmodules b/.gitmodules index af600379..c031260b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "cxx"] path = cxx url = ../51degrees-cxx +[submodule "pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties"] + path = pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties + url = ../derived-properties diff --git a/ci/packages.json b/ci/packages.json index d9dcd87d..b95ea458 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -39,6 +39,14 @@ "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, + { + "Name": "fiftyone_pipeline_derivedproperty", + "Product": "pipeline", + "Path": "pipeline/fiftyone_pipeline_derivedproperty", + "Ignore": ["**/tox.ini"], + "DependsOn": ["fiftyone_pipeline_core"], + "Publish": false + }, { "Name": "fiftyone_pipeline_did", "Product": "pipeline", diff --git a/pipeline/ci/build-project.ps1 b/pipeline/ci/build-project.ps1 index a7ae240c..bbf0d7b9 100644 --- a/pipeline/ci/build-project.ps1 +++ b/pipeline/ci/build-project.ps1 @@ -8,7 +8,7 @@ param ( # anything builds. See ci/copy-owid-source.ps1 for the full reasoning. & "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName -$packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_translation" +$packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_translation", "fiftyone_pipeline_derivedproperty" ./python/build-project.ps1 -RepoName $RepoName -Packages $packages exit $LASTEXITCODE diff --git a/pipeline/ci/run-unit-tests.ps1 b/pipeline/ci/run-unit-tests.ps1 index 441a2bcc..c379c287 100644 --- a/pipeline/ci/run-unit-tests.ps1 +++ b/pipeline/ci/run-unit-tests.ps1 @@ -9,7 +9,7 @@ param ( $addopts = $env:PYTEST_ADDOPTS $env:PYTEST_ADDOPTS = '-m "not live"' try { - $packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_translation", "fiftyone_pipeline_did" + $packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_translation", "fiftyone_pipeline_did", "fiftyone_pipeline_derivedproperty" ./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages } finally { $env:PYTEST_ADDOPTS = $addopts diff --git a/pipeline/fiftyone_pipeline_derivedproperty/.pylintrc b/pipeline/fiftyone_pipeline_derivedproperty/.pylintrc new file mode 100644 index 00000000..5bd97998 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/.pylintrc @@ -0,0 +1,628 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=all + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=F, + E, + syntax-error, + unrecognized-inline-option, + bad-option-value, + init-is-generator, + return-in-init, + function-redefined, + not-in-loop, + return-outside-function, + yield-outside-function, + return-arg-in-generator, + nonexistent-operator, + duplicate-argument-name, + abstract-class-instantiated, + bad-reversed-sequence, + too-many-star-expressions, + invalid-star-assignment-target, + star-needs-assignment-target, + nonlocal-and-global, + continue-in-finally, + nonlocal-without-binding, + used-prior-global-declaration, + misplaced-format-function, + method-hidden, + access-member-before-definition, + no-method-argument, + no-self-argument, + invalid-slots-object, + assigning-non-slot, + invalid-slots, + inherit-non-class, + inconsistent-mro, + duplicate-bases, + class-variable-slots-conflict, + non-iterator-returned, + unexpected-special-method-signature, + invalid-length-returned, + invalid-bool-returned, + invalid-index-returned, + invalid-repr-returned, + invalid-str-returned, + invalid-bytes-returned, + invalid-hash-returned, + invalid-length-hint-returned, + invalid-format-returned, + invalid-getnewargs-returned, + invalid-getnewargs-ex-returned, + import-error, + relative-beyond-top-level, + used-before-assignment, + undefined-variable, + undefined-all-variable, + invalid-all-object, + no-name-in-module, + unpacking-non-sequence, + bad-except-order, + raising-bad-type, + bad-exception-context, + misplaced-bare-raise, + raising-non-exception, + notimplemented-raised, + catching-non-exception, + bad-super-call, + no-member, + not-callable, + assignment-from-no-return, + no-value-for-parameter, + unexpected-keyword-arg, + redundant-keyword-arg, + missing-kwoa, + invalid-sequence-index, + invalid-slice-index, + assignment-from-none, + not-context-manager, + invalid-unary-operand-type, + unsupported-binary-operation, + repeated-keyword, + not-an-iterable, + not-a-mapping, + unsupported-membership-test, + unsubscriptable-object, + unsupported-assignment-operation, + unsupported-delete-operation, + invalid-metaclass, + unhashable-dict-key, + dict-iter-missing-items, + logging-unsupported-format, + logging-format-truncated, + logging-too-many-args, + logging-too-few-args, + bad-format-character, + truncated-format-string, + mixed-format-string, + format-needs-mapping, + missing-format-string-key, + too-many-format-args, + too-few-format-args, + bad-string-format-type, + bad-str-strip-call, + invalid-envvar-value, + yield-inside-async-function, + not-async-context-manager, + fatal, + astroid-error, + parse-error, + method-check-failed, + unreachable, + duplicate-key, + unnecessary-semicolon, + global-variable-not-assigned, + unused-variable, + binary-op-exception, + bad-format-string, + anomalous-backslash-in-string, + bad-open-mode + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=builtins.BaseException, + builtins.Exception diff --git a/pipeline/fiftyone_pipeline_derivedproperty/MANIFEST.in b/pipeline/fiftyone_pipeline_derivedproperty/MANIFEST.in new file mode 100644 index 00000000..a2a170c3 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/MANIFEST.in @@ -0,0 +1,3 @@ +include src/fiftyone_pipeline_derivedproperty/derived-properties/scripts/*.yaml +include version.txt +include readme.md diff --git a/pipeline/fiftyone_pipeline_derivedproperty/pyproject.toml b/pipeline/fiftyone_pipeline_derivedproperty/pyproject.toml new file mode 100644 index 00000000..8fe2f47a --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/pipeline/fiftyone_pipeline_derivedproperty/setup.py b/pipeline/fiftyone_pipeline_derivedproperty/setup.py new file mode 100644 index 00000000..a2c92cac --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/setup.py @@ -0,0 +1,76 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import setuptools +import os +import io + + +# Read a text file and return the content as a string. +def read(file_name): + + """Read a text file and return the content as a string.""" + try: + with io.open( + os.path.join(os.path.dirname(__file__), file_name), encoding="utf-8" + ) as f: + return f.read().strip() + except: + return "0.0.0" + + +setuptools.setup( + name="fiftyone_pipeline_derivedproperty", + version=read("version.txt"), + author="51Degrees Engineering", + author_email="engineering@51degrees.com", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_derivedproperty&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_derivedproperty&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, + description=("This package adds the derived property flow element to the fiftyone-pipeline, which computes properties such as HumanConfidence and IsGdpr from properties other elements have produced, following the scripts in the derived-properties repository."), + long_description=read("readme.md"), + long_description_content_type='text/markdown', + python_requires=">=3.10", + install_requires=[ + 'fiftyone_pipeline_core', + 'PyYAML' + ], + packages=["fiftyone_pipeline_derivedproperty"], + package_dir={"": "src"}, + license="EUPL-1.2", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules" + ], + include_package_data=True +) diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties new file mode 160000 index 00000000..cc306386 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties @@ -0,0 +1 @@ +Subproject commit cc30638632050795d9acc80d5118a290ffaba085 diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/__init__.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/country_element.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/country_element.py new file mode 100644 index 00000000..4f804093 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/country_element.py @@ -0,0 +1,33 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +from .helpers import SourceElement + + +class CountryElement(SourceElement): + """! + An element giving the country code FR under the key ip, which a + pipeline configuration can name, as it takes no settings. + """ + + def __init__(self): + super().__init__("ip", {"CountryCode": "FR"}) diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/helpers.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/helpers.py new file mode 100644 index 00000000..c83d5920 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/helpers.py @@ -0,0 +1,138 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os + +from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.logger import Logger +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +## The derived-properties submodule in this checkout. +SUBMODULE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "src", + "fiftyone_pipeline_derivedproperty", "derived-properties") + + +def as_element_gives(value, wrap): + """! + A value as a case file writes it, as a 51Degrees element gives it. A + value with a no value message is an AspectPropertyValue without a + value, and a weighted list has the value and weighting fields the + IP Intelligence engines give. + """ + if isinstance(value, dict): + if "NoValue" in value: + return AspectPropertyValue(value["NoValue"]) + if "String" in value: + return (AspectPropertyValue(None, value["String"]) if wrap + else value["String"]) + if isinstance(value, list) and wrap: + return AspectPropertyValue(None, [ + {"value": m["Value"], "weighting": m.get("Weight")} + if isinstance(m, dict) and "Value" in m else m + for m in value]) + return AspectPropertyValue(None, value) if wrap else value + + +class SourceElement(FlowElement): + """! + An element giving fixed values under one element data key, as plain + element data. It lists the properties it supplies, which need not be + the ones it has a value for. + """ + + def __init__(self, datakey, values, supplies=None, wrap=True): + super().__init__() + self.datakey = datakey + self.values = values + self.wrap = wrap + names = list(values.keys()) if supplies is None else supplies + self.properties = { + name.lower(): {"name": name, "type": "string"} for name in names} + + def process_internal(self, flowdata): + contents = {name: as_element_gives(value, self.wrap) + for name, value in self.values.items()} + flowdata.set_element_data(ElementDataDictionary(self, contents)) + + +class RecordingLogger(Logger): + """! A logger keeping every line, debug included. """ + + def __init__(self): + super().__init__(min_level="debug") + self.lines = [] + + def log_internal(self, level, log): + self.lines.append((level, log["message"])) + + def messages(self, level): + return [message for logged, message in self.lines if logged == level] + + +def band_script(name, prop, version="1.0.0", extra=""): + """! The text of a small script reading one source property. """ + lines = [ + "Format: 1", + "Name: " + name, + "Version: " + version, + extra, + "Output:", + " Name: " + name, + " Description: Whether a number is ten or more.", + " ValueType: string", + " IsList: false", + " Category: Testing", + "Rules:", + " - When: { Property: " + prop + ", Ge: 10 }", + " Then: High", + " - Else: Low" + ] + return "\n".join(line for line in lines if line != "") + + +def pipeline_of(elements, logger=None, **settings): + """! Builds a pipeline holding the elements given, with nothing added. """ + builder = PipelineBuilder(dict({ + "add_javascript_builder": False, + "use_setheader_properties": False + }, **settings)) + if logger is not None: + builder.add_logger(logger) + for element in elements: + builder.add(element) + return builder.build() + + +def processed(pipeline): + """! Processes a request through a pipeline. """ + flowdata = pipeline.create_flowdata() + flowdata.process() + return flowdata + + +def submodule_text(*parts): + """! Reads a file in the submodule. """ + with open(os.path.join(SUBMODULE, *parts), encoding="utf-8") as file: + return file.read() diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_built_in_scripts.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_built_in_scripts.py new file mode 100644 index 00000000..c5ce7ded --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_built_in_scripts.py @@ -0,0 +1,90 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +The scripts built into the package, which are the scripts folder of the +derived-properties submodule. The tests run against the package tox +installs from a wheel, so they also prove the wheel carries the scripts. +""" + +import os +import re + +import pytest + +import fiftyone_pipeline_derivedproperty as package +from fiftyone_pipeline_derivedproperty import ( + DerivedPropertyElementBuilder, ScriptValidationError, + builtin_script_names) +from fiftyone_pipeline_derivedproperty.script_set import BUILT_IN_FOLDER + +from .helpers import SUBMODULE + + +def test_the_built_in_scripts_are_the_scripts_of_the_submodule(): + folder = os.path.join(SUBMODULE, "scripts") + names = sorted(os.path.splitext(f)[0] for f in os.listdir(folder) + if re.search(r"\.(ya?ml|json)$", f, re.IGNORECASE)) + assert names + assert builtin_script_names() == names + for name in names: + with open(os.path.join(folder, name + ".yaml"), + encoding="utf-8") as source, \ + open(os.path.join(BUILT_IN_FOLDER, name + ".yaml"), + encoding="utf-8") as carried: + assert carried.read() == source.read() + + +def test_the_package_reads_the_scripts_it_carries(): + assert BUILT_IN_FOLDER.startswith( + os.path.dirname(os.path.abspath(package.__file__))) + + +def test_every_built_in_script_has_the_shared_cases(): + for name in builtin_script_names(): + assert os.path.exists( + os.path.join(SUBMODULE, "tests", name + ".cases.yaml")) + + +def test_one_element_runs_every_built_in_script(): + builder = DerivedPropertyElementBuilder() + for name in builtin_script_names(): + builder.add_built_in_script(name) + element = builder.build() + assert [s.name for s in element.scripts] == builtin_script_names() + assert [s.source for s in element.scripts] == [ + "built in " + name for name in builtin_script_names()] + + +def test_built_in_names_ignore_case(): + element = (DerivedPropertyElementBuilder() + .add_built_in_script("humanCONFIDENCE").build()) + assert element.scripts[0].name == "HumanConfidence" + + +def test_an_unknown_built_in_script_names_the_ones_there_are(): + with pytest.raises(ScriptValidationError, match=re.escape( + "there is no script called 'Nope' in this package. The scripts " + "in this package are " + ", ".join(builtin_script_names()) + + ". A script from your own environment is added with " + "script_files rather than scripts")): + DerivedPropertyElementBuilder().add_built_in_script("Nope").build() diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_element.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_element.py new file mode 100644 index 00000000..516ef3bb --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_element.py @@ -0,0 +1,333 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +How the element reads the values 51Degrees elements give, writes its +own, logs, takes its settings and describes its properties. +""" + +import json +import os +import re + +import pytest + +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from fiftyone_pipeline_derivedproperty import ( + DerivedPropertyElement, DerivedPropertyElementBuilder, + ScriptValidationError) +from fiftyone_pipeline_derivedproperty.evaluate import USUAL_CAUSES + +from .helpers import ( + RecordingLogger, SourceElement, band_script, pipeline_of, processed) + +BAND_IS_HIGH = "\n".join([ + "Format: 1", + "Name: BandIsHigh", + "Version: 1.0.0", + "Output:", + " Name: BandIsHigh", + " Description: Whether the band is high.", + " ValueType: bool", + " IsList: false", + "Rules:", + " - When: { Property: derived.Band, Eq: \"High\" }", + " Then: true", + " - Else: false" +]) + +NOT_PRESENT = "property not present on this request" + + +def band(): + return (DerivedPropertyElementBuilder() + .add_script("Band", band_script("Band", "src.N")).build()) + + +def band_for(value): + """! The band written where src.N has a value, given as it is. """ + flowdata = processed(pipeline_of([ + SourceElement("src", {"N": value}, wrap=False), band()])) + return flowdata.get("derived").get("Band") + + +def no_band(reason): + return ("Derived property 'Band' has no value because 1 source property " + "was not available. 'src.N' (" + reason + "). " + USUAL_CAUSES) + + +def test_a_value_is_read_under_the_derived_key_whatever_the_case(): + flowdata = processed(pipeline_of([ + SourceElement("src", {"N": 12}), band()])) + assert flowdata.get("derived").get("Band").value() == "High" + assert flowdata.get("derived").get("band").value() == "High" + assert flowdata.derived.BAND.value() == "High" + + +def test_a_property_that_cannot_be_read_raises_the_message_when_read(): + flowdata = processed(pipeline_of([ + SourceElement("src", {}, ["N"]), band()])) + value = flowdata.get("derived").get("Band") + message = no_band("element 'src' has no value for 'N': " + NOT_PRESENT) + assert not value.has_value() + assert value.no_value_message() == message + with pytest.raises(Exception, match=re.escape(message)): + value.value() + + +def test_the_reason_an_engine_gives_for_a_property_it_lacks_is_carried(): + class ReasonData(ElementDataDictionary): + def get(self, key): + raise Exception( + "The property " + key + " is not in the resource key.") + + class Engine(SourceElement): + def process_internal(self, flowdata): + flowdata.set_element_data(ReasonData(self, {})) + + flowdata = processed(pipeline_of([Engine("src", {}, ["N"]), band()])) + assert flowdata.get("derived").get("Band").no_value_message() == \ + no_band("element 'src' has no value for 'N': The property N is " + "not in the resource key.") + + +def test_a_source_value_with_its_own_message_carries_it(): + assert band_for({"NoValue": "The JavaScript has not run."}) \ + .no_value_message() == no_band( + "element 'src' has no value for 'N': The JavaScript has not " + "run.") + + +def test_a_source_value_with_no_message_reads_as_not_present(): + assert band_for({"NoValue": ""}).no_value_message() == no_band( + "element 'src' has no value for 'N': " + NOT_PRESENT) + + +def test_a_weighted_list_gives_the_heaviest_value_the_first_of_two_equal(): + # As on-premise IP Intelligence gives a weighted list. + assert band_for([{"value": 5, "weighting": 0.2}, + {"value": 15, "weighting": 0.8}]).value() == "High" + # As the cloud gives one before it is normalised. + assert band_for([{"value": 15, "rawweighting": 26214}, + {"value": 5, "rawweighting": 39321}]).value() == "Low" + assert band_for([{"value": 15, "weighting": 0.5}, + {"value": 5, "weighting": 0.5}]).value() == "High" + + +def test_a_list_that_is_not_weighted_cannot_be_read(): + assert band_for([5, 15]).no_value_message() == no_band( + "held a list where a single value is needed") + + +def test_a_value_that_is_not_a_number_cannot_be_read_as_one(): + assert band_for("Unknown").no_value_message() == no_band( + "held 'Unknown' which cannot be read as int") + + +def test_no_element_data_for_a_source_is_logged_once(): + # The element supplying src.N runs after the derived property element. + logger = RecordingLogger() + pipeline = pipeline_of([band(), SourceElement("src", {"N": 12})], logger) + first = processed(pipeline) + processed(pipeline) + assert not first.get("derived").get("Band").has_value() + assert logger.messages("error") == [ + "The script 'Band' reads 'src.N', and the request carried no " + "element data 'src' at all, so the script has no value. The usual " + "cause is the element that supplies it running after this element " + "rather than before it. This is reported once."] + + +def test_building_the_pipeline_logs_what_each_script_produces(): + logger = RecordingLogger() + pipeline_of([SourceElement("src", {"N": 1}), band()], logger) + assert ("Derived property script 'Band' version 1.0.0, format 1, from " + "code, produces 'derived.Band' as string." + in logger.messages("info")) + compiled = [m for m in logger.messages("debug") if m.startswith( + "Derived property script 'Band' compiled to ")][0] + model = json.loads(compiled[compiled.index("{"):]) + assert model["Properties"] == {"src.N": {"Type": "int"}} + assert model["Output"]["Dependencies"] == ["src.N"] + + +def test_a_deprecated_script_logs_a_warning_with_its_note(): + logger = RecordingLogger() + element = (DerivedPropertyElementBuilder().add_script( + "Band", band_script("Band", "src.N", extra=( + "Deprecated: true\nDeprecationNote: Use BandTwo instead."))) + .build()) + pipeline_of([SourceElement("src", {"N": 1}), element], logger) + assert logger.messages("warning") == [ + "Derived property script 'Band' is deprecated. Use BandTwo " + "instead."] + + +def test_a_later_script_reads_what_an_earlier_one_wrote(): + element = (DerivedPropertyElementBuilder() + .add_script("Band", band_script("Band", "src.N")) + .add_script("BandIsHigh", BAND_IS_HIGH) + .build()) + flowdata = processed(pipeline_of([ + SourceElement("src", {"N": 12}), element])) + # A bool output is a boolean. + assert flowdata.get("derived").get("BandIsHigh").value() is True + + +@pytest.fixture +def folder(tmp_path): + (tmp_path / "Band.yaml").write_text(band_script("Band", "src.N")) + (tmp_path / "Other.yaml").write_text(band_script("Other", "src.N")) + (tmp_path / "notes.txt").write_text("not a script") + (tmp_path / "with,comma").mkdir() + (tmp_path / "with,comma" / "Third.yaml").write_text( + band_script("Third", "src.N")) + return str(tmp_path) + + +def test_settings_name_built_in_scripts_and_files_in_any_case(folder): + element = DerivedPropertyElement(**{ + "Scripts": "isgdpr, HumanConfidence", + "ScriptFiles": os.path.join(folder, "*.yaml") + ", \"" + + os.path.join(folder, "with,comma", "Third.yaml") + "\"" + }) + assert [s.name for s in element.scripts] == [ + "IsGdpr", "HumanConfidence", "Band", "Other", "Third"] + assert element.scripts[0].source == "built in IsGdpr" + assert element.scripts[2].source == os.path.join(folder, "Band.yaml") + + +def test_settings_may_be_lists_of_strings(folder): + element = DerivedPropertyElement( + scripts=["IsGdpr"], script_files=[os.path.join(folder, "Band.yaml")]) + assert [s.name for s in element.scripts] == ["IsGdpr", "Band"] + + +def test_a_setting_the_element_does_not_have_is_refused(): + with pytest.raises(ValueError, match=re.escape( + "The derived property element has no setting 'script'. Its " + "settings are scripts and script_files.")): + DerivedPropertyElement(script="IsGdpr") + + +def test_a_file_or_folder_that_is_not_there_is_a_fault(folder): + with pytest.raises(ScriptValidationError): + DerivedPropertyElement( + script_files=os.path.join(folder, "Missing.yaml")) + with pytest.raises(ScriptValidationError, match="does not exist"): + DerivedPropertyElement( + script_files=os.path.join(folder, "nowhere", "*.yaml")) + + +def test_a_pipeline_configuration_builds_the_element(): + pipeline = PipelineBuilder().build_from_configuration({ + "PipelineOptions": { + "Elements": [ + {"elementPath": "tests.country_element", + "elementName": "CountryElement"}, + {"elementPath": "fiftyone_pipeline_derivedproperty", + "elementName": "DerivedPropertyElement", + "elementParameters": {"Scripts": "IsGdpr"}} + ] + } + }) + flowdata = processed(pipeline) + assert flowdata.get("derived").get("IsGdpr").value() is True + + +def test_faults_in_every_script_are_reported_together(): + with pytest.raises(ScriptValidationError) as raised: + (DerivedPropertyElementBuilder() + .add_script("One", band_script("One", "src.N") + .replace("Ge: 10", "Gx: 10")) + .add_script("Two", band_script("Two", "N")) + .add_built_in_script("Nothing") + .build()) + scripts = [] + for fault in raised.value.faults: + if fault.script not in scripts: + scripts.append(fault.script) + assert scripts == ["One", "Two", "Nothing"] + message = str(raised.value) + assert "unknown operator 'Gx'" in message + assert "'N' is not a source property" in message + assert "there is no script called 'Nothing'" in message + + +def test_an_element_with_no_script_is_refused(): + with pytest.raises(ValueError, + match="At least one script must be configured."): + DerivedPropertyElement() + + +def test_two_scripts_in_one_element_cannot_write_one_property(): + builder = (DerivedPropertyElementBuilder() + .add_script("Band", band_script("Band", "src.N")) + .add_script("BandTwo", band_script("BandTwo", "src.N") + .replace(" Name: BandTwo", " Name: Band"))) + with pytest.raises(ScriptValidationError, match=re.escape( + "two scripts in the same element write the property 'Band', " + "being 'Band' and 'BandTwo'. One element writes each property " + "once")): + builder.build() + + +def test_the_property_metadata_is_the_output_block(): + element = band() + assert element.get_properties() == { + "band": { + "name": "Band", + "type": "string", + "category": "Testing", + "description": "Whether a number is ten or more.", + "islist": False, + "dependencies": ["src.N"] + } + } + flowdata = processed(pipeline_of([ + SourceElement("src", {"N": 12}), element])) + assert list(flowdata.get_where("category", "Testing").keys()) == \ + ["band"] + + +def test_the_json_bundler_carries_the_values_of_every_derived_element(): + pipeline = pipeline_of([ + SourceElement("src", {"N": 12}), + band(), + DerivedPropertyElementBuilder() + .add_script("BandIsHigh", BAND_IS_HIGH).build() + ], add_javascript_builder=True, add_javascript_builder_element=False) + flowdata = processed(pipeline) + output = flowdata.get("jsonbundler").get("json") + # Both derived property elements write one element data, and each + # gives the properties of both. + assert output["derived"]["band"] == "High" + assert output["derived"]["bandishigh"] is True + assert sorted(pipeline.get_element("derived").get_properties()) == \ + ["band", "bandishigh"] + + +def test_the_element_accepts_no_evidence(): + assert band().get_evidence_key_filter().filter_evidence({ + "header.user-agent": "x", "query.n": "12"}) == {} diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_pipeline_check.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_pipeline_check.py new file mode 100644 index 00000000..e183d71c --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_pipeline_check.py @@ -0,0 +1,166 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +The checks run when a pipeline adds the element, following +docs/authoring.md of derived-properties, "What the pipeline checks when +the element is added", and the tests of the same check in the .NET, Rust +and PHP elements. +""" + +from fiftyone_pipeline_core.pipeline import Pipeline + +from fiftyone_pipeline_derivedproperty import ( + DerivedPropertyElementBuilder, PipelineConfigurationError) + +from .helpers import SourceElement, band_script + + +def human_confidence(): + return (DerivedPropertyElementBuilder() + .add_built_in_script("HumanConfidence").build()) + + +def read_by_human_confidence(key): + names = [] + for prop in human_confidence().scripts[0].properties: + element_key, name = prop.name.split(".") + if element_key == key: + names.append(name) + return names + + +def supplier(key, names): + return SourceElement(key, {}, names) + + +def derived(scripts): + builder = DerivedPropertyElementBuilder() + for name, source in scripts.items(): + builder.add_script(name, band_script(name, source)) + return builder.build() + + +def needs(script, prop): + return ("The script '" + script + "' needs the property '" + prop + + "', and no element in the pipeline supplies it. Either add the " + "element that supplies '" + prop + "', or change the script so " + "that it does not name '" + prop + "'.") + + +def writes(script, output): + return ("The script '" + script + "' writes the derived property '" + + output + "', and another element in the pipeline already " + "writes a property of that name. One pipeline writes each " + "derived property once.") + + +def faults(elements): + """! The faults building a pipeline of the elements gives, or None. """ + try: + Pipeline(elements) + except PipelineConfigurationError as error: + assert str(error) == ("The derived property element cannot be used " + "in this pipeline.\n" + "\n".join(error.faults)) + return error.faults + return None + + +def test_a_pipeline_supplying_nothing_names_every_property(): + properties = human_confidence().scripts[0].properties + assert properties + assert faults([human_confidence()]) == [ + needs("HumanConfidence", p.name) for p in properties] + + +def test_suppliers_may_sit_anywhere_in_the_pipeline(): + assert faults([ + supplier("device", read_by_human_confidence("device")), + human_confidence(), + supplier("ip", read_by_human_confidence("ip"))]) is None + assert faults([ + human_confidence(), + supplier("device", read_by_human_confidence("device")), + supplier("ip", read_by_human_confidence("ip"))]) is None + + +def test_keys_and_names_are_compared_ignoring_case(): + assert faults([ + supplier("DEVICE", + [n.lower() for n in read_by_human_confidence("device")]), + supplier("Ip", [n.upper() for n in read_by_human_confidence("ip")]), + human_confidence()]) is None + + +def test_a_property_listed_under_another_key_is_not_supplied(): + assert faults([ + supplier("device", read_by_human_confidence("device")), + supplier("location", read_by_human_confidence("ip")), + human_confidence()]) == [ + needs("HumanConfidence", "ip." + n) + for n in read_by_human_confidence("ip")] + + +def test_an_element_listing_no_properties_may_supply_any_of_its_own(): + # A cloud engine learns its properties from the cloud service. + assert faults([ + supplier("device", read_by_human_confidence("device")), + supplier("ip", []), + human_confidence()]) is None + assert faults([supplier("ip", []), human_confidence()]) == [ + needs("HumanConfidence", "device." + n) + for n in read_by_human_confidence("device")] + + +def test_two_elements_writing_one_derived_property_are_refused(): + assert faults([ + supplier("src", ["N"]), + derived({"Demo": "src.N"}), + derived({"Demo": "src.N"})]) == [writes("Demo", "Demo")] + + +def test_a_script_may_read_what_an_earlier_script_of_its_element_writes(): + assert faults([ + supplier("src", ["N"]), + derived({"Band": "src.N", "BandIsHigh": "derived.Band"})]) is None + # A later script's property has no value when an earlier one reads it. + assert faults([ + supplier("src", ["N"]), + derived({"BandIsHigh": "derived.Band", "Band": "src.N"})]) == [ + needs("BandIsHigh", "derived.Band")] + # A script's own property is not there when it runs either. + assert faults([derived({"Demo": "derived.Demo"})]) == [ + needs("Demo", "derived.Demo")] + + +def test_another_derived_element_supplies_what_it_writes_wherever_it_sits(): + assert faults([ + supplier("src", ["N"]), + derived({"BandIsHigh": "derived.Band"}), + derived({"Band": "src.N"})]) is None + + +def test_properties_written_twice_come_before_those_not_supplied(): + assert faults([ + derived({"Demo": "gone.N"}), + derived({"Demo": "src.N"})]) == [ + writes("Demo", "Demo"), needs("Demo", "gone.N")] diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_scripts.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_scripts.py new file mode 100644 index 00000000..077aaee9 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_scripts.py @@ -0,0 +1,222 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +Reading and running scripts, as docs/format-1.md of derived-properties +describes, for the points the case files do not reach, and for the places +where Python's own rules differ from JavaScript's. +""" + +import json + +from fiftyone_pipeline_derivedproperty.canonical import canonical +from fiftyone_pipeline_derivedproperty.evaluate import USUAL_CAUSES, evaluate +from fiftyone_pipeline_derivedproperty.validate import validate_text + + +def script_with(condition, value_type="string"): + then, otherwise = (("true", "false") if value_type == "bool" + else ('"Yes"', '"No"')) + return "\n".join([ + "Format: 1", + "Name: Demo", + "Version: 1.0.0", + "Output:", + " Name: Demo", + " Description: A demonstration.", + " ValueType: " + value_type, + " IsList: false", + "Rules:", + " - When: " + condition, + " Then: " + then, + " - Else: " + otherwise]) + + +def model_of(text): + model, faults = validate_text(text) + assert faults == [] + return model + + +def test_yes_and_no_are_strings_as_the_yaml_core_schema_reads_them(): + model = model_of(script_with("{ Property: src.A, Eq: yes }")) + assert model.properties[0].type == "string" + assert evaluate(model, {"src.A": "yes"}).value == "Yes" + + +def test_a_date_stays_the_text_written(): + model = model_of(script_with("{ Property: src.A, Eq: 2026-11-16 }")) + assert model.properties[0].type == "string" + assert evaluate(model, {"src.A": "2026-11-16"}).value == "Yes" + + +def test_keys_that_differ_only_in_case_are_refused(): + model, faults = validate_text( + script_with("{ Property: src.A, Eq: 1 }") + "\nname: Demo") + assert model is None + assert "differ only in case" in faults[0].message + + +def test_a_key_written_twice_is_refused(): + model, faults = validate_text( + script_with("{ Property: src.A, Eq: 1 }") + "\nName: Demo") + assert model is None + assert "duplicated mapping key" in faults[0].message + + +def test_keys_are_matched_without_regard_to_case(): + model = model_of(script_with("{ property: src.A, eq: 1 }") + .replace("ValueType", "valuetype") + .replace("Rules:", "RULES:")) + assert evaluate(model, {"SRC.a": 1}).value == "Yes" + + +def test_a_json_script_gives_the_same_model_as_its_yaml(): + yaml_model = model_of(script_with("{ Property: src.A, Ge: 8 }")) + json_model = model_of(json.dumps({ + "Format": 1, + "Name": "Demo", + "Version": "1.0.0", + "Output": { + "Name": "Demo", + "Description": "A demonstration.", + "ValueType": "string", + "IsList": False + }, + "Rules": [ + {"When": {"Property": "src.A", "Ge": 8.0}, "Then": "Yes"}, + {"Else": "No"} + ] + })) + assert canonical(json_model) == canonical(yaml_model) + + +def test_a_whole_number_written_with_a_point_still_infers_int(): + model = model_of(script_with("{ Property: src.A, Lt: 2.0 }")) + assert model.properties[0].type == "int" + assert evaluate(model, {"src.A": 1.5}).missing == ["src.A"] + + +def test_a_whole_number_beyond_an_int_infers_double(): + model = model_of(script_with("{ Property: src.A, Lt: 3000000000 }")) + assert model.properties[0].type == "double" + + +def test_an_int_is_a_signed_32_bit_whole_number(): + model = model_of(script_with("{ Property: src.A, Ge: 0 }")) + assert evaluate(model, {"src.A": 2147483647}).value == "Yes" + assert evaluate(model, {"src.A": -2147483648}).value == "No" + assert evaluate(model, {"src.A": 2147483648}).missing == ["src.A"] + assert evaluate(model, {"src.A": {"String": "3000000000"}}).missing == \ + ["src.A"] + + +def test_a_boolean_is_not_a_number(): + model = model_of(script_with("{ Property: src.A, Eq: 1 }")) + assert evaluate(model, {"src.A": True}).missing == ["src.A"] + bool_model = model_of(script_with("{ Property: src.A, Eq: true }")) + assert evaluate(bool_model, {"src.A": 1}).missing == ["src.A"] + + +def test_values_that_mean_nothing_never_become_false_or_zero(): + as_bool = model_of(script_with("{ Property: src.A, Eq: false }")) + as_int = model_of(script_with("{ Property: src.A, Eq: 0 }")) + for nothing in ["N/A", "Unknown", "", "1", "yes"]: + assert evaluate(as_bool, {"src.A": nothing}).missing == ["src.A"] + for nothing in ["N/A", "Unknown", "", "1.0", "١"]: + assert evaluate(as_int, {"src.A": nothing}).missing == ["src.A"] + + +def test_a_native_value_read_as_a_string_takes_its_printed_form(): + model = model_of(script_with('{ Property: src.A, In: ["8", "True"] }')) + assert evaluate(model, {"src.A": 8}).value == "Yes" + assert evaluate(model, {"src.A": 8.0}).value == "Yes" + assert evaluate(model, {"src.A": True}).value == "Yes" + assert evaluate(model, {"src.A": 9}).value == "No" + + +def test_strings_compare_ordinally_and_with_regard_to_case(): + model = model_of(script_with('{ Property: src.A, StartsWith: "Mob" }')) + assert evaluate(model, {"src.A": "Mobile"}).value == "Yes" + assert evaluate(model, {"src.A": "mobile"}).value == "No" + + +def test_a_list_of_whole_and_decimal_numbers_reads_as_double(): + model = model_of(script_with("{ Property: src.A, In: [1, 2.5] }")) + assert model.properties[0].type == "double" + assert evaluate(model, {"src.A": 2.5}).value == "Yes" + + +def test_every_fault_in_a_script_is_reported_together(): + model, faults = validate_text( + script_with("{ Property: A, Equals: 1 }") + .replace("Format: 1", "Format: 2")) + assert model is None + paths = [fault.path for fault in faults] + assert "Format" in paths + assert "Rules[0].When.Property" in paths + + +def test_the_message_names_every_missing_property_in_order(): + model = model_of("\n".join([ + "Format: 1", + "Name: Demo", + "Version: 1.0.0", + "Output:", + " Name: Demo", + " Description: A demonstration.", + " ValueType: string", + " IsList: false", + "Checks:", + " First: { Property: src.B, Eq: true }", + "Rules:", + " - When: { Property: src.A, Eq: 1 }", + " Then: \"Yes\"", + " - Else: \"No\""])) + result = evaluate(model, {"src.A": {"NoValue": "Not today."}}) + assert result.missing == ["src.B", "src.A"] + assert result.message == ( + "Derived property 'Demo' has no value because 2 source properties " + "were not available. 'src.B' (element 'src' has no value for 'B': " + "property not present on this request). 'src.A' (element 'src' has " + "no value for 'A': Not today.). " + USUAL_CAUSES) + + +def test_a_value_that_cannot_be_read_is_shown_as_javascript_shows_it(): + model = model_of(script_with('{ Property: src.A, Eq: "x" }')) + model_int = model_of(script_with("{ Property: src.A, Eq: 1 }")) + assert "held '1.5' which cannot be read as int" in \ + evaluate(model_int, {"src.A": 1.5}).message + assert "held 'False' which cannot be read as int" in \ + evaluate(model_int, {"src.A": False}).message + assert evaluate(model, {"src.A": 8.0}).missing is None + + +def test_a_bool_output_is_a_boolean(): + model = model_of(script_with("{ Property: src.A, Eq: 1 }", "bool")) + assert evaluate(model, {"src.A": 1}).value is True + assert evaluate(model, {"src.A": 2}).value is False + + +def test_canonical_json_prints_whole_numbers_without_a_point(): + model = model_of(script_with("{ Property: src.A, Lt: 2.0 }")) + assert '"Lt": 2\n' in canonical(model) diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_shared_cases.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_shared_cases.py new file mode 100644 index 00000000..d45948bb --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_shared_cases.py @@ -0,0 +1,146 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +Runs every case file of the derived-properties submodule, which every +language runs, so that this package gives the same answer as the +reference tools, .NET, Rust and PHP for every case. +""" + +import os +import re + +import pytest + +from fiftyone_pipeline_derivedproperty import ( + DerivedPropertyElementBuilder, ScriptValidationError) +from fiftyone_pipeline_derivedproperty.evaluate import evaluate +from fiftyone_pipeline_derivedproperty.jsvalues import js_string +from fiftyone_pipeline_derivedproperty.parse import parse +from fiftyone_pipeline_derivedproperty.script_set import ScriptSet + +from .helpers import SUBMODULE, SourceElement, pipeline_of, processed + +TESTS = os.path.join(SUBMODULE, "tests") +INVALID = os.path.join(TESTS, "invalid") + + +def _read(path): + with open(path, encoding="utf-8") as file: + return parse(file.read()).document + + +def _value_cases(): + cases = [] + for file in sorted(os.listdir(TESTS)): + if not file.endswith(".cases.yaml"): + continue + document = _read(os.path.join(TESTS, file)) + for case in document["Cases"]: + cases.append(pytest.param( + document["Script"], case, + id=document["Script"] + ": " + case["Name"])) + return cases + + +def _rejection_cases(): + return [pytest.param(_read(os.path.join(INVALID, file)), id=file) + for file in sorted(os.listdir(INVALID)) + if re.search(r"\.ya?ml$", file, re.IGNORECASE)] + + +def _names_in(message): + """! The source property names a no value message gives. """ + return [name.lower() for name in re.findall( + r"'([A-Za-z][A-Za-z0-9]*\.[A-Za-z][A-Za-z0-9]*)' \(", message)] + + +_elements = {} + + +def _element_for(name): + if name not in _elements: + _elements[name] = (DerivedPropertyElementBuilder() + .add_built_in_script(name).build()) + return _elements[name] + + +def _run(element, case, wrap): + """! + Runs one case through a pipeline. Each element data key the script + reads gets an element listing every property the script reads under + that key, holding the values the case gives. + """ + script = element.scripts[0] + by_key = {} + + def entry_for(written): + key, name = written.split(".") + entry = by_key.setdefault(key.lower(), {"names": [], "values": {}}) + return entry, name + + for prop in script.properties: + entry, name = entry_for(prop.name) + entry["names"].append(name) + for written, value in (case.get("Properties") or {}).items(): + entry, name = entry_for(written) + entry["values"][name] = value + sources = [SourceElement(key, entry["values"], entry["names"], wrap) + for key, entry in by_key.items()] + flowdata = processed(pipeline_of(sources + [element])) + return flowdata.get("derived").get(script.output["Name"]) + + +@pytest.mark.parametrize("wrap", [True, False], + ids=["as AspectPropertyValues", "as plain values"]) +@pytest.mark.parametrize("name, case", _value_cases()) +def test_every_value_case(name, case, wrap): + element = _element_for(name) + value = _run(element, case, wrap) + reference = evaluate(element.scripts[0], case.get("Properties")) + expect = case["Expect"] + if "Value" in expect: + assert value.has_value(), value.no_value_message() + assert js_string(value.value()) == js_string(expect["Value"]) + else: + assert not value.has_value() + expected = sorted(p.lower() for p in expect["Missing"]) + assert sorted(_names_in(value.no_value_message())) == expected + # The element reads each value as the reference tools do, so it + # gives the reference message word for word. + assert value.no_value_message() == reference.message + with pytest.raises(Exception, match=re.escape(reference.message)): + value.value() + + +@pytest.mark.parametrize("document", _rejection_cases()) +def test_every_rejection_case_is_refused(document): + script_set = ScriptSet() + script_set.add_text(document.get("Name"), document["Script"], "case") + with pytest.raises(ScriptValidationError) as raised: + script_set.compile() + paths = [fault.path for fault in raised.value.faults] + messages = [fault.message for fault in raised.value.faults] + for expected in (document.get("Expect") or {}).get("Paths") or []: + assert expected in paths, messages + for fragment in (document.get("Expect") or {}).get("Mentions") or []: + assert any(fragment in message for message in messages), messages diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tox.ini b/pipeline/fiftyone_pipeline_derivedproperty/tox.ini new file mode 100644 index 00000000..a3858aa3 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tox.ini @@ -0,0 +1,21 @@ +[tox] +minversion = 4.0 + +[common] +deps = + pytest>=6 + pytest-cov + +[testenv] +description = run the tests with pytest +package = wheel +wheel_build_env = .pkg +deps = + {[common]deps} + -e../fiftyone_pipeline_core/ +commands = + pytest --cov={envsitepackagesdir}/fiftyone_pipeline_derivedproperty {tty:--color=yes} {posargs} + +# The package is not published, so Publish has built no copy of it to test. +# Its tests then run as they do in every build, on a wheel built here. +[testenv:pre-publish] From 55bf271ad6f20f3cffef1c329486477f24d42c07 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 23:52:29 +0000 Subject: [PATCH 2/2] Offer the derived properties with a derived property element fiftyone_pipeline_derivedproperty runs the scripts of derived-properties, built into the package from the submodule's scripts folder or given as files or text, and writes one property for each under derived. parse, validate, evaluate and canonical follow the reference tools of derived-properties function for function, reading YAML with the YAML core schema. The package is tested but not yet published. --- AGENTS.md | 12 + README.md | 2 +- .../readme.md | 105 ++ .../__init__.py | 27 + .../canonical.py | 123 ++ .../derived_property_data.py | 40 + .../derived_property_element.py | 462 ++++++++ .../derived_property_element_builder.py | 82 ++ .../errors.py | 54 + .../evaluate.py | 388 +++++++ .../jsvalues.py | 124 ++ .../parse.py | 235 ++++ .../script_set.py | 176 +++ .../validate.py | 1010 +++++++++++++++++ pipeline/readme.md | 3 +- 15 files changed, 2841 insertions(+), 2 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/readme.md create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/__init__.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/canonical.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_element.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_element_builder.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/errors.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/evaluate.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/jsvalues.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/parse.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/script_set.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/validate.py diff --git a/AGENTS.md b/AGENTS.md index 01221404..6c259e65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -342,6 +342,18 @@ A package's `ExtraPaths` in `ci/packages.json` cannot be a folder inside cxx update therefore releases no package, even when it brings new translations in `cxx/data/ip-intelligence`. +`pipeline/fiftyone_pipeline_derivedproperty` runs the scripts of +[derived-properties](https://github.com/51Degrees/derived-properties), which is +its submodule `src/fiftyone_pipeline_derivedproperty/derived-properties`, inside +the package so that the wheel carries its `scripts` folder (`MANIFEST.in`) and +nothing else from it. Its tests run every case file in the submodule's `tests` +folder, so moving the submodule moves the cases the tests run. `parse.py`, +`validate.py`, `evaluate.py` and `canonical.py` follow the reference tools in +the submodule's `tools` folder function for function, with `jsvalues.py` +holding the JavaScript rules they rely on, so a change there is made here the +same way. The package is `"Publish": false` in `ci/packages.json`, so it is +tested in every build and not published. + ## Releases Pull requests go to `rc`, not `main`, and stay drafts until they have been diff --git a/README.md b/README.md index 0a8ba812..23688d15 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ own folder, which has its own README. | Product | Folder | PyPI packages | | --- | --- | --- | -| Pipeline API | [pipeline](pipeline/readme.md) | `fiftyone_pipeline_core`, `_engines`, `_engines_fiftyone`, `_cloudrequestengine`, `_translation`, `_did` | +| Pipeline API | [pipeline](pipeline/readme.md) | `fiftyone_pipeline_core`, `_engines`, `_engines_fiftyone`, `_cloudrequestengine`, `_translation`, `_did`, `_derivedproperty` (not yet published) | | Device Detection | [device-detection](device-detection/readme.md) | `fiftyone_devicedetection`, `_shared`, `_cloud`, `_onpremise` | | IP Intelligence | [ip-intelligence](ip-intelligence/readme.md) | `fiftyone_ipintelligence`, `_shared`, `_cloud`, `_onpremise`, `_translation` | | Location | [location](location/readme.md) | `fiftyone_location` | diff --git a/pipeline/fiftyone_pipeline_derivedproperty/readme.md b/pipeline/fiftyone_pipeline_derivedproperty/readme.md new file mode 100644 index 00000000..e6854ca9 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/readme.md @@ -0,0 +1,105 @@ +# Pipeline Derived Properties + +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python Pipeline Derived Properties** + +[Developer Documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_derivedproperty&utm_term=pipeline-derived-properties "Developer Documentation") + +## Introduction + +The Pipeline is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines) + +## Requirements + +* Python 3.10+ + +## This package fiftyone_pipeline_derivedproperty + +This package adds the derived property flow element to the pipeline. It computes new properties from properties other elements have already put into the flow data, such as HumanConfidence from Device Detection and IP Intelligence properties, and IsGdpr from the country of an IP address. It holds no data file, sends no request and reads no evidence. + +Each property is defined by a script in the format the [derived-properties](https://github.com/51Degrees/derived-properties) repository describes, written in YAML or JSON. The scripts that repository holds are built into this package, and the same scripts give the same answers in every 51Degrees language, which the case files in that repository prove. Every script writes one property under the element data key `derived`. + +Where a property a script reads is not available, the derived property has no value, and reading it raises a message naming every property that was missing and why. + +This package is built on top of: + +* [fiftyone_pipeline_core](https://pypi.org/project/fiftyone-pipeline-core/) + +## Scripts + +A script built into the package is named by its name. A script of your own is given as a file, where the file name may hold a wildcard, or as text held by your code. The script's `Name` must equal its file name without the extension, or the name given with the text. + +```python +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_derivedproperty import ( + DerivedPropertyElementBuilder, builtin_script_names) + +print(builtin_script_names()) # ['HumanConfidence', 'IsGdpr'] + +derived = (DerivedPropertyElementBuilder() + # A script built into this package. + .add_built_in_script("IsGdpr") + # A file, or several files, from your own environment. + .add_script_file("derived/*.yaml") + # The text of a script held by your own code, YAML or JSON. + .add_script("MyProperty", script_text) + .build()) + +# Add the element after the elements that supply what its scripts read. +pipeline = (PipelineBuilder() + .add(ip_intelligence_engine) + .add(derived) + .build()) + +flowdata = pipeline.create_flowdata() +flowdata.evidence.add_from_dict(evidence) +flowdata.process() +is_gdpr = flowdata.derived.IsGdpr +print(is_gdpr.value() if is_gdpr.has_value() else is_gdpr.no_value_message()) +``` + +Every fault in every script is reported together when the element is built, in a `ScriptValidationError` whose `faults` list each one. + +## Configuration + +A pipeline configuration names the element with `elementPath` `fiftyone_pipeline_derivedproperty` and `elementName` `DerivedPropertyElement`. Its settings are `scripts`, naming scripts built into the package, and `script_files`, giving paths to script files. Each is a list of strings, or one string with commas between the entries, where an entry holding a comma is wrapped in double quotes. The setting names are matched without regard to case or underscores, so `Scripts` and `ScriptFiles` as the other languages write them work too, and any other setting is refused. + +```json +{ + "PipelineOptions": { + "Elements": [ + { + "elementPath": "fiftyone_pipeline_derivedproperty", + "elementName": "DerivedPropertyElement", + "elementParameters": { "scripts": "IsGdpr", "script_files": "derived/*.yaml" } + } + ] + } +} +``` + +IsGdpr reads `ip.CountryCode`, so the pipeline also needs an element giving that, such as the IP Intelligence cloud engine or the on-premise engine with a data file that has the property. The free Lite IP Intelligence data file does not have it, so the element refuses a pipeline built on that file, as the next section describes. + +## When the pipeline is built + +The element refuses, with a `PipelineConfigurationError`, a pipeline in which no element supplies a property a script reads, or in which another element already writes a property a script writes. Where an element sits in the pipeline is not judged, so an element supplying a property that runs after this one builds, and on each request the derived property then has no value. That is logged once, at error level. + +When the pipeline is built, the element logs one line for each script at information level, naming the script, its version, its format, where it came from and the property it writes, a warning for a script marked deprecated, and the compiled script as JSON at debug level. + +## Installation + +The package is not yet published to PyPI. To use it from this repository, clone the repository with its submodules, because the built in scripts are the `scripts` folder of the `derived-properties` submodule. + +``` +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git +``` + +## Tests + +The tests run every case file in the `tests` folder of the `derived-properties` submodule, including the scripts under `tests/invalid` that every language must refuse, with `python -m tox` in this folder. + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_derivedproperty&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_derivedproperty&utm_term=find-out-more-documentation) +- https://github.com/51Degrees/derived-properties +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/__init__.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/__init__.py new file mode 100644 index 00000000..71ada997 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/__init__.py @@ -0,0 +1,27 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +from .derived_property_element import DerivedPropertyElement +from .derived_property_element_builder import DerivedPropertyElementBuilder +from .derived_property_data import DerivedPropertyData +from .errors import PipelineConfigurationError, ScriptValidationError +from .script_set import builtin_script_names diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/canonical.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/canonical.py new file mode 100644 index 00000000..e18f4e01 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/canonical.py @@ -0,0 +1,123 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +Prints a validated model as canonical JSON, which the element writes to +the debug log at build, so that anyone holding the log can see exactly +what was evaluated without the file. The form is PascalCase keys in the +order of the format reference, two space indent, literal types kept, the +inferred type of each source property and the computed Dependencies list. + +This follows tools/canonical.mjs of derived-properties, and prints the +same text. +""" + +from .jsvalues import js_json +from .validate import OUTPUT_KEYS + + +def canonical(model): + """! + @param model: a model from validate.py. + @return: canonical JSON, without a trailing newline. + """ + return js_json(canonical_object(model), indent=2) + + +def canonical_object(model): + """! The canonical form as a dict, before it is printed. """ + result = { + "Format": model.format, + "Name": model.name, + "Version": model.version + } + if model.deprecated: + result["Deprecated"] = True + result["DeprecationNote"] = model.deprecation_note + result["Output"] = _canonical_output(model.output) + + # The inferred type of every source property, which the file does not + # carry because it is worked out from the literals compared against. + result["Properties"] = {} + for prop in model.properties: + result["Properties"][prop.name] = {"Type": prop.type} + + if model.checks: + result["Checks"] = {} + for check in model.checks: + result["Checks"][check.name] = _canonical_condition( + check.condition, model) + + result["Rules"] = [_canonical_rule(rule, model) for rule in model.rules] + return result + + +def _canonical_rule(rule, model): + if rule.when is None: + return {"Else": rule.value} + return { + "When": _canonical_condition(rule.when, model), + "Then": rule.value + } + + +def _canonical_output(output): + result = {} + for key in OUTPUT_KEYS: + if key == "Values": + continue + if output.get(key) is None: + continue + result[key] = output[key] + if output.get("Values") is not None: + result["Values"] = [ + {"Name": value["Name"]} if value["Description"] is None + else {"Name": value["Name"], "Description": value["Description"]} + for value in output["Values"]] + return result + + +def _canonical_condition(condition, model): + kind = condition.kind + if kind == "compare": + return {"Property": condition.propertyName, + condition.op: condition.operand} + if kind == "check": + return {"Check": condition.name} + if kind == "aggregate": + return {condition.agg: _canonical_group(condition.group, model), + condition.op: condition.operand} + if kind == "all": + return {"All": [_canonical_condition(i, model) + for i in condition.items]} + if kind == "any": + return {"Any": [_canonical_condition(i, model) + for i in condition.items]} + if kind == "not": + return {"Not": _canonical_condition(condition.item, model)} + raise RuntimeError("unknown condition kind '" + kind + "'") + + +def _canonical_group(group, model): + if group is None: + return "Checks" + return [model.checks[index].name for index in group] diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py new file mode 100644 index 00000000..dcca50e6 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py @@ -0,0 +1,40 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary + + +class DerivedPropertyData(ElementDataDictionary): + """! + The element data every derived property element in a pipeline writes + into, under the key derived. Each property is an AspectPropertyValue, + and reading the value of one with no value raises the message naming + every source property that was not available. Names are matched + without regard to case. + """ + + def __init__(self, flow_element): + super().__init__(flow_element, {}) + + def set_value(self, name, value): + """! Writes the value of one property. """ + self.contents[name.lower()] = value diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_element.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_element.py new file mode 100644 index 00000000..2771940f --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_element.py @@ -0,0 +1,462 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.flowelement import FlowElement + +from .canonical import canonical +from .derived_property_data import DerivedPropertyData +from .errors import PipelineConfigurationError, ScriptValidationError +from .evaluate import NoValue, evaluate +from .jsvalues import is_number +from .script_set import ScriptSet +from .validate import Fault + +## The element data key every derived property element writes under. +DATA_KEY = "derived" + +_NOT_PRESENT = "property not present on this request" + +# The settings the element takes, matched without regard to case or +# underscores, so that Scripts and ScriptFiles, as the other languages +# write them, work too. +_SETTINGS = {"scripts": "scripts", "scriptfiles": "script_files"} + +# The Output fields given in each property's metadata beyond the name, +# type, category and description every element gives. +_OUTPUT_METADATA = [ + ("StoredValueType", "storedvaluetype"), + ("DefaultValue", "defaultvalue"), + ("IsList", "islist"), + ("IsMandatory", "ismandatory"), + ("IsObsolete", "isobsolete"), + ("IsPopular", "ispopular"), + ("ExportValues", "exportvalues"), + ("Url", "url"), + ("DisplayOrder", "displayorder"), + ("PropertyId", "propertyid"), + ("VendorIds", "vendorids"), + ("Dependencies", "dependencies"), + ("Values", "values") +] + +# What a read gives where the request carried no element data at all. +_NO_ELEMENT_DATA = object() + + +class DerivedPropertyElement(FlowElement): + """! + Computes new properties from properties other elements have already + put into the flow data, following one or more scripts in the format of + the derived-properties repository. Every script writes one property + under the element data key derived. + + The element holds no data file, sends no request and reads no + evidence, so it accepts no evidence. + """ + + def __init__(self, compiled=None, **settings): + """! + Builds the element from its settings, as a pipeline configuration + file gives them, and from scripts a builder has already read. + + @param compiled: scripts read by DerivedPropertyElementBuilder. + @param settings: scripts names scripts built into this package and + script_files gives paths to script files, where a file name may + hold a wildcard. Each is a list of strings, or one string with + commas between the entries, where an entry holding a comma is + wrapped in double quotes. The names are matched without regard + to case or underscores, and any other setting is refused. + @raise ScriptValidationError: where a script does not validate, or + two scripts write one property. + """ + super().__init__() + self.datakey = DATA_KEY + script_set = ScriptSet() + _read_settings(settings, script_set) + self.scripts = script_set.compile() + list(compiled or []) + if not self.scripts: + raise ValueError( + "At least one script must be configured. Name the scripts " + "built into this package with the scripts setting, give " + "script files with script_files, or use " + "DerivedPropertyElementBuilder.") + _check_output_names_are_unique(self.scripts) + + ## The metadata of the properties this element writes, being the + # Output block of each script, by lower case name. + self.script_properties = {} + for script in self.scripts: + self.script_properties[script.output["Name"].lower()] = \ + _metadata(script) + self.properties = dict(self.script_properties) + + # The scripts that have reported a missing element data, as that + # is reported once rather than on every request. + self._reported = set() + + def get_properties(self): + """! + The properties written under derived, being those of every derived + property element in the pipelines this one is in. They all write + one element data, so each element gives them all. + """ + properties = {} + for pipeline in self.pipelines: + for element in pipeline.flow_elements: + if element is not self and _is_derived_element(element): + properties.update(element.script_properties) + properties.update(self.script_properties) + return properties + + def on_registration(self, pipeline): + """! + Refuses a pipeline that cannot supply a property a script reads, + or in which another element writes a property a script writes, + then logs what each script produces. + + @raise PipelineConfigurationError: where the pipeline cannot be + used. + """ + self._check_pipeline(pipeline) + self._log_scripts(pipeline) + + def process_internal(self, flowdata): + """! + Runs every script in order, so a script can read what an earlier + one wrote. + """ + # Every derived property element in a pipeline writes one element + # data, as the properties of all of them are under derived. + data = flowdata.data.get(DATA_KEY) + if not isinstance(data, DerivedPropertyData): + data = DerivedPropertyData(self) + flowdata.set_element_data(data) + for index, script in enumerate(self.scripts): + properties = {} + for prop in script.properties: + value = self._read_source(flowdata, script, index, prop) + if value is not _NO_ELEMENT_DATA: + properties[prop.name] = value + result = evaluate(script, properties) + data.set_value( + script.output["Name"], + AspectPropertyValue(None, result.value) if result.has_value + else AspectPropertyValue(result.message)) + + # ------------------------------------------------------------------- + # Reading source properties. + # ------------------------------------------------------------------- + + def _read_source(self, flowdata, script, index, prop): + """! + Reads one source property for one request, in the form the + evaluator takes. A value with no value of its own becomes a + NoValue, and a weighted list becomes a list of { Value, Weight }. + """ + dot = prop.name.index(".") + key = prop.name[:dot] + name = prop.name[dot + 1:] + + element_data = flowdata.data.get(key.lower()) + if element_data is None: + self._report_missing_element_data(flowdata, script, index, prop, + key) + return _NO_ELEMENT_DATA + + try: + raw = element_data.get(name) + except Exception as error: + # An element that does not have the property says why, and + # that reason is carried. + return NoValue(str(error) or _NOT_PRESENT) + + # A source value with its own no value state hands its message + # on, so the reason reaches back to the element that knows it. + if _is_aspect_property_value(raw): + if not raw.has_value(): + return NoValue(raw.no_value_message() or _NOT_PRESENT) + raw = raw.value() + if isinstance(raw, list): + return _weighted(raw) + return raw + + def _report_missing_element_data(self, flowdata, script, index, prop, + key): + """! + Reports, once for each script, that a request carried no element + data at all under a key the pipeline has an element for, which + means that element ran after this one. + """ + if index in self._reported: + return + self._reported.add(index) + flowdata.pipeline.log( + "error", + "The script '" + script.name + "' reads '" + prop.name + "', " + "and the request carried no element data '" + key + "' at all, " + "so the script has no value. The usual cause is the element " + "that supplies it running after this element rather than " + "before it. This is reported once.") + + # ------------------------------------------------------------------- + # Building the pipeline. + # ------------------------------------------------------------------- + + def _log_scripts(self, pipeline): + """! + Logs one line for each script at information level, a warning for + a deprecated one, and the compiled script as canonical JSON at + debug level. + """ + for script in self.scripts: + pipeline.log( + "info", + "Derived property script '" + script.name + "' version " + + script.version + ", format " + str(script.format) + + ", from " + script.source + ", produces '" + DATA_KEY + + "." + script.output["Name"] + "' as " + + script.output["ValueType"] + ".") + if script.deprecated: + pipeline.log( + "warning", + "Derived property script '" + script.name + + "' is deprecated. " + script.deprecation_note) + pipeline.log( + "debug", + "Derived property script '" + script.name + + "' compiled to " + canonical(script)) + + def _check_pipeline(self, pipeline): + """! + The checks run when the pipeline adds the element. Where an + element sits is not judged, because the element may read a + property from an element added after it on purpose. + """ + others = [e for e in pipeline.flow_elements if e is not self] + faults = [] + + for script in self.scripts: + output_name = script.output["Name"] + if any(_same_key(e.datakey, DATA_KEY) and + _supplies(e, output_name) is True for e in others): + faults.append( + "The script '" + script.name + "' writes the derived " + "property '" + output_name + "', and another element " + "in the pipeline already writes a property of that " + "name. One pipeline writes each derived property once.") + + for index, script in enumerate(self.scripts): + for prop in script.properties: + if self._is_supplied(others, prop, index): + continue + # Every property a script names is needed, so a pipeline + # that cannot supply one would give no value on every + # request. + faults.append( + "The script '" + script.name + "' needs the property '" + + prop.name + "', and no element in the pipeline supplies " + "it. Either add the element that supplies '" + + prop.name + "', or change the script so that it does " + "not name '" + prop.name + "'.") + + if faults: + raise PipelineConfigurationError(faults) + + def _is_supplied(self, others, prop, index): + dot = prop.name.index(".") + key = prop.name[:dot] + name = prop.name[dot + 1:] + # A script can read what an earlier script of this element + # writes, and nothing that a later one writes, as the scripts run + # in order. + if _same_key(key, DATA_KEY): + for earlier in self.scripts[:index]: + if _same_key(earlier.output["Name"], name): + return True + return any(_same_key(e.datakey, key) and _supplies(e, name) + is not False for e in others) + + +# --------------------------------------------------------------------- +# Settings and metadata. +# --------------------------------------------------------------------- + +def _read_settings(settings, script_set): + for key, value in settings.items(): + setting = _SETTINGS.get(key.replace("_", "").lower()) + if setting is None: + # A setting written wrongly would otherwise leave its scripts + # out without a word. + raise ValueError( + "The derived property element has no setting '" + key + + "'. Its settings are scripts and script_files.") + for entry in _list_setting(setting, value): + if setting == "scripts": + script_set.add_built_in(entry) + else: + script_set.add_files(entry) + + +def _list_setting(name, value): + if value is None: + return [] + if isinstance(value, str): + return [] if value.strip() == "" else _split(value) + if (isinstance(value, (list, tuple)) and + all(isinstance(entry, str) for entry in value)): + return list(value) + raise TypeError( + "The " + name + " setting must be a list of strings, or one string " + "with commas between the entries.") + + +def _split(value): + """! + Splits a setting on its commas, where an entry wrapped in double + quotes may hold a comma, and two double quotes inside the quotes are + one. + """ + entries = [] + current = "" + quoted = False + i = 0 + while i < len(value): + character = value[i] + if character == '"': + if quoted and i + 1 < len(value) and value[i + 1] == '"': + current += '"' + i += 1 + else: + quoted = not quoted + elif character == "," and not quoted: + entries.append(current.strip()) + current = "" + else: + current += character + i += 1 + entries.append(current.strip()) + return entries + + +def _check_output_names_are_unique(scripts): + seen = {} + faults = [] + for script in scripts: + key = script.output["Name"].lower() + if key in seen: + faults.append(Fault( + script.name, script.source, "Output.Name", None, + "two scripts in the same element write the property '" + + script.output["Name"] + "', being '" + seen[key] + "' and '" + + script.name + "'. One element writes each property once")) + continue + seen[key] = script.name + if faults: + raise ScriptValidationError(faults) + + +def _metadata(script): + """! + The metadata of the property a script writes, with the keys the other + elements give followed by the rest of the Output block. + """ + output = script.output + result = { + "name": output["Name"], + "type": output["ValueType"], + "category": output.get("Category") or "", + "description": output.get("Description") + } + for field, key in _OUTPUT_METADATA: + if output.get(field) is not None: + result[key] = output[field] + return result + + +# --------------------------------------------------------------------- +# Helpers. +# --------------------------------------------------------------------- + +def _is_derived_element(element): + return (_same_key(getattr(element, "datakey", None), DATA_KEY) and + isinstance(getattr(element, "script_properties", None), dict)) + + +def _same_key(a, b): + return (isinstance(a, str) and isinstance(b, str) and + a.lower() == b.lower()) + + +def _supplies(element, name): + """! + Whether an element gives a property, or None where it does not know + yet. A cloud engine learns its properties from the cloud service, so + an element listing no properties may give any property under its key. + """ + try: + # Another derived property element is asked only for what it + # writes itself. + properties = (element.script_properties + if _is_derived_element(element) + else element.get_properties()) + except Exception: + return None + if not isinstance(properties, dict) or not properties: + return None + for key, meta in properties.items(): + if _same_key(key, name): + return True + if isinstance(meta, dict) and _same_key(meta.get("name"), name): + return True + return False + + +def _is_aspect_property_value(value): + return isinstance(value, AspectPropertyValue) or ( + callable(getattr(value, "has_value", None)) and + callable(getattr(value, "value", None)) and + callable(getattr(value, "no_value_message", None))) + + +def _weighted(items): + """! + A list as the evaluator takes it. A list of weighted values, whose + members each carry a value and a weight as 51Degrees elements give + them, becomes a list of { Value, Weight }, using the raw weighting + where there is one. Any other list is left as it is, which the + evaluator refuses where a single value is needed. + """ + members = [] + for item in items: + if not isinstance(item, dict): + return items + fields = {str(k).lower(): v for k, v in item.items()} + if "value" not in fields: + return items + weight = 0 + for field in ("rawweighting", "weighting", "weight"): + if is_number(fields.get(field)): + weight = fields[field] + break + members.append({"Value": fields["value"], "Weight": weight}) + return members if members else items diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_element_builder.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_element_builder.py new file mode 100644 index 00000000..d2a70e85 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_element_builder.py @@ -0,0 +1,82 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +from .derived_property_element import DerivedPropertyElement +from .script_set import ScriptSet + + +class DerivedPropertyElementBuilder(object): + """! + Builds a DerivedPropertyElement from scripts built into this package, + script files, and script text held by code, in any mix. Every fault in + every script is reported together when the element is built. + """ + + def __init__(self): + self._scripts = ScriptSet() + + def add_built_in_script(self, name): + """! + Adds a script built into this package. + + @param name: the name of the script in any case, such as + HumanConfidence or IsGdpr. + @return: this builder. + """ + self._scripts.add_built_in(name) + return self + + def add_script_file(self, path): + """! + Adds a script file, or every file a wildcard in the file name + matches. The script's Name must equal the file name without its + extension. + + @return: this builder. + """ + self._scripts.add_files(path) + return self + + def add_script(self, name, text): + """! + Adds a script held as text by your own code, as YAML or JSON. + + @param name: the name to know the script by, which the script's own + Name must equal. + @param text: the script. + @return: this builder. + """ + if not isinstance(name, str) or name.strip() == "": + raise TypeError("A script added from code needs its name.") + if not isinstance(text, str): + raise TypeError("The script '" + name + "' added from code " + "needs its text.") + self._scripts.add_text(name, text) + return self + + def build(self): + """! + Builds the element. + + @raise ScriptValidationError: where any script does not validate. + """ + return DerivedPropertyElement(compiled=self._scripts.compile()) diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/errors.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/errors.py new file mode 100644 index 00000000..89242b70 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/errors.py @@ -0,0 +1,54 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + + +class ScriptValidationError(Exception): + """! + Raised when one or more scripts do not validate. The message lists + every fault in every script, one per line, so one failure says + everything that has to be fixed. + """ + + def __init__(self, faults): + """! + @param faults: every fault, each a validate.Fault. + """ + from .validate import faults_message + super().__init__("The derived property scripts are not valid.\n" + + faults_message(faults)) + self.faults = faults + + +class PipelineConfigurationError(Exception): + """! + Raised when a pipeline cannot supply what the scripts of a derived + property element read, or when two elements write one derived + property. + """ + + def __init__(self, faults): + """! + @param faults: one sentence for each problem. + """ + super().__init__("The derived property element cannot be used in " + "this pipeline.\n" + "\n".join(faults)) + self.faults = faults diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/evaluate.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/evaluate.py new file mode 100644 index 00000000..77191385 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/evaluate.py @@ -0,0 +1,388 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +The evaluator. Given a validated model and the source property values for +one request, it produces the output value, or the reason there is no +output value. + +This follows tools/evaluate.mjs of derived-properties function for +function, with the same messages, so that the conformance cases in that +repository prove it. +""" + +import math +import re + +from .jsvalues import ( + is_bool, is_number, is_whole, is_string, is_list, is_mapping, + js_string) + +## The sentence appended to every "no value" message, naming the things +# that usually cause a source property to be missing. +USUAL_CAUSES = ( + "Usual causes are the element that supplies the property not being in " + "the pipeline, or being added after this element rather than before " + "it, the property being excluded in the engine configuration, the " + "property not being included in the resource key, or JavaScript that " + "populates the property not having run yet.") + +## The smallest value the type int holds. +INT_MIN = -2147483648 +## The largest value the type int holds. +INT_MAX = 2147483647 + +_NOT_PRESENT = "property not present on this request" +_INT_TEXT = re.compile(r"^[+-]?[0-9]+$") +_DOUBLE_TEXT = re.compile( + r"^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$") + + +class Result(object): + """! + What a script gives for one request, being a value, or the source + properties that were missing and the message naming them. + """ + + def __init__(self, value=None, has_value=False, missing=None, + message=None, trace=None): + self.value = value + self.has_value = has_value + self.missing = missing + self.message = message + self.trace = trace + + +class NoValue(object): + """! + A source value that exists but carries a no value message, as a case + file writes it with { NoValue: "..." }. + """ + + def __init__(self, message): + self.message = message + + +def evaluate(model, properties, trace=False): + """! + Runs one script for one request. + + @param model: a model from validate.py. + @param properties: a dict from "element.Property" to a native value, a + { "String": "..." } dict holding the string form, a + { "NoValue": "..." } dict or a NoValue standing for a source value + that exists but carries a no value message, or a list of + { "Value", "Weight" } dicts. A property that is not a key of the + dict is absent. + @param trace: whether to record what was read, which checks passed + and which rule matched. + @rtype: Result + """ + lookup = _build_lookup(properties) + slots = [_read_slot(p, lookup) for p in model.properties] + + recorded = None + if trace: + recorded = { + "properties": [ + {"name": p.name, "available": slot[0], + "value": slot[1] if slot[0] else None, "reason": slot[2]} + for p, slot in zip(model.properties, slots)], + "checks": [], + "matchedRule": None, + "matchedBy": None + } + + # Every source property the script names is read before anything is + # evaluated. A property is either there or it is not, and where any + # one of them is missing the script says so rather than guessing. + missing = [] + reasons = [] + for prop, slot in zip(model.properties, slots): + if not slot[0]: + missing.append(prop) + reasons.append(slot[2]) + if missing: + return Result( + missing=[p.name for p in missing], + message=_missing_message(model, missing, reasons), + trace=recorded) + + state = _State(model, slots) + for index, check in enumerate(model.checks): + state.checks[index] = _evaluate_condition(check.condition, state) + if recorded is not None: + recorded["checks"].append({ + "name": check.name, + "state": describe_state(state.checks[index])}) + + for index, rule in enumerate(model.rules): + # An Else rule has no condition and always matches. + if rule.when is None or _evaluate_condition(rule.when, state): + if recorded is not None: + recorded["matchedRule"] = index + recorded["matchedBy"] = "Else" if rule.when is None else "When" + return Result(value=rule.value, has_value=True, trace=recorded) + + # Every script ends in an Else, which the validator enforces, so the + # loop above always returns. + raise RuntimeError( + "the rules of '" + model.output["Name"] + "' do not end in an " + "Else, which format 1 does not allow") + + +def describe_state(value): + """! "true" or "false", for the trace. """ + return "true" if value else "false" + + +class _State(object): + def __init__(self, model, slots): + self.model = model + self.slots = slots + self.checks = [None] * len(model.checks) + + +# --------------------------------------------------------------------- +# Reading source properties. +# --------------------------------------------------------------------- + +def _build_lookup(properties): + lookup = {} + if properties: + for key, value in properties.items(): + lookup[key.lower()] = value + return lookup + + +def _read_slot(prop, lookup): + """! + Reads one source property for one request, converting it to the type + the script inferred for it. + + @return: (available, value, reason) + """ + dot = prop.name.index(".") + element_key = prop.name[:dot] + property_name = prop.name[dot + 1:] + + def absent(reason): + return (False, None, reason) + + def not_available(detail): + return absent("element '" + element_key + "' has no value for '" + + property_name + "': " + detail) + + if prop.key not in lookup: + return not_available(_NOT_PRESENT) + raw = lookup[prop.key] + if raw is None: + return not_available(_NOT_PRESENT) + if isinstance(raw, NoValue): + return not_available(js_string(raw.message)) + if is_mapping(raw): + if "NoValue" in raw: + return not_available(js_string(raw["NoValue"])) + if "String" in raw: + raw = raw["String"] + if raw is None: + return not_available(_NOT_PRESENT) + if is_list(raw): + # A list of weighted values takes the value with the highest + # weight. Any other list where a single value is needed is invalid. + weighted = len(raw) > 0 and all( + is_mapping(m) and "Value" in m for m in raw) + if not weighted: + return absent("held a list where a single value is needed") + best = raw[0] + for member in raw: + if _weight(member) > _weight(best): + best = member + raw = best["Value"] + if raw is None: + return not_available(_NOT_PRESENT) + + converted = convert_value(raw, prop.type) + if not converted[0]: + return absent("held '" + _display_raw(raw) + + "' which cannot be read as " + prop.type) + return (True, converted[1], None) + + +def _weight(member): + weight = member.get("Weight") + return weight if is_number(weight) else 0 + + +def _display_raw(raw): + if is_string(raw): + return raw + if is_bool(raw): + return "True" if raw else "False" + return js_string(raw) + + +def _is_int32(value): + return is_whole(value) and INT_MIN <= value <= INT_MAX + + +def convert_value(raw, type): + """! + Converts a source value, which arrives either as the native type or + as a string, to the type the script inferred. + + @return: (ok, value) + """ + if is_string(raw): + return convert_string(raw, type) + if type == "bool": + return (True, raw) if is_bool(raw) else (False, None) + if type == "int": + return (True, int(raw)) if _is_int32(raw) else (False, None) + if type == "double": + if is_number(raw) and math.isfinite(raw): + return (True, raw) + return (False, None) + if type == "string": + if is_bool(raw): + return (True, "True" if raw else "False") + if is_number(raw): + return (True, js_string(raw)) + return (False, None) + return (False, None) + + +def convert_string(raw, type): + """! + Converts the string form of a value to the type given. Values are + never coerced loosely, so N/A, Unknown and an empty string never + become false or zero. + + @return: (ok, value) + """ + if not is_string(raw): + return (False, None) + if type == "bool": + trimmed = raw.strip().lower() + if trimmed == "true": + return (True, True) + if trimmed == "false": + return (True, False) + return (False, None) + if type == "int": + trimmed = raw.strip() + if not _INT_TEXT.match(trimmed): + return (False, None) + value = int(trimmed) + return (True, value) if _is_int32(value) else (False, None) + if type == "double": + trimmed = raw.strip() + if not _DOUBLE_TEXT.match(trimmed): + return (False, None) + value = float(trimmed) + return (True, value) if math.isfinite(value) else (False, None) + if type == "string": + return (True, raw) + return (False, None) + + +# --------------------------------------------------------------------- +# Conditions. +# --------------------------------------------------------------------- + +def _evaluate_condition(condition, state): + """! + Every condition is true or false, because the rules only run once + every source property the script names has been read. + """ + kind = condition.kind + if kind == "compare": + return _compare(state.slots[condition.slot][1], condition.op, + condition.operand, condition.type) + if kind == "check": + return state.checks[condition.index] + if kind == "aggregate": + return _compare(_count_of(condition.agg, condition.group, state), + condition.op, condition.operand, "int") + if kind == "all": + return all(_evaluate_condition(i, state) for i in condition.items) + if kind == "any": + return any(_evaluate_condition(i, state) for i in condition.items) + if kind == "not": + return not _evaluate_condition(condition.item, state) + raise RuntimeError("unknown condition kind '" + kind + "'") + + +def _count_of(agg, group, state): + """! + Counts checks in a group. A group of None means every named check. + """ + indexes = range(len(state.checks)) if group is None else group + count = 0 + for index in indexes: + value = state.checks[index] + if agg == "Passed" and value is True: + count += 1 + elif agg == "Failed" and value is False: + count += 1 + return count + + +def _compare(left, op, right, type): + if op == "Eq": + return left == right + if op == "Ne": + return left != right + if op == "Gt": + return left > right + if op == "Ge": + return left >= right + if op == "Lt": + return left < right + if op == "Le": + return left <= right + if op == "In": + return any(member == left for member in right) + if op == "NotIn": + return not any(member == left for member in right) + if op == "StartsWith": + return left.startswith(right) + if op == "EndsWith": + return left.endswith(right) + if op == "Contains": + return right in left + raise RuntimeError("unknown operator '" + op + "' on type " + type) + + +# --------------------------------------------------------------------- +# Messages. +# --------------------------------------------------------------------- + +def _missing_message(model, missing, reasons): + count = len(missing) + noun = ("1 source property was not available" if count == 1 + else str(count) + " source properties were not available") + details = " ".join( + "'" + p.name + "' (" + reason + ")." + for p, reason in zip(missing, reasons)) + return ("Derived property '" + model.output["Name"] + "' has no value " + "because " + noun + ". " + details + " " + USUAL_CAUSES) diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/jsvalues.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/jsvalues.py new file mode 100644 index 00000000..63b19e76 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/jsvalues.py @@ -0,0 +1,124 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +The few JavaScript rules the reference tools of derived-properties rely +on, so that this package reads, compares and prints values as they do. +JavaScript has one number type, so 8 and 8.0 are the same value and both +print as 8, and a boolean is never a number, which in Python it is. +""" + +import json +import math + + +def is_bool(value): + """! Whether a value is a boolean. """ + return isinstance(value, bool) + + +def is_number(value): + """! Whether a value is a number, which a boolean is not. """ + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def is_whole(value): + """! Whether a value is a whole number, as Number.isInteger says. """ + if not is_number(value): + return False + if isinstance(value, int): + return True + return math.isfinite(value) and value.is_integer() + + +def is_string(value): + """! Whether a value is a string. """ + return isinstance(value, str) + + +def is_list(value): + """! Whether a value is a list. """ + return isinstance(value, list) + + +def is_mapping(value): + """! Whether a value is a mapping. """ + return isinstance(value, dict) + + +def number_text(value): + """! + A number as JavaScript prints it, so a whole number has no decimal + point. + """ + if isinstance(value, int): + return str(value) + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "Infinity" if value > 0 else "-Infinity" + if value.is_integer() and abs(value) < 1e21: + return str(int(value)) + text = repr(value) + if "e" in text: + mantissa, exponent = text.split("e") + sign = "-" if exponent.startswith("-") else "+" + text = mantissa + "e" + sign + exponent.lstrip("+-").lstrip("0") + return text + + +def js_string(value): + """! A value as JavaScript's String() gives it. """ + if value is None: + return "null" + if is_bool(value): + return "true" if value else "false" + if is_number(value): + return number_text(value) + if is_string(value): + return value + if is_list(value): + return ",".join( + "" if item is None else js_string(item) for item in value) + return "[object Object]" + + +def _plain(value): + """! A value with every whole number held as an int. """ + if is_number(value) and isinstance(value, float) and is_whole(value): + return int(value) + if is_list(value): + return [_plain(item) for item in value] + if is_mapping(value): + return {key: _plain(item) for key, item in value.items()} + return value + + +def js_json(value, indent=None): + """! + A value as JavaScript's JSON.stringify() gives it, with no spaces + unless an indent is given. + """ + if indent is None: + return json.dumps(_plain(value), ensure_ascii=False, + separators=(",", ":")) + return json.dumps(_plain(value), ensure_ascii=False, indent=indent) diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/parse.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/parse.py new file mode 100644 index 00000000..754e6dff --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/parse.py @@ -0,0 +1,235 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +Turns the text of a derived property script into plain Python values. +YAML and JSON are both accepted and give the same values. + +This follows tools/parse.mjs of derived-properties, which reads YAML with +the YAML core schema. PyYAML follows YAML 1.1, where yes, no, on and off +are booleans and a date is a date, so the loader here keeps only the core +schema's null, boolean, integer and float forms, and every other plain +value is a string. Mapping keys are strings, as they are in JavaScript, +and a key written twice is refused. + +Where the text is YAML the parser also records the line each mapping and +each sequence started on, so a validation fault can point at a line. +""" + +import json +import re + +import yaml +from yaml.constructor import ConstructorError +from yaml.nodes import MappingNode, SequenceNode + +from .jsvalues import js_string + + +class ParseError(Exception): + """! Raised when the text is not YAML or JSON, or is not a mapping. """ + + def __init__(self, message, line=None): + """! + @param message: what is wrong with the text. + @param line: the one based line, or None where it is not known. + """ + super().__init__(message) + self.message = message + self.line = line + + +class ParseResult(object): + """! The values read from a script, and where each one started. """ + + def __init__(self, format, document, lines): + self.format = format + self.document = document + # The one based line of each mapping and list, by id. + self._lines = lines + + def line_of(self, node): + """! + The one based line a mapping or list in the document started on, + or None when it is not known. + """ + if not isinstance(node, (dict, list)): + return None + entry = self._lines.get(id(node)) + return None if entry is None else entry[1] + + +def parse(text): + """! + Reads script text. + + @param text: the script, as YAML or as JSON. + @rtype: ParseResult + """ + if not isinstance(text, str): + raise ParseError("script text must be a string") + if re.match(r"^\s*\{", text): + return _parse_json(text) + return _parse_yaml(text) + + +def line_of(result, node): + """! The one based line a mapping or list started on, or None. """ + if result is None: + return None + return result.line_of(node) + + +def _parse_json(text): + try: + # Where a key is written twice the last one is kept, as + # JavaScript's JSON.parse keeps it. + document = json.loads(text) + except ValueError as error: + raise ParseError("the text is not valid JSON: " + str(error)) + _require_mapping(document) + lines = {} + _require_distinct_keys(document, lines) + return ParseResult("json", document, lines) + + +class _CoreLoader(yaml.SafeLoader): + """! A loader reading the YAML core schema, recording lines. """ + + yaml_implicit_resolvers = {} + + def __init__(self, stream): + super().__init__(stream) + # The value of each id is held with its line so the id cannot be + # given to another value while the document is read. + self.lines = {} + + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError( + None, None, "expected a mapping", node.start_mark) + mapping = {} + for key_node, value_node in node.value: + key = js_string(self.construct_object(key_node, deep=True)) + if key in mapping: + raise ConstructorError( + None, None, "duplicated mapping key", + key_node.start_mark) + mapping[key] = self.construct_object(value_node, deep=deep) + return mapping + + def construct_core_map(self, node): + data = {} + self.lines[id(data)] = (data, node.start_mark.line + 1) + yield data + data.update(self.construct_mapping(node)) + + def construct_core_seq(self, node): + data = [] + self.lines[id(data)] = (data, node.start_mark.line + 1) + yield data + data.extend(self.construct_sequence(node)) + + def construct_core_int(self, node): + value = self.construct_scalar(node) + if value.startswith("0o"): + return int(value[2:], 8) + if value.startswith("0x"): + return int(value[2:], 16) + return int(value, 10) + + +_CoreLoader.add_implicit_resolver( + "tag:yaml.org,2002:null", + re.compile(r"^(?:~|null|Null|NULL|)$"), + ["~", "n", "N", ""]) +_CoreLoader.add_implicit_resolver( + "tag:yaml.org,2002:bool", + re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$"), + list("tTfF")) +_CoreLoader.add_implicit_resolver( + "tag:yaml.org,2002:int", + re.compile(r"^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$"), + list("-+0123456789")) +_CoreLoader.add_implicit_resolver( + "tag:yaml.org,2002:float", + re.compile(r"^(?:[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)" + r"(?:[eE][-+]?[0-9]+)?" + r"|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$"), + list("-+0123456789.")) +_CoreLoader.add_constructor( + "tag:yaml.org,2002:map", _CoreLoader.construct_core_map) +_CoreLoader.add_constructor( + "tag:yaml.org,2002:seq", _CoreLoader.construct_core_seq) +_CoreLoader.add_constructor( + "tag:yaml.org,2002:int", _CoreLoader.construct_core_int) + + +def _parse_yaml(text): + loader = _CoreLoader(text) + try: + document = loader.get_single_data() + except yaml.YAMLError as error: + mark = getattr(error, "problem_mark", None) + line = mark.line + 1 if mark is not None else None + reason = getattr(error, "problem", None) or str(error) + raise ParseError("the text is not valid YAML: " + reason, line) + finally: + loader.dispose() + _require_mapping(document) + _require_distinct_keys(document, loader.lines) + return ParseResult("yaml", document, loader.lines) + + +def _require_distinct_keys(node, lines): + """! + Every key in the format is matched without regard to case, so two + keys in one mapping that differ only in case are one key written + twice, and both are refused. + """ + if isinstance(node, list): + for item in node: + _require_distinct_keys(item, lines) + return + if not isinstance(node, dict): + return + seen = {} + for key in node.keys(): + lower = key.lower() + if lower in seen: + entry = lines.get(id(node)) + raise ParseError( + "the keys '" + seen[lower] + "' and '" + key + "' differ " + "only in case, and keys are matched without regard to " + "case, so one of the two would be dropped", + None if entry is None else entry[1]) + seen[lower] = key + _require_distinct_keys(node[key], lines) + + +def _require_mapping(document): + if document is None: + raise ParseError("the script is empty") + if not isinstance(document, dict): + raise ParseError( + "the script must be a mapping of keys to values at the top " + "level") diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/script_set.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/script_set.py new file mode 100644 index 00000000..fb48724d --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/script_set.py @@ -0,0 +1,176 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +Collects scripts from the three places a script can come from, being the +scripts built into this package, files, and text held by code, along with +every fault found in any of them. +""" + +import fnmatch +import os +import re + +from .errors import ScriptValidationError +from .validate import Fault, validate_text + +## The scripts built into this package, which are the scripts folder of the +# derived-properties submodule. The package carries that folder and nothing +# else from the submodule. +BUILT_IN_FOLDER = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "derived-properties", + "scripts") + +_SCRIPT_FILE = re.compile(r"\.(ya?ml|json)$", re.IGNORECASE) + + +def _built_in_files(): + if not os.path.isdir(BUILT_IN_FOLDER): + return None + return sorted(f for f in os.listdir(BUILT_IN_FOLDER) + if _SCRIPT_FILE.search(f)) + + +def _name_of(path): + return os.path.splitext(os.path.basename(path))[0] + + +def builtin_script_names(): + """! + The names of the scripts built into this package. + + @return: the names, such as HumanConfidence and IsGdpr. + """ + files = _built_in_files() + return [] if files is None else [_name_of(f) for f in files] + + +class ScriptSet(object): + """! The scripts read so far, and every fault found in them. """ + + def __init__(self): + self.scripts = [] + self.faults = [] + + def add_built_in(self, name): + """! + Adds a script built into this package, by its name in any case. + """ + def fault(message): + self.faults.append(Fault( + name if isinstance(name, str) else None, "built in", "", + None, message)) + + if not isinstance(name, str) or name.strip() == "": + fault("a built in script name is empty") + return + files = _built_in_files() + if files is None: + fault("the built in scripts are not in this copy of the " + "package, because the folder '" + BUILT_IN_FOLDER + + "' does not exist. In a clone of the repository, git " + "submodule update --init fetches them") + return + lower = name.strip().lower() + found = [f for f in files if _name_of(f).lower() == lower] + if not found: + names = [_name_of(f) for f in files] + fault("there is no script called '" + name.strip() + "' in " + "this package. The scripts in this package are " + + (", ".join(names) if names else "(none)") + + ". A script from your own environment is added with " + "script_files rather than scripts") + return + script_name = _name_of(found[0]) + with open(os.path.join(BUILT_IN_FOLDER, found[0]), + encoding="utf-8") as file: + text = file.read() + self.add_text(script_name, text, "built in " + script_name) + + def add_files(self, path): + """! + Adds a script file, or every file a wildcard in the file name + matches, in order of their names. A relative path is read from the + current folder. + """ + if not isinstance(path, str) or path.strip() == "": + self.faults.append(Fault( + None, "file", "", None, "a script file path is empty")) + return + trimmed = path.strip() + if "*" not in trimmed and "?" not in trimmed: + self.add_file(trimmed) + return + folder = os.path.dirname(trimmed) or "." + pattern = os.path.basename(trimmed) + if not os.path.isdir(folder): + self.faults.append(Fault( + None, trimmed, "", None, + "the folder '" + folder + "' does not exist")) + return + # Ordered so that a wildcard gives the same element on every + # machine, whatever order the file system lists in. + for file in sorted(os.listdir(folder)): + full = os.path.join(folder, file) + if fnmatch.fnmatch(file, pattern) and os.path.isfile(full): + self.add_file(full) + + def add_file(self, path): + """! + Adds one script file. The script's Name must equal the file name + without its extension. + """ + try: + with open(path, encoding="utf-8-sig") as file: + text = file.read() + except OSError as error: + self.faults.append(Fault( + _name_of(path), path, "", None, + "the file could not be read. " + str(error))) + return + self.add_text(_name_of(path), text, path) + + def add_text(self, name, text, source="code"): + """! + Adds the text of a script, as YAML or JSON. + + @param name: the name the script's Name must equal, or None for no + such check. + @param text: the script. + @param source: where the script came from, for the log and the + fault messages. + """ + model, faults = validate_text(text, name, source) + if model is None: + self.faults.extend(faults) + return + self.scripts.append(model) + + def compile(self): + """! + The scripts, once every one has validated. + + @raise ScriptValidationError: where any script did not validate. + """ + if self.faults: + raise ScriptValidationError(self.faults) + return self.scripts diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/validate.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/validate.py new file mode 100644 index 00000000..1e2682e1 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/validate.py @@ -0,0 +1,1010 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +Checks a parsed script against format 1 and turns it into the model the +evaluator runs. Every fault is collected rather than stopping at the +first, so an author sees everything wrong with a file in one go. + +Each fault has a path, being a place in the document such as +Rules[3].When.All[1]. + +This follows tools/validate.mjs of derived-properties function for +function, with the same messages. +""" + +import re +from types import SimpleNamespace + +from .evaluate import convert_string, INT_MIN, INT_MAX +from .jsvalues import ( + is_bool, is_number, is_whole, is_string, is_list, is_mapping, + js_string, js_json) +from .parse import parse, line_of, ParseError + +## The value types format 1 allows for an output property. +VALUE_TYPES = ["string", "bool", "int", "double"] + +## The comparison operators, and the inferred types each is allowed on. +OPERATORS = { + "Eq": ["bool", "int", "double", "string"], + "Ne": ["bool", "int", "double", "string"], + "Gt": ["int", "double"], + "Ge": ["int", "double"], + "Lt": ["int", "double"], + "Le": ["int", "double"], + "In": ["bool", "int", "double", "string"], + "NotIn": ["bool", "int", "double", "string"], + "StartsWith": ["string"], + "EndsWith": ["string"], + "Contains": ["string"] +} + +## The two counts a rule can compare a group of checks by. +AGGREGATES = ["Passed", "Failed"] + +_TOP_LEVEL_KEYS = ["Format", "Name", "Version", "Deprecated", + "DeprecationNote", "Output", "Checks", "Rules"] + +## The Output keys, in the order the canonical form prints them. +OUTPUT_KEYS = ["Name", "Description", "ValueType", "StoredValueType", + "DefaultValue", "IsList", "IsMandatory", "IsObsolete", + "Category", "IsPopular", "ExportValues", "Url", + "DisplayOrder", "PropertyId", "VendorIds", "Dependencies", + "Values"] + +_OUTPUT_BOOLEANS = ["IsList", "IsMandatory", "IsObsolete", "IsPopular", + "ExportValues"] + +_OUTPUT_CARRIED = [ + ("Category", "string"), + ("Url", "string"), + ("StoredValueType", "string"), + ("DisplayOrder", "integer"), + ("PropertyId", "integer"), + ("VendorIds", "array"), + ("Dependencies", "array") +] + +_IDENTIFIER = "^[A-Za-z][A-Za-z0-9]*$" +_IDENTIFIER_RE = re.compile(r"^[A-Za-z][A-Za-z0-9]*\Z") +_SOURCE_PROPERTY = re.compile( + r"^[A-Za-z][A-Za-z0-9]*\.[A-Za-z][A-Za-z0-9]*\Z") +_SEMANTIC_VERSION = re.compile( + r"^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+(?:\.[0-9A-Za-z.-]+)*)?" + r"(?:\+[0-9A-Za-z.-]+)?\Z") + + +class Fault(object): + """! One thing wrong with a script, and where it is. """ + + def __init__(self, script, source, path, line, message): + self.script = script + self.source = source + self.path = path + self.line = line + self.message = message + + def __repr__(self): + return faults_message([self]) + + +class Model(object): + """! A script once it has been read and validated. """ + + def __init__(self, source): + self.format = None + self.name = None + self.version = None + self.deprecated = False + self.deprecation_note = None + self.source = source + self.output = None + self.properties = [] + self.checks = [] + self.rules = [] + + +def validate_text(text, name=None, source="code"): + """! + Parses and validates script text. + + @param text: the script as YAML or JSON. + @param name: the file name without its extension, which the script's + Name must equal, or None for no such check. + @param source: where the script came from, for the fault messages. + @return: (model, faults), where model is None when there are faults. + """ + try: + parsed = parse(text) + except ParseError as error: + return (None, [Fault(name, source, "", error.line, error.message)]) + return validate(parsed, name, source) + + +def validate(parsed, name=None, source="code"): + """! + Validates an already parsed script. + + @return: (model, faults), where model is None when there are faults. + """ + context = _Context(parsed, name, source) + model = _build_model(context) + if context.faults: + return (None, context.faults) + return (model, []) + + +def faults_message(faults): + """! + The message an implementation raises when a script does not + validate, being one line per fault. + """ + lines = [] + for f in faults: + where = "(document)" if f.path == "" else f.path + line = "" if f.line is None else " line " + str(f.line) + lines.append( + ("script" if f.script is None else f.script) + + " (" + f.source + ")" + line + " at " + where + ": " + + f.message) + return "\n".join(lines) + + +# --------------------------------------------------------------------- + +class _Context(object): + def __init__(self, parsed, name, source): + self.parsed = parsed + self.document = parsed.document + self.script = name + self.source = source + self.faults = [] + # Where each source property was first seen, its inferred type + # and the path that inferred it, so a conflict can name both + # places. + self.properties = {} + + def fault(self, path, node, message): + self.faults.append(Fault( + self.script, self.source, path, + line_of(self.parsed, node), message)) + + +def _get(mapping, name): + """! Reads a key from a mapping without regard to case. """ + if not is_mapping(mapping): + return None + lower = name.lower() + for key in mapping.keys(): + if key.lower() == lower: + return mapping[key] + return None + + +def _found(mapping, name): + """! Whether a key is written, whatever its value. """ + if not is_mapping(mapping): + return False + lower = name.lower() + return any(key.lower() == lower for key in mapping.keys()) + + +def _unknown_keys(mapping, allowed): + lower = [k.lower() for k in allowed] + return [k for k in mapping.keys() if k.lower() not in lower] + + +def _build_model(context): + document = context.document + + for key in _unknown_keys(document, _TOP_LEVEL_KEYS): + context.fault(key, document, + "unknown key '" + key + "' at the top level. " + "Expected one of " + ", ".join(_TOP_LEVEL_KEYS)) + + model = Model(context.source) + model.format = _read_format(context) + model.name = _read_name(context) + model.version = _read_version(context) + + _read_deprecation(context, model) + model.output = _read_output(context) + + # Checks are read before rules so that a rule can reference one. + check_names = _read_check_names(context) + _read_checks(context, model, check_names) + _read_rules(context, model, check_names) + _finalise_properties(model) + _compute_dependencies(model) + + return model + + +# The reference tools tell a key that is not written (undefined) from one +# written as null. Python reads both as None, so where the difference +# changes the message, _found says whether the key was written. + +def _read_format(context): + if not _found(context.document, "Format"): + context.fault("Format", context.document, + "required key 'Format' is missing") + return None + value = _get(context.document, "Format") + if is_bool(value) or value != 1: + context.fault("Format", context.document, + "Format must be 1, found " + js_json(value)) + return None + return 1 + + +def _read_name(context): + if not _found(context.document, "Name"): + context.fault("Name", context.document, + "required key 'Name' is missing") + return None + value = _get(context.document, "Name") + if not is_string(value): + context.fault("Name", context.document, + "Name expected a string, found " + _type_of(value)) + return None + if not _IDENTIFIER_RE.match(value): + context.fault("Name", context.document, + "script name '" + value + "' does not match the " + "pattern " + _IDENTIFIER) + return value + if context.script is not None and context.script != value: + context.fault("Name", context.document, + "script name '" + value + "' must equal the file " + "name '" + context.script + "'") + return value + + +def _read_version(context): + if not _found(context.document, "Version"): + context.fault("Version", context.document, + "required key 'Version' is missing") + return None + value = _get(context.document, "Version") + if not is_string(value) or not _SEMANTIC_VERSION.match(value): + context.fault("Version", context.document, + "Version expected a semantic version such as 1.0.0, " + "found " + js_json(value)) + return None + return value + + +def _read_deprecation(context, model): + if _found(context.document, "Deprecated"): + deprecated = _get(context.document, "Deprecated") + if not is_bool(deprecated): + context.fault("Deprecated", context.document, + "Deprecated expected a boolean, found " + + _type_of(deprecated)) + else: + model.deprecated = deprecated + if _found(context.document, "DeprecationNote"): + note = _get(context.document, "DeprecationNote") + if not is_string(note): + context.fault("DeprecationNote", context.document, + "DeprecationNote expected a string, found " + + _type_of(note)) + else: + model.deprecation_note = note + if model.deprecated and not model.deprecation_note: + context.fault("DeprecationNote", context.document, + "a deprecated script must say what to use instead " + "in DeprecationNote") + if not model.deprecated and model.deprecation_note: + context.fault("DeprecationNote", context.document, + "DeprecationNote is only allowed when Deprecated is " + "true") + + +def _read_output(context): + unknown = context.script if context.script is not None else "unknown" + if not _found(context.document, "Output"): + context.fault("Output", context.document, + "required key 'Output' is missing") + return {"Name": unknown, "ValueType": None, "Values": None} + raw = _get(context.document, "Output") + if not is_mapping(raw): + context.fault("Output", context.document, + "Output expected a mapping, found " + _type_of(raw)) + return {"Name": unknown, "ValueType": None, "Values": None} + for key in _unknown_keys(raw, OUTPUT_KEYS): + context.fault("Output." + key, raw, + "unknown key '" + key + "' under Output. A typo in a " + "metadata field is a fault rather than a value that " + "is quietly dropped") + + output = {} + + # Name. + if not _found(raw, "Name"): + context.fault("Output.Name", raw, "required key 'Name' is missing") + else: + name = _get(raw, "Name") + if not is_string(name) or not _IDENTIFIER_RE.match(name): + context.fault("Output.Name", raw, + "Output.Name " + js_json(name) + " does not " + "match the pattern " + _IDENTIFIER) + else: + output["Name"] = name + if "Name" not in output: + output["Name"] = unknown + + # Description. + if not _found(raw, "Description"): + context.fault("Output.Description", raw, + "required key 'Description' is missing. Say what the " + "property asserts, not how far to trust it") + else: + description = _get(raw, "Description") + if not is_string(description) or description.strip() == "": + context.fault("Output.Description", raw, + "Output.Description expected a non empty string, " + "found " + _type_of(description)) + else: + output["Description"] = description + + # ValueType. + if not _found(raw, "ValueType"): + context.fault("Output.ValueType", raw, + "required key 'ValueType' is missing") + output["ValueType"] = None + else: + value_type = _get(raw, "ValueType") + if (not is_string(value_type) or + value_type.lower() not in VALUE_TYPES): + context.fault("Output.ValueType", raw, + "Output.ValueType '" + js_string(value_type) + + "' is not allowed in format 1. Expected one of " + + ", ".join(VALUE_TYPES)) + output["ValueType"] = None + else: + output["ValueType"] = value_type.lower() + + # IsList. + if not _found(raw, "IsList"): + context.fault("Output.IsList", raw, + "required key 'IsList' is missing") + else: + is_list_value = _get(raw, "IsList") + if not is_bool(is_list_value): + context.fault("Output.IsList", raw, + "Output.IsList expected a boolean, found " + + _type_of(is_list_value)) + elif is_list_value is True: + context.fault("Output.IsList", raw, + "Output.IsList must be false in format 1. List " + "outputs are deferred") + else: + output["IsList"] = False + if "IsList" not in output: + output["IsList"] = False + + # Values. + output["Values"] = None + if _found(raw, "Values"): + values = _get(raw, "Values") + if not is_list(values): + context.fault("Output.Values", raw, + "Output.Values expected a list, found " + + _type_of(values)) + elif (output["ValueType"] is not None and + output["ValueType"] != "string" and + output["ValueType"] != "int"): + context.fault("Output.Values", raw, + "Output.Values is only allowed where ValueType is " + "string or int, not " + output["ValueType"]) + else: + output["Values"] = _read_values(context, values) + + # Booleans carried through unchanged. + for key in _OUTPUT_BOOLEANS: + if key == "IsList" or not _found(raw, key): + continue + value = _get(raw, key) + if not is_bool(value): + context.fault("Output." + key, raw, + "Output." + key + " expected a boolean, found " + + _type_of(value)) + continue + output[key] = value + + # Strings and the remaining fields carried through unchanged. + for key, kind in _OUTPUT_CARRIED: + if not _found(raw, key): + continue + value = _get(raw, key) + if kind == "string" and not is_string(value): + context.fault("Output." + key, raw, + "Output." + key + " expected a string, found " + + _type_of(value)) + continue + if kind == "integer" and not is_whole(value): + context.fault("Output." + key, raw, + "Output." + key + " expected an integer, found " + + _type_of(value)) + continue + if kind == "array" and not is_list(value): + context.fault("Output." + key, raw, + "Output." + key + " expected a list, found " + + _type_of(value)) + continue + output[key] = value + + # DefaultValue is the string form of the value, so it is checked + # against the value type and the value list. + if _found(raw, "DefaultValue"): + default_value = _get(raw, "DefaultValue") + if not is_string(default_value): + context.fault("Output.DefaultValue", raw, + "Output.DefaultValue expected a string holding " + "the string form of the value, found " + + _type_of(default_value)) + else: + output["DefaultValue"] = default_value + if output["ValueType"] is not None: + if not convert_string(default_value, + output["ValueType"])[0]: + context.fault("Output.DefaultValue", raw, + "Output.DefaultValue '" + default_value + + "' cannot be read as " + + output["ValueType"]) + if (output["Values"] is not None and + not any(js_string(v["Name"]) == default_value + for v in output["Values"])): + context.fault("Output.DefaultValue", raw, + "Output.DefaultValue '" + default_value + + "' is not one of the values listed under " + "Output.Values") + + return output + + +def _read_values(context, values): + result = [] + for index, entry in enumerate(values): + path = "Output.Values[" + str(index) + "]" + if not is_mapping(entry): + context.fault(path, values, + "a value expected a mapping of Name and " + "Description, found " + _type_of(entry)) + continue + for key in _unknown_keys(entry, ["Name", "Description"]): + context.fault(path + "." + key, entry, + "unknown key '" + key + "' in a value. Expected " + "Name and Description") + value_name = _get(entry, "Name") + if not (is_string(value_name) or is_number(value_name)): + context.fault(path + ".Name", entry, "a value must have a Name") + continue + value_description = _get(entry, "Description") + if (_found(entry, "Description") and + not is_string(value_description)): + context.fault(path + ".Description", entry, + "a value Description expected a string, found " + + _type_of(value_description)) + result.append({ + "Name": value_name, + "Description": value_description + if is_string(value_description) else None + }) + seen = set() + for entry in result: + key = js_string(entry["Name"]) + if key in seen: + context.fault("Output.Values", values, + "the value '" + key + "' is listed more than once") + seen.add(key) + return result + + +def _read_check_names(context): + names = [] + if not _found(context.document, "Checks"): + return names + raw = _get(context.document, "Checks") + if not is_mapping(raw): + context.fault("Checks", context.document, + "Checks expected a mapping of names to conditions, " + "found " + _type_of(raw)) + return names + for name in raw.keys(): + if not _IDENTIFIER_RE.match(name): + context.fault("Checks." + name, raw, + "check name '" + name + "' does not match the " + "pattern " + _IDENTIFIER) + continue + names.append(name) + return names + + +def _read_checks(context, model, check_names): + raw = _get(context.document, "Checks") + if not is_mapping(raw): + return + for name in check_names: + condition = _read_condition( + context, model, raw[name], "Checks." + name, raw, check_names, + name in raw) + model.checks.append(SimpleNamespace(name=name, condition=condition)) + + +def _read_rules(context, model, check_names): + if not _found(context.document, "Rules"): + context.fault("Rules", context.document, + "required key 'Rules' is missing") + return + raw = _get(context.document, "Rules") + if not is_list(raw): + context.fault("Rules", context.document, + "Rules expected a list, found " + _type_of(raw)) + return + if len(raw) == 0: + context.fault("Rules", raw, "Rules must hold at least one rule") + return + # Every script ends in an Else, so a script always chooses a value + # once its source properties have been read. + if not _found(raw[-1], "Else"): + context.fault("Rules", raw, + "the last rule must be an Else, which is what a " + "script falls back to when no earlier rule matched") + for index, entry in enumerate(raw): + path = "Rules[" + str(index) + "]" + if not is_mapping(entry): + context.fault(path, raw, + "a rule expected a mapping, found " + + _type_of(entry)) + continue + for key in _unknown_keys(entry, ["When", "Then", "Else"]): + context.fault(path + "." + key, entry, + "unknown key '" + key + "' in a rule. Expected " + "When and Then, or Else") + has_when = _found(entry, "When") + has_then = _found(entry, "Then") + has_else = _found(entry, "Else") + is_last = index == len(raw) - 1 + + if has_else and has_when: + context.fault(path, entry, + "a rule has both When and Else. A rule is either " + "When with Then, or Else on its own") + continue + if has_else and not is_last: + context.fault(path, entry, + "Else is only allowed on the last rule") + continue + if not has_else and not has_when: + context.fault(path, entry, + "a rule needs a When, or an Else on the last rule") + continue + if has_when and not has_then: + context.fault(path, entry, "a rule with When needs a Then") + continue + + value_path = path + (".Else" if has_else else ".Then") + value = _read_rule_value( + context, model, + _get(entry, "Else") if has_else else _get(entry, "Then"), + value_path, entry) + when = None + if has_when: + when = _read_condition(context, model, _get(entry, "When"), + path + ".When", entry, check_names, True) + model.rules.append(SimpleNamespace(when=when, value=value)) + + +def _read_rule_value(context, model, raw, path, node): + """! + Reads a Then or an Else, which is a literal of Output.ValueType and + must be one of Output.Values where that list is given. + """ + value_type = model.output["ValueType"] + if raw is None: + context.fault(path, node, + "a rule value is a null literal, which format 1 does " + "not allow") + return None + if is_list(raw) or is_mapping(raw): + context.fault(path, node, + "a rule value is a literal of the output value type, " + "found " + _type_of(raw)) + return None + if value_type is not None: + literal_type = infer_type(raw) + if not _literal_matches(literal_type, value_type): + context.fault(path, node, + "expected a " + value_type + " to match " + "Output.ValueType, found " + _type_of(raw)) + return None + values = model.output["Values"] + if values is not None and not any( + js_string(v["Name"]) == js_string(raw) for v in values): + context.fault(path, node, + "'" + js_string(raw) + "' is not one of the values " + "listed under Output.Values (" + + ", ".join(js_string(v["Name"]) for v in values) + ")") + return None + return raw + + +# --------------------------------------------------------------------- +# Conditions. +# --------------------------------------------------------------------- + +def _faulted(): + """! + What a condition that could not be read becomes. A fault has been + recorded by the time it is returned, so the model is thrown away and + it is never evaluated. + """ + return SimpleNamespace(kind="all", items=[]) + + +def _read_condition(context, model, raw, path, parent, check_names, + written=True): + if not is_mapping(raw): + context.fault(path, parent, + "a condition expected a mapping, found " + + (_type_of(raw) if written else "nothing")) + return _faulted() + keys = list(raw.keys()) + if len(keys) == 0: + context.fault(path, raw, "a condition is empty") + return _faulted() + lower = [k.lower() for k in keys] + + if "property" in lower: + return _read_comparison(context, model, raw, path) + if "check" in lower: + return _read_check_reference(context, raw, path, keys, check_names) + if any(a.lower() in lower for a in AGGREGATES): + return _read_aggregate_condition(context, raw, path, keys, + check_names) + if "all" in lower or "any" in lower: + which = "All" if "all" in lower else "Any" + if len(keys) != 1: + context.fault(path, raw, + which + " must be the only key of its condition, " + "found " + ", ".join(keys)) + items = _get(raw, which) + if not is_list(items): + context.fault(path + "." + which, raw, + which + " expected a list of conditions, found " + + _type_of(items)) + return _faulted() + if len(items) == 0: + context.fault(path + "." + which, raw, + which + " must list at least one condition") + return _faulted() + return SimpleNamespace( + kind=which.lower(), + items=[_read_condition(context, model, item, + path + "." + which + "[" + str(index) + + "]", items, check_names) + for index, item in enumerate(items)]) + if "not" in lower: + if len(keys) != 1: + context.fault(path, raw, + "Not must be the only key of its condition, " + "found " + ", ".join(keys)) + return SimpleNamespace( + kind="not", + item=_read_condition(context, model, _get(raw, "Not"), + path + ".Not", raw, check_names, + _found(raw, "Not"))) + context.fault(path, raw, + "a condition must be a comparison, a Check reference, an " + "aggregate, All, Any or Not. Found the keys " + + ", ".join(keys)) + return _faulted() + + +def _operator_named(key): + for operator in OPERATORS: + if operator.lower() == key.lower(): + return operator + return None + + +def _read_comparison(context, model, raw, path): + keys = list(raw.keys()) + property_key = next(k for k in keys if k.lower() == "property") + prop = raw[property_key] + operator_keys = [k for k in keys if k != property_key] + + if not is_string(prop) or not _SOURCE_PROPERTY.match(prop): + context.fault(path + ".Property", raw, + "'" + js_string(prop) + "' is not a source property. " + "Write it as elementKey.PropertyName, for example " + "device.IsCrawler") + return _faulted() + if len(operator_keys) == 0: + context.fault(path, raw, + "a comparison on '" + prop + "' has no operator. " + "Expected exactly one of " + ", ".join(OPERATORS)) + return _faulted() + known = [k for k in operator_keys if _operator_named(k) is not None] + for key in operator_keys: + if key not in known: + context.fault(path, raw, + "unknown operator '" + key + "', expected one of " + + ", ".join(OPERATORS)) + if len(known) == 0: + return _faulted() + if len(operator_keys) > 1: + context.fault(path, raw, + "a condition takes exactly one operator, found " + + ", ".join(operator_keys)) + return _faulted() + op = _operator_named(known[0]) + operand = raw[known[0]] + + if operand is None: + context.fault(path + "." + op, raw, + "a null literal is not allowed. Give the value to " + "compare against") + return _faulted() + + if op == "In" or op == "NotIn": + if not is_list(operand): + context.fault(path + "." + op, raw, + op + " expects a list of values, found " + + _type_of(operand)) + return _faulted() + if len(operand) == 0: + context.fault(path + "." + op, raw, + op + " expects a non empty list") + return _faulted() + if any(m is None for m in operand): + context.fault(path + "." + op, raw, + "a null literal is not allowed in a list") + return _faulted() + member_types = [infer_type(m) for m in operand] + first = member_types[0] + # int and double may sit together and the list reads as double. + numeric = all(t == "int" or t == "double" for t in member_types) + if not numeric and any(t != first for t in member_types): + context.fault(path + "." + op, raw, + "every member of a list must be of the same type, " + "found " + + ", ".join("" if t is None else t + for t in member_types)) + return _faulted() + if not numeric: + type = first + elif "double" in member_types: + type = "double" + else: + type = "int" + else: + type = infer_type(operand) + if type is None: + context.fault(path + "." + op, raw, + "the literal " + js_json(operand) + " has no type " + "format 1 knows") + return _faulted() + + if type not in OPERATORS[op]: + context.fault(path, raw, + "operator '" + op + "' is not allowed on type " + + js_string(type) + ". It is allowed on " + + ", ".join(OPERATORS[op])) + return _faulted() + + slot = _use_property(context, model, prop, type, path, raw) + return SimpleNamespace( + kind="compare", slot=slot, property=prop.lower(), + propertyName=prop, op=op, operand=operand, type=type) + + +def _read_check_reference(context, raw, path, keys, check_names): + if len(keys) != 1: + context.fault(path, raw, + "Check must be the only key of its condition, found " + + ", ".join(keys)) + name = _get(raw, "Check") + if not is_string(name): + context.fault(path + ".Check", raw, + "Check expected the name of a check, found " + + _type_of(name)) + return _faulted() + if name not in check_names: + context.fault(path + ".Check", raw, + "check '" + name + "' is not defined. The checks are " + + ("(none)" if len(check_names) == 0 + else ", ".join(check_names))) + return _faulted() + return SimpleNamespace(kind="check", name=name, + index=check_names.index(name)) + + +def _read_aggregate_condition(context, raw, path, keys, check_names): + aggregate_keys = [k for k in keys + if any(a.lower() == k.lower() for a in AGGREGATES)] + if len(aggregate_keys) > 1: + context.fault(path, raw, + "an aggregate condition takes one of " + + ", ".join(AGGREGATES) + ", found " + + ", ".join(aggregate_keys)) + return _faulted() + agg = next(a for a in AGGREGATES + if a.lower() == aggregate_keys[0].lower()) + group = _read_group(context, raw[aggregate_keys[0]], path + "." + agg, + raw, check_names) + + operator_keys = [k for k in keys if k != aggregate_keys[0]] + if len(operator_keys) == 0: + context.fault(path, raw, + "an aggregate condition has no operator. Expected " + "exactly one of Eq, Ne, Gt, Ge, Lt, Le") + return _faulted() + if len(operator_keys) > 1: + context.fault(path, raw, + "a condition takes exactly one operator, found " + + ", ".join(operator_keys)) + return _faulted() + op = _operator_named(operator_keys[0]) + if op is None: + context.fault(path, raw, + "unknown operator '" + operator_keys[0] + "', expected " + "one of Eq, Ne, Gt, Ge, Lt, Le") + return _faulted() + if "int" not in OPERATORS[op]: + context.fault(path, raw, + "operator '" + op + "' is not allowed on a count, " + "which is an int") + return _faulted() + + # A count is compared against a whole number and against nothing + # else. + operand = raw[operator_keys[0]] + if not is_whole(operand): + context.fault(path + "." + op, raw, + "an aggregate is compared with a whole number, " + "found " + _type_of(operand)) + return _faulted() + + return SimpleNamespace(kind="aggregate", agg=agg, group=group, op=op, + operand=operand) + + +def _read_group(context, raw, path, node, check_names): + """! + A group is the word Checks, meaning every named check, or a list of + check names. It becomes a list of indexes, or None for every check. + """ + if is_string(raw): + if raw.lower() == "checks": + return None + context.fault(path, node, + "a group is the word Checks, meaning every check, or " + "a list of check names. Found '" + raw + "'") + return [] + if is_list(raw): + indexes = [] + for name in raw: + if not is_string(name): + context.fault(path, node, + "a group lists check names, found " + + _type_of(name)) + continue + if name not in check_names: + context.fault(path, node, + "check '" + name + "' is not defined. The " + "checks are " + + ("(none)" if len(check_names) == 0 + else ", ".join(check_names))) + continue + indexes.append(check_names.index(name)) + return indexes + context.fault(path, node, + "a group is the word Checks or a list of check names, " + "found " + _type_of(raw)) + return [] + + +# --------------------------------------------------------------------- +# Source properties. +# --------------------------------------------------------------------- + +def _use_property(context, model, name, type, path, node): + """! + Records that a condition names a source property, and that the + literal it is compared against infers a type. Returns the slot index + the evaluator reads the property from. + """ + key = name.lower() + entry = context.properties.get(key) + if entry is None: + entry = SimpleNamespace(key=key, name=name, type=type, + type_path=path, slot=len(model.properties)) + context.properties[key] = entry + model.properties.append(entry) + return entry.slot + if entry.type != type: + context.fault(path, node, + "'" + name + "' is inferred as " + type + " here but " + "was already inferred as " + entry.type + " at " + + entry.type_path + ". Every use of a property must " + "infer the same type") + return entry.slot + + +def _finalise_properties(model): + """! The model exposes the properties without the bookkeeping. """ + model.properties = [SimpleNamespace(key=p.key, name=p.name, type=p.type) + for p in model.properties] + + +def _compute_dependencies(model): + if model.output.get("Dependencies") is not None: + return + model.output["Dependencies"] = [p.name for p in model.properties] + + +# --------------------------------------------------------------------- +# Literals. +# --------------------------------------------------------------------- + +def infer_type(literal): + """! + The type a literal in the script infers. A number infers its type + from the value rather than from the way the value was written, so 8, + 8.0 and 8e0 all infer int. A whole number too large for a signed 32 + bit integer infers double instead. + """ + if is_bool(literal): + return "bool" + if is_number(literal): + if not is_whole(literal): + return "double" + return "int" if INT_MIN <= literal <= INT_MAX else "double" + if is_string(literal): + return "string" + return None + + +def _literal_matches(literal_type, value_type): + if literal_type == value_type: + return True + # A whole number written without a decimal point reads as a double. + return literal_type == "int" and value_type == "double" + + +def _type_of(value): + if value is None: + return "a null literal" + if is_list(value): + return "a list" + if is_bool(value): + return "a boolean" + if is_number(value): + return "an integer" if is_whole(value) else "a number" + if is_string(value): + return "a string (" + js_json(value) + ")" + if is_mapping(value): + return "a mapping" + return type(value).__name__ diff --git a/pipeline/readme.md b/pipeline/readme.md index e873cbdd..eb27cefc 100644 --- a/pipeline/readme.md +++ b/pipeline/readme.md @@ -18,8 +18,9 @@ This repository contains the following modules: - **fiftyone_pipeline_cloudrequestengine** - An engine used to make requests to the 51Degrees cloud service. - **fiftyone_pipeline_translation** - A flow element that translates values from a source element into another language using YAML translation files. - **fiftyone_pipeline_did** - A reader and cloud client for the 51Did (51Degrees Identifier) returned by the 51Degrees cloud service. +- **fiftyone_pipeline_derivedproperty** - A flow element that computes properties such as HumanConfidence and IsGdpr from properties other elements have produced, following the scripts in the derived-properties repository. Not yet published to PyPI. -The examples are described under Examples below. The `owid-python` folder is a git submodule holding the OWID library that `fiftyone_pipeline_did` carries. +The examples are described under Examples below. The `owid-python` folder is a git submodule holding the OWID library that `fiftyone_pipeline_did` carries, and `fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties` is a git submodule of [derived-properties](https://github.com/51Degrees/derived-properties), whose `scripts` folder the package carries and whose `tests` folder its tests run. ## Dependencies