From 1a8c58bccfb905207a99b3aaecb37032bf6ce41c Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Wed, 14 Oct 2026 17:00:44 +0200 Subject: [PATCH 1/2] feat(rdf): add opt-in diff-stable blank-node labels (#3985) --- AGENTS.md | 2 + docs/howtos/collaborative-development.md | 30 +++ packages/linkml/pyproject.toml | 4 + .../linkml/src/linkml/generators/owlgen.py | 14 +- .../linkml/src/linkml/generators/rdfgen.py | 14 +- .../linkml/src/linkml/generators/shaclgen.py | 14 +- .../linkml/src/linkml/generators/shexgen.py | 14 +- packages/linkml_runtime/pyproject.toml | 1 + .../linkml_runtime/utils/rdf_canonicalize.py | 31 ++- pyproject.toml | 1 + tests/conftest.py | 37 ++- tests/linkml/test_scripts/test_diff_stable.py | 45 ++++ .../test_utils/test_rdf_canonicalize.py | 254 +++++++++++++++++- uv.lock | 30 ++- 14 files changed, 481 insertions(+), 10 deletions(-) create mode 100644 tests/linkml/test_scripts/test_diff_stable.py diff --git a/AGENTS.md b/AGENTS.md index 772f7c59ab..c9a178e6c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,8 @@ All commands use `uv run` prefix (e.g., `uv run pytest`). * Do not "fix" issues by changing or weakening test conditions. Try harder, or ask questions if a test fails. * Avoid try/except blocks, these can mask bugs * Failing fast is a good principle +* Optional dependencies are the documented exception to the two rules above. Guard the import, not the spec: `try: import x` / `except ImportError as exc: raise ImportError("x is required. Install with: pip install 'linkml[extra]'") from exc` - see `generators/bigquerygen.py`. `importlib.util.find_spec` raises under the test fixture below, which hides your message. +* Testing that an optional dependency is absent is not a mock test: use the `mock_missing_import` fixture in `tests/conftest.py`. Do not build isolated environments in CI for this. * Follow the DRY principle * Avoid repeating chunks of code, but also avoid premature over-abstraction * Declarative principles are favored diff --git a/docs/howtos/collaborative-development.md b/docs/howtos/collaborative-development.md index 94515df2d1..221337e843 100644 --- a/docs/howtos/collaborative-development.md +++ b/docs/howtos/collaborative-development.md @@ -40,6 +40,36 @@ Some projects may opt to manage their data alongside the schema in the same repo for "registry" style projects, but in other cases the data may be best managed separately in a dedicated store or a repository like Zenodo or Figshare. +(rdf-in-version-control)= +### Keep generated RDF diffs small + +When you track generated RDF in Git, adding a class or property can change many +blank-node labels. The default RDFC-1.0 numbering is repeatable for the same graph, +but an insertion can renumber existing nodes and obscure the actual schema edit. + +The optional `--diff-stable` flag uses +[diffable-rdf](https://github.com/ASCS-eV/diffable-rdf) to reduce this label churn. +In a uv project containing your schema, install LinkML with the extra and generate +SHACL as follows (using a release that includes this option): + +```bash +uv add 'linkml[diff-stable]' +uv run gen-shacl --diff-stable schema.yaml > schema.shacl.ttl +``` + +The flag also works with `gen-owl`, `gen-rdf`, and `gen-shex --format rdf`. +Python callers of `canonicalize_rdf_graph(..., diff_stable=True)` can install +`linkml-runtime[diff-stable]`. The feature is disabled by default; enabling it +changes existing labels once, so keep that regeneration separate from schema edits. + +Labels depend on blank-node neighborhoods. Edits within connected structures or +among symmetric nodes can still affect other labels; minimum diffs are not +guaranteed. Keep tool versions consistent when comparing generated files. +The relabeled output is not the standard RDFC canonical representation. +ShExC, ShEx JSON and instance-data conversion are outside this option's scope. +Non-standard RDF takes the existing fallback path, which warns that it cannot +apply diff-stable labels. + ### Permissively License Your Code and Data (adapted from [O3 guidelines](https://osf.io/vuzt3/)) diff --git a/packages/linkml/pyproject.toml b/packages/linkml/pyproject.toml index 3d8ee1cdf2..9ff6fd1404 100644 --- a/packages/linkml/pyproject.toml +++ b/packages/linkml/pyproject.toml @@ -68,6 +68,9 @@ dependencies = [ # Specifier syntax: https://peps.python.org/pep-0631/ "platformdirs>=4.12.2", ] +[project.optional-dependencies] +diff-stable = ["linkml-runtime[diff-stable]>=1.10.0,<2.0.0"] + [dependency-groups] lint = [ "black >= 24.0.0", @@ -79,6 +82,7 @@ shacl = [ "pyshacl>=0.40.1", ] tests = [ + "linkml-runtime[diff-stable]", { include-group = "lint" }, { include-group = "typing" }, { include-group = "shacl" }, diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 7ba15df672..3f16c4965c 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -122,6 +122,9 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + diff_stable: bool = False + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" + metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -353,7 +356,7 @@ def serialize(self, **kwargs: Any) -> str: """ self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(self.graph, output_format=fmt) + return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: """ @@ -1844,6 +1847,15 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "specified language tag. Element-level in_language overrides this." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 2da1701787..5b8f8dc1c7 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -78,6 +78,9 @@ class RDFGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" + emit_metadata: bool = False context: list[str] = None original_schema: SchemaDefinition = None @@ -89,7 +92,7 @@ def __post_init__(self): def _data(self, g: Graph) -> str: fmt = "turtle" if self.format == "ttl" else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def end_schema(self, output: str | None = None, context: str = None, **_) -> str: gen = JSONLDGenerator( @@ -137,6 +140,15 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str multiple=True, help="JSONLD context file (default: vendored meta.context.jsonld)", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **kwargs): """Generate an RDF representation of a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4731b9f0b8..1804b1204e 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -142,6 +142,9 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + diff_stable: bool = False + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" + emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -196,7 +199,7 @@ def generate_header(self) -> str: def serialize(self, **args) -> str: g = self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def as_graph(self) -> Graph: sv = self.schemaview @@ -929,6 +932,15 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 40a93ffbc9..e74e45225e 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -40,6 +40,9 @@ class ShExGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Reduce blank-node label churn in RDF output. See :ref:`rdf-in-version-control`.""" + shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) shape: Shape | None = None # Current shape being defined @@ -177,7 +180,7 @@ def end_schema(self, output: str | None = None, **_) -> str: g = Graph() g.parse(data=shex, format="json-ld", version="1.1") g.bind("owl", OWL) - shex = canonicalize_rdf_graph(g, output_format="turtle") + shex = canonicalize_rdf_graph(g, output_format="turtle", diff_stable=self.diff_stable) elif self.format == "shex": g = Graph() self.namespaces.load_graph(g) @@ -258,6 +261,15 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: help="If --expand-subproperty-of (default), slots with subproperty_of will generate NodeConstraint " "values containing all slot descendants. Use --no-expand-subproperty-of to disable this behavior.", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Reduce blank-node label churn across edits (--format rdf). See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate a ShEx Schema for a LinkML model""" diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 5f69503d48..d4c0a2f167 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -62,6 +62,7 @@ dev = [ linkml-normalize = "linkml_runtime.processing.referencevalidator:cli" [project.optional-dependencies] +diff-stable = ["diffable-rdf>=0.4.0"] dev = [ "coverage", "requests-cache>=1.3.3", diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 31c5580348..bc06a969b8 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -287,6 +287,7 @@ def _is_safe_prefix_iri(iri: str) -> bool: def canonicalize_rdf_graph( graph: rdflib.Graph, output_format: str = "turtle", + diff_stable: bool = False, ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. @@ -300,8 +301,22 @@ def canonicalize_rdf_graph( :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). + :param diff_stable: Use diffable-rdf labels to reduce blank-node churn across + edits. Requires the ``diff-stable`` extra; disabled by default. The + non-standard RDF fallback warns when it cannot apply these labels. + See https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control :return: Deterministic string serialization of the graph. + :raises ImportError: If diff-stable labels are requested without diffable-rdf. """ + if diff_stable: + try: + from diffable_rdf import wl_relabel_quads + except ImportError as exc: + raise ImportError( + "diff_stable=True requires diffable-rdf. Install " + "'linkml-runtime[diff-stable]' or 'linkml[diff-stable]'." + ) from exc + ox_format = _FORMAT_MAP.get(output_format.lower()) if ox_format is None: warnings.warn( @@ -330,6 +345,16 @@ def canonicalize_rdf_graph( RDFCanonicalizationWarning, stacklevel=2, ) + if diff_stable: + warnings.warn( + "diff_stable=True was requested but this graph took the rdflib fallback, " + "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " + "deterministic but NOT diff-stable: an unrelated edit may still renumber " + "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " + "predicates) to get diff-stable labels.", + RDFCanonicalizationWarning, + stacklevel=2, + ) return _deterministic_fallback_serialize(graph, output_format) dataset = ox.Dataset() @@ -339,13 +364,17 @@ def canonicalize_rdf_graph( # 3. Canonicalize blank node labels with RDFC-1.0. dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) + quads = list(dataset) + + if diff_stable: + quads = wl_relabel_quads(quads) + # 4. Sort triples for deterministic ordering. # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset # iteration order is not sorted and varies across processes (verified # empirically against pyoxigraph 0.5.8). The explicit string-key sort # is load-bearing for byte-identical output across runs; see # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - quads = list(dataset) sorted_triples = sorted( (ox.Triple(q.subject, q.predicate, q.object) for q in quads), key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), diff --git a/pyproject.toml b/pyproject.toml index 559cf265ac..fd62dc2986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,7 @@ markers = [ "yarrrml: End-to-end tests for the YARRRML generator", "typedbgen: Tests for the TypeDB generator", "arrays: Array specifications!", + "diffable_rdf: tests that exercise the diffable-rdf integration (--diff-stable); run as a canary against linkml in diffable-rdf's own CI", ] # https://docs.astral.sh/ruff/settings/ diff --git a/tests/conftest.py b/tests/conftest.py index bcf958b9e5..ba13d6f764 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -422,7 +422,8 @@ class MockImportErrorFinder(MetaPathFinder): """ Fake like we don't have a module when we really do. - see the ``mock_black_import`` fixture for example usage. + see the ``mock_missing_import`` fixture to hide any module, or ``mock_black_import`` + for a single-purpose example. .. note:: @@ -442,6 +443,40 @@ def find_spec(self, fullname, path, target): return None +@pytest.fixture(scope="function") +def mock_missing_import(): + """Pretend an optional dependency is not installed, for any module. + + Yields a callable taking a module name, so a test for one optional + dependency does not need a fixture of its own:: + + def test_requires_the_extra(mock_missing_import): + mock_missing_import("diffable_rdf") + with pytest.raises(ImportError, match="linkml-runtime[diff-stable]"): + ... + + See :class:`MockImportErrorFinder` for the caveat about reimporting + modules that already imported the one being hidden. + """ + removed = {} + finders = [] + + def hide(module: str) -> None: + for name, loaded in list(sys.modules.items()): + if name.startswith(module): + removed[name] = loaded + del sys.modules[name] + finder = MockImportErrorFinder(module) + finders.append(finder) + sys.meta_path.insert(0, finder) + + yield hide + + sys.modules.update(removed) + for finder in finders: + sys.meta_path.remove(finder) + + @pytest.fixture(scope="function") def mock_black_import(): """ diff --git a/tests/linkml/test_scripts/test_diff_stable.py b/tests/linkml/test_scripts/test_diff_stable.py new file mode 100644 index 0000000000..d1c292276b --- /dev/null +++ b/tests/linkml/test_scripts/test_diff_stable.py @@ -0,0 +1,45 @@ +"""CLI checks for the optional diff-stable RDF labels.""" + +import re +from pathlib import Path + +import pyoxigraph as ox +import pytest +from click.testing import CliRunner + +from linkml.generators.shaclgen import cli as shacl_cli + +_SCHEMA = """id: https://example.org/cli +name: cli +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/cli/ +default_prefix: ex +imports: [linkml:types] +classes: + Thing: + attributes: + label: + range: string +""" + + +@pytest.mark.parametrize( + ("flag", "label_pattern"), + [ + pytest.param("--no-diff-stable", r"c14n[0-9]+", id="disabled"), + pytest.param("--diff-stable", r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?", id="enabled", marks=pytest.mark.diffable_rdf), + ], +) +def test_the_flag_reaches_the_serializer(tmp_path: Path, flag: str, label_pattern: str) -> None: + """The CLI flag selects the blank-node labelling it names.""" + schema = tmp_path / "schema.yaml" + schema.write_text(_SCHEMA, encoding="utf-8") + + result = CliRunner().invoke(shacl_cli, [str(schema), flag]) + + assert result.exit_code == 0, result.output + quads = list(ox.parse(result.stdout, format=ox.RdfFormat.TURTLE)) + blank_nodes = {term for q in quads for term in (q.subject, q.object) if isinstance(term, ox.BlankNode)} + assert blank_nodes + assert all(re.fullmatch(label_pattern, node.value) for node in blank_nodes) diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 63dbf095a1..8da48fa8b3 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -1,17 +1,24 @@ """Tests for deterministic RDF serialization via pyoxigraph RDFC-1.0.""" +import difflib import os import re import subprocess import sys import textwrap +from pathlib import Path import pyoxigraph as ox import pytest import rdflib -from rdflib import BNode, Graph, Literal, URIRef -from rdflib.namespace import RDF - +from rdflib import BNode, Graph, Literal, Namespace, URIRef +from rdflib.namespace import OWL, RDF, RDFS, SH, XSD + +from linkml.generators.owlgen import OwlSchemaGenerator +from linkml.generators.rdfgen import RDFGenerator +from linkml.generators.shaclgen import ShaclGenerator +from linkml.generators.shexgen import ShExGenerator +from linkml.utils.generator import Generator from linkml_runtime.utils import rdf_canonicalize as rdf_canon_mod from linkml_runtime.utils.rdf_canonicalize import ( RDFCanonicalizationWarning, @@ -507,3 +514,244 @@ def run(seed: str) -> str: assert out_a == out_b, ( "Fallback output differs across PYTHONHASHSEED values; blank-node canonicalization may be missing" ) + + +def _shapes_graph(count: int, extra: bool = False) -> Graph: + """A graph shaped like generator output: one blank node per named subject.""" + g = Graph() + g.bind("ex", "http://example.com/") + names = [f"{i:02d}" for i in range(count)] + (["AAAinserted"] if extra else []) + for name in names: + subject = URIRef(f"http://example.com/Shape{name}") + prop = BNode() + g.add((subject, URIRef("http://example.com/property"), prop)) + g.add((prop, URIRef("http://example.com/path"), URIRef(f"http://example.com/p{name}"))) + return g + + +def _changed_line_count(before: str, after: str) -> int: + """Count added and removed content lines in a unified diff.""" + + diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") + return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) + + +def test_diff_stable_without_the_extra_names_the_extra(mock_missing_import) -> None: + """Requesting the opt-in path without diffable-rdf says how to install it.""" + mock_missing_import("diffable_rdf") + + with pytest.raises(ImportError, match=r"linkml-runtime\[diff-stable\]"): + canonicalize_rdf_graph(_make_graph_with_bnodes(), diff_stable=True) + + +@pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) +@pytest.mark.parametrize("output_format", ["nt", "turtle"]) +def test_diff_stable_preserves_semantics(diff_stable: bool, output_format: str) -> None: + """Each serialization preserves the source graph under blank-node renaming.""" + graph = _make_graph_with_bnodes() + output = canonicalize_rdf_graph(graph, output_format=output_format, diff_stable=diff_stable) + parsed = Graph().parse(data=output, format=output_format) + assert len(parsed) == len(graph) + assert rdflib.compare.isomorphic(graph, parsed) + + +@pytest.mark.diffable_rdf +def test_diff_stable_is_deterministic() -> None: + """A cycle's tied signatures remain independent of input blank-node IDs.""" + graphs = [] + predicate = URIRef("http://example.com/next") + for labels, reverse in [("abcd", False), ("dbac", True)]: + nodes = [BNode(label) for label in labels] + triples = [(node, predicate, nodes[(i + 1) % len(nodes)]) for i, node in enumerate(nodes)] + graph = Graph() + for triple in reversed(triples) if reverse else triples: + graph.add(triple) + graphs.append(graph) + + assert rdflib.compare.isomorphic(*graphs) + outputs = [canonicalize_rdf_graph(graph, output_format="nt", diff_stable=True) for graph in graphs] + assert outputs[0] == outputs[1] + for graph, output in zip(graphs, outputs): + assert rdflib.compare.isomorphic(graph, Graph().parse(data=output, format="nt")) + + +@pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) +def test_diff_stable_insertion_labels(diff_stable: bool) -> None: + """An independent insertion renumbers RDFC labels but preserves the old WL label.""" + before = Graph().parse( + data="""@prefix ex: . + ex:Shape00 ex:property _:old . + _:old ex:path ex:p00 . + """, + format="turtle", + ) + after = Graph() + for triple in before: + after.add(triple) + inserted = BNode() + ex = Namespace("http://example.com/") + after.add((ex.ShapeAAAinserted, ex.property, inserted)) + after.add((inserted, ex.path, ex.pAAAinserted)) + + outputs = [canonicalize_rdf_graph(graph, output_format="nt", diff_stable=diff_stable) for graph in (before, after)] + parsed = [set(ox.parse(output, format=ox.RdfFormat.N_TRIPLES)) for output in outputs] + old_subject = ox.NamedNode(str(ex.Shape00)) + new_subject = ox.NamedNode(str(ex.ShapeAAAinserted)) + property_iri = ox.NamedNode(str(ex.property)) + path_iri = ox.NamedNode(str(ex.path)) + (old_before,) = {q.object for q in parsed[0] if q.subject == old_subject and q.predicate == property_iri} + (old_after,) = {q.object for q in parsed[1] if q.subject == old_subject and q.predicate == property_iri} + (new_after,) = {q.object for q in parsed[1] if q.subject == new_subject and q.predicate == property_iri} + assert all(isinstance(node, ox.BlankNode) for node in (old_before, old_after, new_after)) + assert old_after != new_after + inserted_quads = { + ox.Quad(new_subject, property_iri, new_after), + ox.Quad(new_after, path_iri, ox.NamedNode(str(ex.pAAAinserted))), + } + assert parsed[0] == { + ox.Quad(old_subject, property_iri, old_before), + ox.Quad(old_before, path_iri, ox.NamedNode(str(ex.p00))), + } + assert ( + parsed[1] + == { + ox.Quad(old_subject, property_iri, old_after), + ox.Quad(old_after, path_iri, ox.NamedNode(str(ex.p00))), + } + | inserted_quads + ) + + if diff_stable: + assert all(re.fullmatch(_STABLE_LABEL, node.value) for node in (old_before, old_after, new_after)) + assert old_before == old_after + assert parsed[1] - parsed[0] == inserted_quads + assert parsed[0] - parsed[1] == set() + else: + assert (old_before.value, old_after.value, new_after.value) == ("c14n0", "c14n1", "c14n0") + for graph, output in zip((before, after), outputs): + assert rdflib.compare.isomorphic(graph, Graph().parse(data=output, format="nt")) + + +@pytest.mark.diffable_rdf +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches() -> None: + """Diff-stable labels reduce Turtle line churn for independent property nodes.""" + before, after = _shapes_graph(20), _shapes_graph(20, extra=True) + baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) + stable = _changed_line_count( + canonicalize_rdf_graph(before, diff_stable=True), + canonicalize_rdf_graph(after, diff_stable=True), + ) + assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" + + +_STABLE_LABEL = r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?" + + +_DIFF_STABLE_SCHEMA = """\ +id: https://example.org/diffstable +name: diffstable +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/diffstable/ +default_prefix: ex +default_range: string +imports: + - linkml:types +classes: + Person: + slots: [name, knows] + Organization: + slots: [name] +slots: + name: + range: string + knows: + range: Person + multivalued: true +""" + + +@pytest.fixture +def diff_stable_schema(tmp_path: Path) -> Path: + """Write the small schema for real generator calls.""" + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8") + return schema + + +@pytest.mark.diffable_rdf +@pytest.mark.parametrize( + ("generator", "kwargs"), + [ + pytest.param(OwlSchemaGenerator, {}, id="owlgen"), + pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), + pytest.param(ShaclGenerator, {}, id="shaclgen"), + pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), + ], +) +def test_diff_stable_reaches_every_rdf_generator( + diff_stable_schema: Path, generator: type[Generator], kwargs: dict[str, str] +) -> None: + """Every RDF generator emits the designed schema with diff-stable labels.""" + output = generator(str(diff_stable_schema), diff_stable=True, **kwargs).serialize() + quads = list(ox.parse(output, format=ox.RdfFormat.TURTLE)) + blank_nodes = {term for quad in quads for term in (quad.subject, quad.object) if isinstance(term, ox.BlankNode)} + assert blank_nodes + assert all(re.fullmatch(_STABLE_LABEL, node.value) for node in blank_nodes) + graph = Graph().parse(data=output, format="turtle") + ex = Namespace("https://example.org/diffstable/") + if generator is OwlSchemaGenerator: + assert (ex.Person, RDF.type, OWL.Class) in graph + assert (ex.Organization, RDF.type, OWL.Class) in graph + (restriction,) = { + node + for node in graph.objects(ex.Person, RDFS.subClassOf) + if (node, OWL.onProperty, ex.knows) in graph and (node, OWL.allValuesFrom, ex.Person) in graph + } + assert isinstance(restriction, BNode) + assert (restriction, RDF.type, OWL.Restriction) in graph + assert (restriction, OWL.allValuesFrom, ex.Person) in graph + elif generator is ShaclGenerator: + for subject, paths in [(ex.Person, {ex.name, ex.knows}), (ex.Organization, {ex.name})]: + assert (subject, RDF.type, SH.NodeShape) in graph + assert (subject, SH.targetClass, subject) in graph + properties = set(graph.objects(subject, SH.property)) + assert {graph.value(prop, SH.path) for prop in properties} == paths + (ignored,) = graph.objects(subject, SH.ignoredProperties) + assert set(graph.predicate_objects(ignored)) == {(RDF.first, RDF.type), (RDF.rest, RDF.nil)} + (knows,) = {node for node in graph.objects(ex.Person, SH.property) if (node, SH.path, ex.knows) in graph} + assert (knows, SH["class"], ex.Person) in graph + elif generator is RDFGenerator: + linkml = Namespace("https://w3id.org/linkml/") + schema = ex.diffstable + assert (schema, RDF.type, linkml.SchemaDefinition) in graph + assert set(graph.objects(schema, linkml.classes)) == {ex.Person, ex.Organization} + (prefix,) = {node for node in graph.objects(schema, SH.declare) if (node, SH.prefix, Literal("ex")) in graph} + assert (prefix, SH.namespace, Literal(str(ex), datatype=XSD.anyURI)) in graph + else: + assert generator is ShExGenerator + shex = Namespace("http://www.w3.org/ns/shex#") + assert (ex.Person, RDF.type, shex.Shape) in graph + (expression,) = graph.objects(ex.Person, shex.expression) + assert (expression, RDF.type, shex.EachOf) in graph + (expressions,) = graph.objects(expression, shex.expressions) + assert ex.Person_tes in set(graph.items(expressions)) + assert (ex.Person_tes, RDF.type, shex.EachOf) in graph + (slots,) = graph.objects(ex.Person_tes, shex.expressions) + (knows,) = {node for node in graph.items(slots) if (node, shex.predicate, ex.knows) in graph} + assert (knows, RDF.type, shex.TripleConstraint) in graph + assert (knows, shex.valueExpr, ex.Person) in graph + assert (knows, shex.min, Literal(0)) in graph + assert (knows, shex.max, Literal(-1)) in graph + + +@pytest.mark.diffable_rdf +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback() -> None: + """A relative IRI requires the fallback and a warning that labels cannot apply.""" + graph = _make_graph_with_bnodes() + graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) + with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + stable = canonicalize_rdf_graph(graph, diff_stable=True) + with pytest.warns(RDFCanonicalizationWarning): + plain = canonicalize_rdf_graph(graph, diff_stable=False) + assert stable == plain diff --git a/uv.lock b/uv.lock index 3b8ef0ceb1..7c22c3d061 100644 --- a/uv.lock +++ b/uv.lock @@ -1075,6 +1075,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "diffable-rdf" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyoxigraph" }, + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/df/1d71c0c984eac1cfd25531aabf84528941b01083c875da49d44882d1b5af/diffable_rdf-0.4.0.tar.gz", hash = "sha256:a84afaa10332e6a039b6ddcabf76bced3049e4da467c88ee786d0fa35218111e", size = 838040, upload-time = "2026-09-11T11:43:12.844Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/49/b62ee864fe6769b759f04e3709fd79711b1cdf49223b1bb0e985e507ead3/diffable_rdf-0.4.0-py3-none-any.whl", hash = "sha256:c59c57429465f786fdbc8d38ddbb687da16748c5752f0c2ac4a77b5527aa9aed", size = 46126, upload-time = "2026-09-11T11:43:11.536Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2395,6 +2408,11 @@ dependencies = [ { name = "watchdog" }, ] +[package.optional-dependencies] +diff-stable = [ + { name = "linkml-runtime", extra = ["diff-stable"] }, +] + [package.dev-dependencies] bigquery = [ { name = "sqlalchemy-bigquery" }, @@ -2409,6 +2427,7 @@ dev = [ { name = "ipython-genutils" }, { name = "jsonpatch" }, { name = "jupyter" }, + { name = "linkml-runtime", extra = ["diff-stable"] }, { name = "mock" }, { name = "myst-nb" }, { name = "nbconvert" }, @@ -2461,6 +2480,7 @@ shacl = [ tests = [ { name = "black" }, { name = "duckdb" }, + { name = "linkml-runtime", extra = ["diff-stable"] }, { name = "numpydantic" }, { name = "pyshacl" }, { name = "sqlalchemy-bigquery" }, @@ -2500,6 +2520,7 @@ requires-dist = [ { name = "jsonasobj2", specifier = ">=1.0.3,<2.0.0" }, { name = "jsonschema", extras = ["format"], specifier = ">=4.26.0" }, { name = "linkml-runtime", editable = "packages/linkml_runtime" }, + { name = "linkml-runtime", extras = ["diff-stable"], marker = "extra == 'diff-stable'", editable = "packages/linkml_runtime" }, { name = "openapi-spec-validator", specifier = ">=0.8.4" }, { name = "openpyxl" }, { name = "parse" }, @@ -2520,6 +2541,7 @@ requires-dist = [ { name = "typing-extensions", marker = "python_full_version < '3.12'", specifier = ">=4.6.0" }, { name = "watchdog", specifier = ">=0.9.0" }, ] +provides-extras = ["diff-stable"] [package.metadata.requires-dev] bigquery = [{ name = "sqlalchemy-bigquery", specifier = ">=1.9.0" }] @@ -2533,6 +2555,7 @@ dev = [ { name = "ipython-genutils" }, { name = "jsonpatch", specifier = ">=1.33" }, { name = "jupyter" }, + { name = "linkml-runtime", extras = ["diff-stable"], editable = "packages/linkml_runtime" }, { name = "mock", specifier = ">=5.1.0" }, { name = "myst-nb", marker = "python_full_version >= '3.10'", specifier = ">=1.4.0" }, { name = "nbconvert" }, @@ -2580,6 +2603,7 @@ shacl = [{ name = "pyshacl", specifier = ">=0.40.1" }] tests = [ { name = "black", specifier = ">=24.0.0" }, { name = "duckdb", specifier = ">=1.5.5" }, + { name = "linkml-runtime", extras = ["diff-stable"], editable = "packages/linkml_runtime" }, { name = "numpydantic", specifier = ">=1.10.0" }, { name = "pyshacl", specifier = ">=0.40.1" }, { name = "sqlalchemy-bigquery", specifier = ">=1.17.2" }, @@ -2628,6 +2652,9 @@ dev = [ { name = "coverage" }, { name = "requests-cache" }, ] +diff-stable = [ + { name = "diffable-rdf" }, +] [package.dev-dependencies] dev = [ @@ -2641,6 +2668,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", marker = "extra == 'diff-stable'", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, @@ -2655,7 +2683,7 @@ requires-dist = [ { name = "requests" }, { name = "requests-cache", marker = "extra == 'dev'", specifier = ">=1.3.3" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "diff-stable"] [package.metadata.requires-dev] dev = [ From 4bde563c55d482d508470b87420423480985c7b8 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 16:34:30 +0200 Subject: [PATCH 2/2] docs(owl): document deterministic serialization and --diff-stable Describe in the gen-owl documentation what RDFC-1.0 canonicalization guarantees for standard RDF, why its sequential blank-node numbering can still produce noisy diffs, and that --diff-stable reduces that churn without guaranteeing minimal diffs. Note that the option also exists on gen-rdf, gen-shacl and gen-shex --format rdf, and that graphs that are not standard RDF are serialized by rdflib with a warning, where --diff-stable has no effect. Signed-off-by: jdsika --- docs/generators/owl.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index 4b6f076fe8..facbbfdc16 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -311,6 +311,27 @@ Other examples translation of Biolink schema to OWL +Deterministic output +^^^^^^^^^^^^^^^^^^^^ + +``gen-owl`` canonicalizes the graph with `RDFC-1.0 `_ +before serializing it. For standard RDF, repeated runs over the same schema +therefore produce byte-identical Turtle, and checked-in artifacts do not change +between runs. + +RDFC-1.0 numbers blank nodes in canonical order (``_:c14n0``, ``_:c14n1``, ...), +so adding a statement can renumber unrelated blank nodes. ``--diff-stable`` +derives labels from blank-node neighbourhoods instead, which reduces this churn +but does not guarantee minimal diffs; see :ref:`rdf-in-version-control`. The +option is also available on ``gen-rdf``, ``gen-shacl`` and +``gen-shex --format rdf``. + +Graphs that are not standard RDF, such as the metamodel with its relative IRI +``bibo:status ``, cannot be canonicalized under RDFC-1.0. They are +serialized by rdflib instead, with an ``RDFCanonicalizationWarning``, and +``--diff-stable`` has no effect on them. + + Docs ----