diff --git a/docs/PROMPT_RENDERING.md b/docs/PROMPT_RENDERING.md index f8f882f2..8d6c4558 100644 --- a/docs/PROMPT_RENDERING.md +++ b/docs/PROMPT_RENDERING.md @@ -106,3 +106,37 @@ must never be cached globally. - Secrets must not appear in YAML or prompt files. - Prompt text is not a security boundary; access and data restrictions belong in access/tool/plugin logic. + +--- + +## Prompt Provenance + +To support reproducible agent runs and future remote prompt management integrations +(e.g., Langfuse), prompt source templates carry provider-agnostic provenance metadata +(`PromptProvenance`). + +### Provenance Contract + +```python +@dataclass(frozen=True) +class PromptProvenance: + source: str + identifier: str + content_hash: str + requested_ref: str | None = None + resolved_revision: str | None = None +``` + +| Field | Description | Example (Local File) | Example (Remote / Langfuse) | +| ----- | ----------- | -------------------- | --------------------------- | +| `source` | Origin provider of the prompt | `"file"` | `"langfuse"` | +| `identifier` | Stable source identifier or path | `"prompts/billing/system.md"` | `"billing-agent"` | +| `content_hash` | Deterministic SHA-256 fingerprint (`sha256:`) of the source content | `"sha256:4a8b..."` | `"sha256:4a8b..."` | +| `requested_ref` | Requested label, tag, or version (if applicable) | `None` | `"production"` or `"12"` | +| `resolved_revision` | Exact immutable revision resolved by the source | `None` | `"17"` | + +### Source Content vs. Rendered Content + +- **Source content hash**: The fingerprint is computed deterministically over the raw source template text in UTF-8 before variable substitution (`compute_prompt_hash(content)`). +- **No sensitive leak**: Provenance tracks source templates, versions, and hashes rather than persisting sensitive request-specific rendered prompt content. +- **Local files**: Loaded via `load_file_with_provenance(base_dir, relative_path)` returning `ResolvedPrompt(content=..., provenance=...)` with normalized forward-slash paths. diff --git a/src/agent_engine/core/__init__.py b/src/agent_engine/core/__init__.py index e69de29b..30537711 100644 --- a/src/agent_engine/core/__init__.py +++ b/src/agent_engine/core/__init__.py @@ -0,0 +1,11 @@ +from agent_engine.core.provenance import ( + PromptProvenance, + ResolvedPrompt, + compute_prompt_hash, +) + +__all__ = [ + "PromptProvenance", + "ResolvedPrompt", + "compute_prompt_hash", +] diff --git a/src/agent_engine/core/provenance.py b/src/agent_engine/core/provenance.py new file mode 100644 index 00000000..096c0224 --- /dev/null +++ b/src/agent_engine/core/provenance.py @@ -0,0 +1,94 @@ +"""Prompt provenance models and deterministic content hashing. + +Provides provider-agnostic provenance metadata tracking the source origin, +stable identifier, requested reference, immutable resolved revision, and +deterministic content fingerprint of prompt source templates. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path, PurePath, PureWindowsPath +from typing import Any + + +def normalize_prompt_path(path: str | Path | PurePath) -> str: + """Normalize a local prompt path to a canonical POSIX forward-slash string. + + Interprets backslash separators in relative string paths via PureWindowsPath + while preserving Path and PurePath instances and avoiding reinterpreting + anchored or drive-prefixed Windows paths as relative paths on POSIX. + """ + if isinstance(path, PurePath): + return path.as_posix() + win_path = PureWindowsPath(path) + if not win_path.drive and not win_path.root: + return win_path.as_posix() + return PurePath(path).as_posix() + + +def compute_prompt_hash(content: str) -> str: + """Compute a deterministic SHA-256 fingerprint of prompt source text. + + Args: + content: The raw prompt source string to hash. + + Returns: + Deterministic fingerprint formatted as 'sha256:<64-char-hex-digest>'. + """ + digest = hashlib.sha256(content.encode("utf-8")).hexdigest() + return f"sha256:{digest}" + + +@dataclass(frozen=True) +class PromptProvenance: + """Provider-agnostic provenance metadata for a prompt source. + + Captures the origin and exact immutable revision or content fingerprint + of a prompt template before runtime variable substitution. + """ + + source: str + identifier: str + content_hash: str + requested_ref: str | None = None + resolved_revision: str | None = None + + @classmethod + def for_local_file( + cls, + path: str | Path | PurePath, + content: str, + ) -> PromptProvenance: + """Create provenance for a local prompt file. + + The path identifier is normalized to POSIX forward-slash format to ensure + consistent provenance identifiers across platforms. + """ + normalized_identifier = normalize_prompt_path(path) + return cls( + source="file", + identifier=normalized_identifier, + content_hash=compute_prompt_hash(content), + requested_ref=None, + resolved_revision=None, + ) + + def to_dict(self) -> dict[str, Any]: + """Return a dictionary representation of the provenance metadata.""" + return { + "source": self.source, + "identifier": self.identifier, + "requested_ref": self.requested_ref, + "resolved_revision": self.resolved_revision, + "content_hash": self.content_hash, + } + + +@dataclass(frozen=True) +class ResolvedPrompt: + """A prompt template loaded from a source, paired with its provenance.""" + + content: str + provenance: PromptProvenance diff --git a/src/agent_engine/engine/langgraph/prompting.py b/src/agent_engine/engine/langgraph/prompting.py index 73ce17fd..01de6f28 100644 --- a/src/agent_engine/engine/langgraph/prompting.py +++ b/src/agent_engine/engine/langgraph/prompting.py @@ -5,9 +5,25 @@ import re from pathlib import Path +from agent_engine.core.provenance import ( + PromptProvenance, + ResolvedPrompt, + compute_prompt_hash, + normalize_prompt_path, +) from agent_engine.core.spec import NodeSpec from agent_engine.loaders.resolver_loader import ResolverLoader +__all__ = [ + "PromptProvenance", + "ResolvedPrompt", + "compute_prompt_hash", + "load_file", + "load_file_with_provenance", + "render_prompt", + "resolve_prompt_context", +] + def render_prompt(template: str, context: dict[str, str]) -> str: def replace(match: re.Match[str]) -> str: @@ -28,8 +44,31 @@ def resolve_prompt_context(loader: ResolverLoader, spec: NodeSpec) -> dict[str, return context -def load_file(base_dir: Path, relative_path: str | None) -> str: +def load_file_with_provenance( + base_dir: Path, + relative_path: str | None, +) -> ResolvedPrompt | None: + """Load a local prompt file and construct its provenance metadata. + + Returns: + A ResolvedPrompt containing the raw file content and its provenance, + or None if relative_path is empty or the file does not exist. + """ if not relative_path: - return "" - path = base_dir / relative_path - return path.read_text(encoding="utf-8") if path.is_file() else "" + return None + normalized_path = normalize_prompt_path(relative_path) + path = base_dir / normalized_path + if not path.is_file(): + return None + content = path.read_text(encoding="utf-8") + provenance = PromptProvenance.for_local_file(normalized_path, content) + return ResolvedPrompt(content=content, provenance=provenance) + + +def load_file(base_dir: Path, relative_path: str | None) -> str: + """Load the raw text of a prompt template from disk. + + Backward-compatible convenience helper that discards provenance metadata. + """ + resolved = load_file_with_provenance(base_dir, relative_path) + return resolved.content if resolved is not None else "" diff --git a/tests/core/test_provenance.py b/tests/core/test_provenance.py new file mode 100644 index 00000000..646f817f --- /dev/null +++ b/tests/core/test_provenance.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError +from pathlib import Path, PurePosixPath, PureWindowsPath + +import pytest + +from agent_engine.core.provenance import ( + PromptProvenance, + ResolvedPrompt, + compute_prompt_hash, +) + + +class TestDeterministicPromptHashing: + def test_same_content_produces_identical_hash(self) -> None: + content = "You are a helpful customer support agent for {{company}}." + hash1 = compute_prompt_hash(content) + hash2 = compute_prompt_hash(content) + + assert hash1 == hash2 + assert hash1.startswith("sha256:") + assert len(hash1) == len("sha256:") + 64 + + def test_content_changes_produce_different_hash(self) -> None: + content_v1 = "You are a customer assistant." + content_v2 = "You are a helpful customer assistant." + + assert compute_prompt_hash(content_v1) != compute_prompt_hash(content_v2) + + def test_whitespace_and_newline_differences_change_hash(self) -> None: + base = "System prompt" + with_trailing_space = "System prompt " + with_newline = "System prompt\n" + + hash_base = compute_prompt_hash(base) + assert compute_prompt_hash(with_trailing_space) != hash_base + assert compute_prompt_hash(with_newline) != hash_base + + def test_utf8_multibyte_characters_handled_deterministically(self) -> None: + content = "Support: こんにちは世界 🚀" + hash1 = compute_prompt_hash(content) + hash2 = compute_prompt_hash(content) + + assert hash1 == hash2 + assert hash1.startswith("sha256:") + + def test_empty_string_produces_known_sha256(self) -> None: + # Standard SHA-256 of empty string is: + # e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + expected = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + assert compute_prompt_hash("") == expected + + +class TestPromptProvenanceModel: + def test_local_file_provenance_creation(self) -> None: + path = "prompts/billing/system.md" + content = "You are a billing specialist." + provenance = PromptProvenance.for_local_file(path, content) + + assert provenance.source == "file" + assert provenance.identifier == "prompts/billing/system.md" + assert provenance.content_hash == compute_prompt_hash(content) + assert provenance.requested_ref is None + assert provenance.resolved_revision is None + + def test_path_normalization_converts_backslashes_to_forward_slashes(self) -> None: + windows_path = PureWindowsPath("prompts\\billing\\system.md") + content = "You are a billing specialist." + provenance = PromptProvenance.for_local_file(windows_path, content) + + assert provenance.identifier == "prompts/billing/system.md" + + def test_plain_backslash_string_produces_forward_slash_identifier(self) -> None: + content = "You are a billing specialist." + provenance = PromptProvenance.for_local_file("prompts\\billing\\system.md", content) + + assert provenance.identifier == "prompts/billing/system.md" + + def test_forward_and_backslash_relative_strings_produce_identical_identifiers(self) -> None: + content = "You are a billing specialist." + posix_prov = PromptProvenance.for_local_file("prompts/billing/system.md", content) + win_prov = PromptProvenance.for_local_file("prompts\\billing\\system.md", content) + + assert posix_prov.identifier == "prompts/billing/system.md" + assert posix_prov == win_prov + + def test_posix_path_and_pure_path_objects_produce_normalized_identifier(self) -> None: + content = "You are a billing specialist." + from_path = PromptProvenance.for_local_file( + Path("prompts") / "billing" / "system.md", content + ) + from_pure_posix = PromptProvenance.for_local_file( + PurePosixPath("prompts/billing/system.md"), + content, + ) + + assert from_path.identifier == "prompts/billing/system.md" + assert from_pure_posix.identifier == "prompts/billing/system.md" + + def test_remote_compatible_representation_with_mutable_label(self) -> None: + provenance = PromptProvenance( + source="langfuse", + identifier="billing-agent", + requested_ref="production", + resolved_revision="17", + content_hash=compute_prompt_hash("Raw template v17"), + ) + + assert provenance.source == "langfuse" + assert provenance.identifier == "billing-agent" + assert provenance.requested_ref == "production" + assert provenance.resolved_revision == "17" + assert provenance.content_hash.startswith("sha256:") + + def test_remote_compatible_representation_with_pinned_version(self) -> None: + provenance = PromptProvenance( + source="langfuse", + identifier="billing-agent", + requested_ref="12", + resolved_revision="12", + content_hash=compute_prompt_hash("Raw template v12"), + ) + + assert provenance.requested_ref == "12" + assert provenance.resolved_revision == "12" + + def test_provenance_is_immutable(self) -> None: + provenance = PromptProvenance.for_local_file("prompts/system.md", "hello") + + with pytest.raises(FrozenInstanceError): + provenance.identifier = "prompts/other.md" # type: ignore[misc] + + with pytest.raises(FrozenInstanceError): + provenance.resolved_revision = "99" # type: ignore[misc] + + def test_to_dict_serialization(self) -> None: + provenance = PromptProvenance( + source="langfuse", + identifier="billing-agent", + requested_ref="production", + resolved_revision="17", + content_hash="sha256:123456", + ) + data = provenance.to_dict() + + assert data == { + "source": "langfuse", + "identifier": "billing-agent", + "requested_ref": "production", + "resolved_revision": "17", + "content_hash": "sha256:123456", + } + + def test_sensitive_rendered_content_not_in_provenance(self) -> None: + raw_template = "Hello {{customer_ssn}}" + sensitive_rendered = "Hello 123-45-6789" + + provenance = PromptProvenance.for_local_file("prompts/agent.md", raw_template) + + # Provenance only records the source template hash, never the rendered content + assert sensitive_rendered not in str(provenance) + assert sensitive_rendered not in provenance.to_dict().values() + assert provenance.content_hash == compute_prompt_hash(raw_template) + assert provenance.content_hash != compute_prompt_hash(sensitive_rendered) + + +class TestResolvedPrompt: + def test_resolved_prompt_structure_and_immutability(self) -> None: + content = "Template content" + provenance = PromptProvenance.for_local_file("prompts/sys.md", content) + resolved = ResolvedPrompt(content=content, provenance=provenance) + + assert resolved.content == content + assert resolved.provenance == provenance + + with pytest.raises(FrozenInstanceError): + resolved.content = "Changed" # type: ignore[misc] diff --git a/tests/engine/test_prompt_loading.py b/tests/engine/test_prompt_loading.py new file mode 100644 index 00000000..62b6eb75 --- /dev/null +++ b/tests/engine/test_prompt_loading.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from agent_engine.core.provenance import compute_prompt_hash +from agent_engine.engine.langgraph.prompting import ( + load_file, + load_file_with_provenance, + render_prompt, +) + + +@pytest.fixture +def prompts_dir(tmp_path: Path) -> Path: + prompt_dir = tmp_path / "prompts" / "billing" + prompt_dir.mkdir(parents=True) + system_file = prompt_dir / "system.md" + system_file.write_text( + "You are a billing assistant for {{company_name}}.\nContact: {{support_email}}", + encoding="utf-8", + ) + return tmp_path + + +class TestPromptFileLoadingWithProvenance: + def test_load_existing_file_with_provenance(self, prompts_dir: Path) -> None: + rel_path = "prompts/billing/system.md" + resolved = load_file_with_provenance(prompts_dir, rel_path) + + assert resolved is not None + assert "You are a billing assistant for {{company_name}}." in resolved.content + assert resolved.provenance.source == "file" + assert resolved.provenance.identifier == "prompts/billing/system.md" + assert resolved.provenance.requested_ref is None + assert resolved.provenance.resolved_revision is None + assert resolved.provenance.content_hash == compute_prompt_hash(resolved.content) + + def test_load_file_with_provenance_normalizes_relative_path(self, prompts_dir: Path) -> None: + rel_path = "prompts\\billing\\system.md" + resolved = load_file_with_provenance(prompts_dir, rel_path) + + assert resolved is not None + assert resolved.provenance.identifier == "prompts/billing/system.md" + + def test_load_file_with_either_separator_style_produces_equivalent_result( + self, prompts_dir: Path + ) -> None: + from_forward = load_file_with_provenance(prompts_dir, "prompts/billing/system.md") + from_backslash = load_file_with_provenance(prompts_dir, "prompts\\billing\\system.md") + + assert from_forward is not None + assert from_backslash is not None + assert from_forward.content == from_backslash.content + assert from_forward.provenance == from_backslash.provenance + assert from_backslash.provenance.identifier == "prompts/billing/system.md" + + def test_load_nonexistent_file_returns_none(self, prompts_dir: Path) -> None: + resolved = load_file_with_provenance(prompts_dir, "prompts/missing.md") + assert resolved is None + + def test_load_empty_or_none_path_returns_none(self, prompts_dir: Path) -> None: + assert load_file_with_provenance(prompts_dir, None) is None + assert load_file_with_provenance(prompts_dir, "") is None + + def test_load_directory_path_returns_none(self, prompts_dir: Path) -> None: + assert load_file_with_provenance(prompts_dir, "prompts/billing") is None + + +class TestBackwardCompatibilityLoadFile: + def test_load_file_returns_exact_content_string(self, prompts_dir: Path) -> None: + content = load_file(prompts_dir, "prompts/billing/system.md") + assert "You are a billing assistant for {{company_name}}." in content + + def test_load_file_missing_returns_empty_string(self, prompts_dir: Path) -> None: + assert load_file(prompts_dir, "prompts/missing.md") == "" + + def test_load_file_none_or_empty_path_returns_empty_string(self, prompts_dir: Path) -> None: + assert load_file(prompts_dir, None) == "" + assert load_file(prompts_dir, "") == "" + + +class TestPromptProvenanceSeparationFromRenderedOutput: + def test_rendering_interpolates_without_modifying_provenance(self, prompts_dir: Path) -> None: + resolved = load_file_with_provenance(prompts_dir, "prompts/billing/system.md") + assert resolved is not None + + context = { + "company_name": "Acme Corp", + "support_email": "billing@acme.example.com", + } + rendered = render_prompt(resolved.content, context) + + assert "You are a billing assistant for Acme Corp." in rendered + assert "Contact: billing@acme.example.com" in rendered + + # The provenance source hash represents the raw template before variable substitution + assert resolved.provenance.content_hash == compute_prompt_hash(resolved.content) + assert resolved.provenance.content_hash != compute_prompt_hash(rendered)