From 0c630193e86bd0ce4ad11f2f98099f09c5d6f2f0 Mon Sep 17 00:00:00 2001 From: Alan Szmyt Date: Thu, 10 Sep 2026 18:10:47 -0400 Subject: [PATCH] feat: add publication hygiene gates --- crates/renderflow-core/src/evidence.rs | 5 + crates/renderflow-core/src/hygiene.rs | 738 ++++++++++++++++++ crates/renderflow-core/src/lib.rs | 6 + crates/renderflow-core/src/planning.rs | 107 ++- crates/renderflow-core/src/sdk.rs | 3 + crates/renderflow-core/src/spec.rs | 299 ++++++- .../renderflow-core/src/super_resolution.rs | 1 + docs/execution-evidence.md | 4 +- docs/provider-contract.md | 3 +- docs/publication-hygiene.md | 86 ++ docs/user-guide/spec-v2-reference.md | 2 + examples/publication-hygiene.yaml | 45 ++ mkdocs.yml | 1 + schemas/renderflow-hygiene-v1.schema.json | 58 ++ schemas/renderflow-provider-v1.schema.json | 5 +- schemas/renderflow-run-v1.schema.json | 41 + schemas/renderflow-v2.schema.json | 86 ++ .../fixtures/publication-hygiene/blocked.yaml | 25 + tests/fixtures/publication-hygiene/safe.yaml | 29 + .../publication-hygiene/source-blocked.md | 5 + .../publication-hygiene/source-safe.md | 3 + 21 files changed, 1542 insertions(+), 10 deletions(-) create mode 100644 crates/renderflow-core/src/hygiene.rs create mode 100644 docs/publication-hygiene.md create mode 100644 examples/publication-hygiene.yaml create mode 100644 schemas/renderflow-hygiene-v1.schema.json create mode 100644 tests/fixtures/publication-hygiene/blocked.yaml create mode 100644 tests/fixtures/publication-hygiene/safe.yaml create mode 100644 tests/fixtures/publication-hygiene/source-blocked.md create mode 100644 tests/fixtures/publication-hygiene/source-safe.md diff --git a/crates/renderflow-core/src/evidence.rs b/crates/renderflow-core/src/evidence.rs index 8069371..8045dd3 100644 --- a/crates/renderflow-core/src/evidence.rs +++ b/crates/renderflow-core/src/evidence.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::artifact::{Artifact, ArtifactStorageClass}; +use crate::hygiene::HygieneEvidence; use crate::toolchain::ToolchainSnapshot; pub const RUN_MANIFEST_SCHEMA_V1: &str = "renderflow.run/v1"; @@ -160,6 +161,8 @@ pub struct ArtifactEvidence { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub validation_evidence: Vec, pub fidelity: FidelityDeclaration, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hygiene: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub warnings: Vec, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] @@ -201,6 +204,7 @@ impl ArtifactEvidence { validation, validation_evidence: Vec::new(), fidelity, + hygiene: None, warnings: Vec::new(), metadata: artifact .metadata() @@ -552,6 +556,7 @@ mod tests { validation: ValidationState::Valid, validation_evidence: Vec::new(), fidelity: FidelityDeclaration::Lossless, + hygiene: None, warnings: Vec::new(), metadata: BTreeMap::new(), }; diff --git a/crates/renderflow-core/src/hygiene.rs b/crates/renderflow-core/src/hygiene.rs new file mode 100644 index 0000000..8ed3fc2 --- /dev/null +++ b/crates/renderflow-core/src/hygiene.rs @@ -0,0 +1,738 @@ +//! Auditable, non-destructive publication hygiene. +//! +//! Hygiene runs after derivative generation and before terminal materialization. +//! It creates a new artifact record even when payload bytes are unchanged, so +//! the policy decision and lineage remain explicit. + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::artifact::{Artifact, ArtifactDescriptor, ArtifactStorageClass, ArtifactStore}; +use crate::spec::{ + ContentRedactionPolicy, HygienePolicy, MetadataHygienePolicy, PublicationAudience, + RedactionDeterminism, +}; + +pub const HYGIENE_EVIDENCE_SCHEMA_V1: &str = "renderflow.hygiene/v1"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HygieneStatus { + Passed, + ReviewRequired, + Blocked, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HygieneFindingKind { + Metadata, + Secret, + ProtectedReference, + Redaction, + Rights, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HygieneFinding { + pub code: String, + pub kind: HygieneFindingKind, + pub class: String, + pub message: String, + pub blocking: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HygieneEvidence { + pub schema_version: String, + pub policy_id: String, + pub provider_id: String, + pub provider_version: String, + pub source_artifact_id: String, + pub output_artifact_id: String, + pub status: HygieneStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub redaction: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub changed_field_classes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub findings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RedactionEvidence { + pub provider_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_version: Option, + pub determinism: RedactionDeterminism, + pub policy_reviewed: bool, +} + +#[derive(Debug, Clone)] +pub struct HygieneOutcome { + pub artifact: Artifact, + pub evidence: HygieneEvidence, +} + +impl HygieneOutcome { + pub fn releasable(&self) -> bool { + self.evidence.status == HygieneStatus::Passed + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RedactionRequest { + pub artifact_id: String, + pub media_type: String, + pub bytes: Vec, + pub classes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RedactionResult { + pub bytes: Vec, + pub changed_classes: Vec, + pub findings: Vec, +} + +/// Replaceable content-redaction boundary. Implementations must not mutate the +/// source artifact and must return safe findings that omit sensitive values. +pub trait ContentRedactionProvider: Send + Sync { + fn id(&self) -> &str; + fn version(&self) -> &str; + fn determinism(&self) -> RedactionDeterminism; + fn redact(&self, request: &RedactionRequest) -> Result; +} + +#[derive(Default)] +pub struct HygieneEngine { + redaction_providers: Vec>, +} + +impl HygieneEngine { + pub fn new() -> Self { + Self::default() + } + + pub fn with_redaction_provider( + mut self, + provider: impl ContentRedactionProvider + 'static, + ) -> Self { + self.redaction_providers.push(Box::new(provider)); + self + } + + pub fn apply( + &self, + policy_id: &str, + policy: &HygienePolicy, + artifact: &Artifact, + store: &ArtifactStore, + ) -> Result { + let original_bytes = store.read_bytes(artifact)?; + let (mut bytes, mut changed_classes) = sanitize_embedded_metadata( + &original_bytes, + artifact.media_type().as_str(), + &policy.metadata, + ); + let (metadata, metadata_changes) = + sanitize_record_metadata(artifact.metadata(), &policy.metadata); + changed_classes.extend(metadata_changes); + let encoded_metadata = serde_json::to_vec(&metadata) + .context("failed to encode artifact metadata for hygiene scan")?; + + let mut findings = Vec::new(); + if policy.secrets.enabled { + findings.extend(scan_secrets( + &bytes, + artifact.media_type().as_str(), + &policy.secrets.markers, + policy.secrets.block, + )); + findings.extend(scan_secrets( + &encoded_metadata, + "application/json", + &policy.secrets.markers, + policy.secrets.block, + )); + } + findings.extend(scan_protected_references( + &bytes, + artifact.media_type().as_str(), + &policy.protected_references.terms, + policy.protected_references.case_sensitive, + policy.protected_references.block, + )); + for finding in scan_protected_references( + &encoded_metadata, + "application/json", + &policy.protected_references.terms, + policy.protected_references.case_sensitive, + policy.protected_references.block, + ) { + if !findings.contains(&finding) { + findings.push(finding); + } + } + + let mut redaction_evidence = None; + if let Some(redaction) = &policy.redaction { + let provider = self + .redaction_providers + .iter() + .find(|provider| provider.id() == redaction.provider); + redaction_evidence = Some(RedactionEvidence { + provider_id: redaction.provider.clone(), + provider_version: provider.map(|provider| provider.version().to_string()), + determinism: redaction.determinism, + policy_reviewed: redaction.reviewed, + }); + let result = self.apply_redaction(redaction, artifact, bytes)?; + bytes = result.bytes; + changed_classes.extend(result.changed_classes); + findings.extend(result.findings); + if redaction.determinism == RedactionDeterminism::Probabilistic { + findings.push(HygieneFinding { + code: "hygiene.redaction.review_required".to_string(), + kind: HygieneFindingKind::Redaction, + class: "probabilistic_redaction".to_string(), + message: "Probabilistic redaction requires explicit review of this candidate artifact".to_string(), + blocking: false, + }); + } + } + + findings.extend(rights_findings(policy)); + changed_classes.sort(); + changed_classes.dedup(); + + let status = if findings.iter().any(|finding| finding.blocking) { + HygieneStatus::Blocked + } else if findings.iter().any(|finding| { + finding.code == "hygiene.redaction.review_required" + || finding.code == "hygiene.rights.review_required" + }) { + HygieneStatus::ReviewRequired + } else { + HygieneStatus::Passed + }; + + let mut descriptor = ArtifactDescriptor::new( + artifact.format().clone(), + artifact.media_type().clone(), + ArtifactStorageClass::Terminal, + ) + .with_source(artifact.id().clone()); + for (key, value) in metadata { + descriptor = descriptor.with_metadata(key, value); + } + descriptor = descriptor + .with_metadata("renderflow.hygiene.policy", policy_id) + .with_metadata("renderflow.hygiene.status", serde_json::to_value(status)?); + let output = store.put_bytes(&bytes, descriptor)?; + let evidence = HygieneEvidence { + schema_version: HYGIENE_EVIDENCE_SCHEMA_V1.to_string(), + policy_id: policy_id.to_string(), + provider_id: "renderflow.core-hygiene".to_string(), + provider_version: env!("CARGO_PKG_VERSION").to_string(), + source_artifact_id: artifact.id().to_string(), + output_artifact_id: output.id().to_string(), + status, + redaction: redaction_evidence, + changed_field_classes: changed_classes, + findings, + }; + Ok(HygieneOutcome { + artifact: output, + evidence, + }) + } + + fn apply_redaction( + &self, + policy: &ContentRedactionPolicy, + artifact: &Artifact, + bytes: Vec, + ) -> Result { + let Some(provider) = self + .redaction_providers + .iter() + .find(|provider| provider.id() == policy.provider) + else { + return Ok(RedactionResult { + bytes, + changed_classes: Vec::new(), + findings: vec![HygieneFinding { + code: "hygiene.redaction.provider_unavailable".to_string(), + kind: HygieneFindingKind::Redaction, + class: "provider".to_string(), + message: format!( + "Configured redaction provider '{}' is unavailable; candidate was preserved", + policy.provider + ), + blocking: true, + }], + }); + }; + if provider.determinism() != policy.determinism { + return Ok(RedactionResult { + bytes, + changed_classes: Vec::new(), + findings: vec![HygieneFinding { + code: "hygiene.redaction.determinism_mismatch".to_string(), + kind: HygieneFindingKind::Redaction, + class: "provider".to_string(), + message: + "Configured redaction determinism does not match the selected provider" + .to_string(), + blocking: true, + }], + }); + } + provider.redact(&RedactionRequest { + artifact_id: artifact.id().to_string(), + media_type: artifact.media_type().to_string(), + bytes, + classes: policy.classes.clone(), + }) + } +} + +fn sanitize_record_metadata( + metadata: &BTreeMap, + policy: &MetadataHygienePolicy, +) -> (BTreeMap, Vec) { + let mut retained = BTreeMap::new(); + let mut changed = BTreeSet::new(); + for (key, value) in metadata { + let allowed = matches_any(key, &policy.allow); + let denied = matches_any(key, &policy.deny); + if denied || (policy.allowlist_only && !allowed) { + changed.insert(metadata_class(key)); + } else { + retained.insert(key.clone(), value.clone()); + } + } + (retained, changed.into_iter().collect()) +} + +fn matches_any(value: &str, patterns: &[String]) -> bool { + patterns.iter().any(|pattern| { + pattern + .strip_suffix(".*") + .map_or(value == pattern, |prefix| { + value == prefix || value.starts_with(&format!("{prefix}.")) + }) + }) +} + +fn metadata_class(key: &str) -> String { + key.split(['.', ':']) + .next() + .unwrap_or("metadata") + .to_string() +} + +fn sanitize_embedded_metadata( + bytes: &[u8], + media_type: &str, + policy: &MetadataHygienePolicy, +) -> (Vec, Vec) { + if media_type == "image/jpeg" { + return sanitize_jpeg(bytes, policy); + } + if media_type == "image/png" { + return sanitize_png(bytes, policy); + } + (bytes.to_vec(), Vec::new()) +} + +fn should_remove_class(class: &str, policy: &MetadataHygienePolicy) -> bool { + matches_any(class, &policy.deny) + || (policy.allowlist_only && !matches_any(class, &policy.allow)) +} + +fn sanitize_jpeg(bytes: &[u8], policy: &MetadataHygienePolicy) -> (Vec, Vec) { + if bytes.len() < 2 || bytes[..2] != [0xff, 0xd8] { + return (bytes.to_vec(), Vec::new()); + } + let mut output = bytes[..2].to_vec(); + let mut cursor = 2; + let mut changed = BTreeSet::new(); + while cursor + 1 < bytes.len() { + if bytes[cursor] != 0xff { + output.extend_from_slice(&bytes[cursor..]); + break; + } + let marker = bytes[cursor + 1]; + if marker == 0xda || marker == 0xd9 { + output.extend_from_slice(&bytes[cursor..]); + break; + } + if cursor + 4 > bytes.len() { + return (bytes.to_vec(), Vec::new()); + } + let length = u16::from_be_bytes([bytes[cursor + 2], bytes[cursor + 3]]) as usize; + if length < 2 || cursor + 2 + length > bytes.len() { + return (bytes.to_vec(), Vec::new()); + } + let segment_end = cursor + 2 + length; + let payload = &bytes[cursor + 4..segment_end]; + let class = match marker { + 0xe1 if payload.starts_with(b"Exif\0\0") => Some("exif"), + 0xe1 if payload.starts_with(b"http://ns.adobe.com/xap/1.0/") => Some("xmp"), + 0xed => Some("iptc"), + 0xfe => Some("comment"), + _ => None, + }; + if class.is_some_and(|class| should_remove_class(class, policy)) { + changed.insert(class.unwrap().to_string()); + } else { + output.extend_from_slice(&bytes[cursor..segment_end]); + } + cursor = segment_end; + } + (output, changed.into_iter().collect()) +} + +fn sanitize_png(bytes: &[u8], policy: &MetadataHygienePolicy) -> (Vec, Vec) { + const SIGNATURE: &[u8; 8] = b"\x89PNG\r\n\x1a\n"; + if bytes.len() < 8 || &bytes[..8] != SIGNATURE { + return (bytes.to_vec(), Vec::new()); + } + let mut output = bytes[..8].to_vec(); + let mut cursor = 8; + let mut changed = BTreeSet::new(); + while cursor + 12 <= bytes.len() { + let length = u32::from_be_bytes(bytes[cursor..cursor + 4].try_into().unwrap()) as usize; + let end = cursor.saturating_add(12).saturating_add(length); + if end > bytes.len() { + return (bytes.to_vec(), Vec::new()); + } + let chunk = &bytes[cursor + 4..cursor + 8]; + let class = match chunk { + b"eXIf" => Some("exif"), + b"tEXt" | b"zTXt" | b"iTXt" => Some("png.text"), + _ => None, + }; + if class.is_some_and(|class| should_remove_class(class, policy)) { + changed.insert(class.unwrap().to_string()); + } else { + output.extend_from_slice(&bytes[cursor..end]); + } + cursor = end; + if chunk == b"IEND" { + break; + } + } + if cursor < bytes.len() { + output.extend_from_slice(&bytes[cursor..]); + } + (output, changed.into_iter().collect()) +} + +fn scan_secrets( + bytes: &[u8], + media_type: &str, + configured_markers: &[String], + blocking: bool, +) -> Vec { + let Some(text) = scannable_text(bytes, media_type) else { + return Vec::new(); + }; + let lower = text.to_ascii_lowercase(); + let mut classes = BTreeSet::new(); + if lower.contains("-----begin private key-----") + || lower.contains("-----begin rsa private key-----") + { + classes.insert("private_key".to_string()); + } + if has_prefixed_token(&lower, "github_pat_", 12) || has_prefixed_token(&lower, "ghp_", 20) { + classes.insert("github_token".to_string()); + } + if has_prefixed_token(text, "AKIA", 16) { + classes.insert("aws_access_key".to_string()); + } + if has_prefixed_token(&lower, "sk-", 20) { + classes.insert("openai_key".to_string()); + } + if has_credential_assignment(&lower, "authorization: bearer ", 8) { + classes.insert("authorization".to_string()); + } + if ["password=", "api_key=", "api-key="] + .iter() + .any(|marker| has_credential_assignment(&lower, marker, 6)) + { + classes.insert("credential_assignment".to_string()); + } + for marker in configured_markers { + if !marker.is_empty() && text.contains(marker) { + classes.insert("configured_secret".to_string()); + } + } + classes + .into_iter() + .map(|class| HygieneFinding { + code: "hygiene.secret.detected".to_string(), + kind: HygieneFindingKind::Secret, + class, + message: "Potential credential detected; the value was omitted from evidence" + .to_string(), + blocking, + }) + .collect() +} + +fn has_prefixed_token(text: &str, prefix: &str, minimum_tail: usize) -> bool { + text.match_indices(prefix).any(|(offset, _)| { + text[offset + prefix.len()..] + .chars() + .take_while(|character| { + character.is_ascii_alphanumeric() || matches!(character, '_' | '-') + }) + .count() + >= minimum_tail + }) +} + +fn has_credential_assignment(text: &str, marker: &str, minimum_value: usize) -> bool { + text.match_indices(marker).any(|(offset, _)| { + text[offset + marker.len()..] + .trim_start_matches(['\'', '"']) + .chars() + .take_while(|character| { + !character.is_ascii_whitespace() && !matches!(character, '\'' | '"' | ',' | ';') + }) + .count() + >= minimum_value + }) +} + +fn scan_protected_references( + bytes: &[u8], + media_type: &str, + terms: &[String], + case_sensitive: bool, + blocking: bool, +) -> Vec { + let Some(text) = scannable_text(bytes, media_type) else { + return Vec::new(); + }; + let haystack = if case_sensitive { + text.to_string() + } else { + text.to_lowercase() + }; + terms + .iter() + .filter(|term| { + let needle = if case_sensitive { + (*term).clone() + } else { + term.to_lowercase() + }; + !needle.is_empty() && haystack.contains(&needle) + }) + .map(|term| HygieneFinding { + code: "hygiene.protected_reference.detected".to_string(), + kind: HygieneFindingKind::ProtectedReference, + class: "configured_term".to_string(), + message: format!( + "Configured protected reference '{}' requires removal or explicit review", + term + ), + blocking, + }) + .collect() +} + +fn scannable_text<'a>(bytes: &'a [u8], media_type: &str) -> Option<&'a str> { + let textual = media_type.starts_with("text/") + || matches!( + media_type, + "application/json" + | "application/yaml" + | "application/toml" + | "application/xml" + | "application/x-latex" + ); + textual.then(|| std::str::from_utf8(bytes).ok()).flatten() +} + +fn rights_findings(policy: &HygienePolicy) -> Vec { + if !policy.rights.required + || matches!( + policy.audience, + PublicationAudience::Candidate | PublicationAudience::Private + ) + { + return Vec::new(); + } + let complete = policy.rights.reviewed + && policy + .rights + .license + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + && policy + .rights + .approval_reference + .as_deref() + .is_some_and(|value| !value.trim().is_empty()); + if complete { + Vec::new() + } else { + vec![HygieneFinding { + code: "hygiene.rights.review_required".to_string(), + kind: HygieneFindingKind::Rights, + class: "publication_rights".to_string(), + message: "Public/commercial release requires reviewed rights, license, and approval metadata; the local candidate was preserved".to_string(), + blocking: true, + }] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::artifact::{ArtifactDescriptor, ArtifactStorageClass}; + use crate::graph::Format; + use crate::spec::{ + MetadataHygienePolicy, ProtectedReferencePolicy, RightsHygienePolicy, SecretHygienePolicy, + }; + + #[test] + fn metadata_sanitization_is_allowlisted_and_non_destructive() { + let directory = tempfile::tempdir().unwrap(); + let store = ArtifactStore::new(directory.path()).unwrap(); + let source = store + .put_bytes( + b"publication candidate", + ArtifactDescriptor::for_format(Format::Markdown, ArtifactStorageClass::Source) + .with_metadata("dc.title", "Allowed title") + .with_metadata("dc.creator", "Private author") + .with_metadata("geolocation.latitude", "42.0"), + ) + .unwrap(); + let policy = HygienePolicy { + metadata: MetadataHygienePolicy { + allow: vec!["dc.title".to_string()], + deny: vec!["geolocation.*".to_string()], + allowlist_only: true, + }, + ..HygienePolicy::default() + }; + + let outcome = HygieneEngine::new() + .apply("public", &policy, &source, &store) + .unwrap(); + + assert_eq!( + source.metadata().get("dc.creator").unwrap(), + "Private author" + ); + assert_eq!(store.read_bytes(&source).unwrap(), b"publication candidate"); + assert_eq!( + outcome.artifact.metadata().get("dc.title").unwrap(), + "Allowed title" + ); + assert!(!outcome.artifact.metadata().contains_key("dc.creator")); + assert!(!outcome + .artifact + .metadata() + .contains_key("geolocation.latitude")); + assert!(outcome + .evidence + .changed_field_classes + .contains(&"geolocation".to_string())); + } + + #[test] + fn jpeg_exif_is_removed_into_a_derived_artifact() { + let directory = tempfile::tempdir().unwrap(); + let store = ArtifactStore::new(directory.path()).unwrap(); + let jpeg = [ + 0xff, 0xd8, 0xff, 0xe1, 0x00, 0x08, b'E', b'x', b'i', b'f', 0x00, 0x00, 0xff, 0xd9, + ]; + let source = store + .put_bytes( + &jpeg, + ArtifactDescriptor::for_format(Format::Jpeg, ArtifactStorageClass::Source), + ) + .unwrap(); + let policy = HygienePolicy { + metadata: MetadataHygienePolicy { + deny: vec!["exif".to_string()], + ..MetadataHygienePolicy::default() + }, + ..HygienePolicy::default() + }; + + let outcome = HygieneEngine::new() + .apply("images", &policy, &source, &store) + .unwrap(); + + assert_eq!(store.read_bytes(&source).unwrap(), jpeg); + assert_eq!( + store.read_bytes(&outcome.artifact).unwrap(), + [0xff, 0xd8, 0xff, 0xd9] + ); + assert_eq!(outcome.evidence.changed_field_classes, vec!["exif"]); + assert_eq!( + outcome.artifact.sources(), + std::slice::from_ref(source.id()) + ); + } + + #[test] + fn secrets_references_and_rights_block_without_leaking_secret_values() { + let directory = tempfile::tempdir().unwrap(); + let store = ArtifactStore::new(directory.path()).unwrap(); + let source = store + .put_bytes( + b"Example Franchise with FIXTURE-CREDENTIAL-MARKER", + ArtifactDescriptor::for_format(Format::Markdown, ArtifactStorageClass::Source), + ) + .unwrap(); + let policy = HygienePolicy { + audience: PublicationAudience::Commercial, + secrets: SecretHygienePolicy { + markers: vec!["FIXTURE-CREDENTIAL-MARKER".to_string()], + ..SecretHygienePolicy::default() + }, + protected_references: ProtectedReferencePolicy { + terms: vec!["Example Franchise".to_string()], + ..ProtectedReferencePolicy::default() + }, + rights: RightsHygienePolicy { + required: true, + ..RightsHygienePolicy::default() + }, + ..HygienePolicy::default() + }; + + let outcome = HygieneEngine::new() + .apply("commercial", &policy, &source, &store) + .unwrap(); + + assert_eq!(outcome.evidence.status, HygieneStatus::Blocked); + let evidence = serde_json::to_string(&outcome.evidence).unwrap(); + assert!(!evidence.contains("FIXTURE-CREDENTIAL-MARKER")); + assert!(outcome + .evidence + .findings + .iter() + .any(|finding| finding.kind == HygieneFindingKind::ProtectedReference)); + assert!(outcome + .evidence + .findings + .iter() + .any(|finding| finding.kind == HygieneFindingKind::Rights)); + } +} diff --git a/crates/renderflow-core/src/lib.rs b/crates/renderflow-core/src/lib.rs index d1c90e4..90e2d0a 100644 --- a/crates/renderflow-core/src/lib.rs +++ b/crates/renderflow-core/src/lib.rs @@ -21,6 +21,7 @@ pub mod detect; pub mod error; pub mod evidence; pub mod graph; +pub mod hygiene; mod image; mod input_format; pub mod intake; @@ -37,6 +38,11 @@ pub mod transforms; pub mod validation; pub use evidence::{ArtifactManifest, RunManifest}; +pub use hygiene::{ + ContentRedactionProvider, HygieneEngine, HygieneEvidence, HygieneFinding, HygieneFindingKind, + HygieneOutcome, HygieneStatus, RedactionEvidence, RedactionRequest, RedactionResult, + HYGIENE_EVIDENCE_SCHEMA_V1, +}; pub use intake::{ ArtifactIntakeProvider, DetectionConfidence, DiscoveredArtifact, EvidenceOrigin, InspectionContext, IntakeBudgetUsage, IntakeBudgets, IntakeConflict, IntakeDiagnostic, diff --git a/crates/renderflow-core/src/planning.rs b/crates/renderflow-core/src/planning.rs index ac8cb81..698e4f7 100644 --- a/crates/renderflow-core/src/planning.rs +++ b/crates/renderflow-core/src/planning.rs @@ -28,11 +28,13 @@ use crate::graph::{ DagExecutionReport, DagExecutor, DiagnosticLevel, ExecutionPlan, Format, MultiTargetDag, TransformEdge, TransformGraph, }; +use crate::hygiene::{HygieneEngine, HygieneEvidence}; use crate::intake::{IntakeEngine, IntakeRequest, ResolvedArtifactProfile}; use crate::optimization::OptimizationMode; use crate::spec::{ - load_spec, AiPolicy, CollisionPolicy, RejectedLossClass, SelectorSet, SourceKind, SourceSpec, - SourceSpecVersion, SpecV2, TargetSelection, TargetSpec, ValidationFailureMode, + load_spec, AiPolicy, CollisionPolicy, HygienePolicy, RejectedLossClass, SelectorSet, + SourceKind, SourceSpec, SourceSpecVersion, SpecV2, TargetSelection, TargetSpec, + ValidationFailureMode, }; use crate::super_resolution::{select_upscayl_variants, UpscaylModelCatalog}; use crate::toolchain::{ @@ -201,6 +203,35 @@ impl CanonicalExecutionResult { } } +fn effective_hygiene_policy(spec: &SpecV2) -> Result> { + let policy_id = if let Some(policy_id) = &spec.execution.hygiene_policy { + Some(policy_id.clone()) + } else { + let profile_policies = spec + .targets + .profiles + .iter() + .filter_map(|profile_id| spec.profiles.get(profile_id)) + .filter_map(|profile| profile.hygiene_policy.as_deref()) + .collect::>(); + match profile_policies.len() { + 0 => None, + 1 => profile_policies.iter().next().map(|value| (*value).to_string()), + _ => anyhow::bail!( + "selected derivative profiles declare conflicting hygiene policies; set execution.hygiene_policy explicitly" + ), + } + }; + policy_id + .map(|policy_id| { + let policy = spec.hygiene.get(&policy_id).cloned().with_context(|| { + format!("hygiene policy '{policy_id}' is not declared in the specification") + })?; + Ok((policy_id, policy)) + }) + .transpose() +} + pub fn resolve(request: PlanningRequest) -> Result { let config_path = request .config_path @@ -566,11 +597,75 @@ pub fn execute(mut resolved: ResolvedExecution, dry_run: bool) -> Result::new(); + let mut hygiene_blocked = HashSet::::new(); + let mut hygiene_failures = false; + if let Some((policy_id, policy)) = effective_hygiene_policy(&resolved.spec)? { + let engine = HygieneEngine::new(); + let mut target_formats = resolved + .targets + .iter() + .map(|target| target.format) + .collect::>(); + target_formats.sort_by_key(ToString::to_string); + target_formats.dedup(); + for format in target_formats { + let Some(candidate) = report.artifacts.get(&format).cloned() else { + continue; + }; + let started = unix_time_ms(); + let outcome = engine.apply(&policy_id, &policy, &candidate, &store)?; + let completed = unix_time_ms(); + let step_id = format!("hygiene:{policy_id}:{format}"); + for finding in &outcome.evidence.findings { + diagnostics.push(ExecutionDiagnostic { + severity: if finding.blocking { + DiagnosticSeverity::RecoverableFailure + } else { + DiagnosticSeverity::Warning + }, + code: finding.code.clone(), + message: finding.message.clone(), + step_id: Some(step_id.clone()), + }); + } + if !outcome.releasable() { + hygiene_failures = true; + hygiene_blocked.insert(outcome.artifact.id().to_string()); + } + let changed = !outcome.evidence.changed_field_classes.is_empty(); + report.steps.push(StepEvidence { + step_id, + transform: "publication.hygiene".to_string(), + transform_version: env!("CARGO_PKG_VERSION").to_string(), + capability: Some("artifact.publication-hygiene".to_string()), + provider: Some("renderflow.core-hygiene".to_string()), + input_artifacts: vec![candidate.id().to_string()], + output_artifacts: vec![outcome.artifact.id().to_string()], + configuration_digest: sha256_serialized(&policy)?, + started_at_unix_ms: started, + completed_at_unix_ms: completed, + duration_ms: completed.saturating_sub(started), + state: StepState::Complete, + cache: crate::evidence::CacheDisposition::Miss, + validation: ValidationState::NotRequested, + fidelity: if changed { + FidelityDeclaration::Partial + } else { + FidelityDeclaration::Lossless + }, + skip_reason: None, + diagnostics: Vec::new(), + }); + hygiene_evidence.insert(outcome.artifact.id().to_string(), outcome.evidence.clone()); + report.artifacts.insert(format, outcome.artifact); + } + } let mut output_locators = HashMap::::new(); let mut validation_outcomes = HashMap::::new(); - let mut blocked_artifacts = HashSet::::new(); + let mut blocked_artifacts = hygiene_blocked; let mut actual_outputs = Vec::new(); - let mut target_failures = false; + let mut target_failures = hygiene_failures; let mut fatal_validation_failure = false; if let Err(error) = validate_post_execution_budgets(&resolved, &report.artifacts) { @@ -718,6 +813,7 @@ pub fn execute(mut resolved: ResolvedExecution, dry_run: bool) -> Result, diagnostics: &[ExecutionDiagnostic], validation_outcomes: &HashMap, + hygiene_evidence: &HashMap, ) -> Vec { let mut evidence = vec![source_artifact_evidence(resolved, source)]; let mut artifacts = report.artifacts.iter().collect::>(); @@ -1044,6 +1141,7 @@ fn artifact_evidence( artifact_evidence.validation_evidence = outcome .map(|outcome| outcome.validators.clone()) .unwrap_or_default(); + artifact_evidence.hygiene = hygiene_evidence.get(artifact.id().as_str()).cloned(); if let Some(step) = producing_step { artifact_evidence.warnings = diagnostics .iter() @@ -1887,6 +1985,7 @@ mod tests { immutable: true, }], profiles: BTreeMap::new(), + hygiene: BTreeMap::new(), targets: TargetSelection::default(), execution: ExecutionPolicy::default(), output: OutputLayout::default(), diff --git a/crates/renderflow-core/src/sdk.rs b/crates/renderflow-core/src/sdk.rs index 8d64a04..f10ae0d 100644 --- a/crates/renderflow-core/src/sdk.rs +++ b/crates/renderflow-core/src/sdk.rs @@ -194,6 +194,7 @@ pub struct ProviderCapabilities { pub checkpoint_schema: String, pub artifact_contract: String, pub intake_schema: String, + pub hygiene_schema: String, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -572,6 +573,7 @@ impl Engine { "inspect_capabilities".to_string(), "inspect_artifact".to_string(), "extract_artifacts".to_string(), + "publication_hygiene".to_string(), "plan".to_string(), "run".to_string(), "assess".to_string(), @@ -582,6 +584,7 @@ impl Engine { checkpoint_schema: crate::checkpoint::CHECKPOINT_SCHEMA_V1.to_string(), artifact_contract: crate::evidence::FLOW_ARTIFACT_SCHEMA_V1.to_string(), intake_schema: crate::intake::INTAKE_SCHEMA_V1.to_string(), + hygiene_schema: crate::hygiene::HYGIENE_EVIDENCE_SCHEMA_V1.to_string(), } } diff --git a/crates/renderflow-core/src/spec.rs b/crates/renderflow-core/src/spec.rs index 83fc459..c737ff3 100644 --- a/crates/renderflow-core/src/spec.rs +++ b/crates/renderflow-core/src/spec.rs @@ -156,6 +156,140 @@ pub struct DerivativeProfile { pub include: SelectorSet, #[serde(default)] pub exclude: SelectorSet, + /// Named publication-hygiene policy applied to artifacts selected by this profile. + #[serde(default)] + pub hygiene_policy: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PublicationAudience { + #[default] + Candidate, + Private, + Public, + Commercial, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RedactionDeterminism { + #[default] + Deterministic, + Probabilistic, +} + +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetadataHygienePolicy { + /// Metadata keys intentionally retained in publication artifacts. + #[serde(default)] + pub allow: Vec, + /// Metadata keys or field classes removed from publication artifacts. + #[serde(default)] + pub deny: Vec, + /// Remove all non-allowlisted metadata rather than only explicit deny entries. + #[serde(default)] + pub allowlist_only: bool, +} + +fn default_hygiene_enabled() -> bool { + true +} + +fn default_block() -> bool { + true +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SecretHygienePolicy { + #[serde(default = "default_hygiene_enabled")] + pub enabled: bool, + #[serde(default = "default_block")] + pub block: bool, + /// Additional literal markers. Values are never included in diagnostics or evidence. + #[serde(default)] + pub markers: Vec, +} + +impl Default for SecretHygienePolicy { + fn default() -> Self { + Self { + enabled: true, + block: true, + markers: Vec::new(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProtectedReferencePolicy { + /// Brand, franchise, company, creator, or work names requiring review. + #[serde(default)] + pub terms: Vec, + #[serde(default = "default_block")] + pub block: bool, + #[serde(default)] + pub case_sensitive: bool, +} + +impl Default for ProtectedReferencePolicy { + fn default() -> Self { + Self { + terms: Vec::new(), + block: true, + case_sensitive: false, + } + } +} + +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContentRedactionPolicy { + /// Provider-neutral redaction provider identifier. + pub provider: String, + #[serde(default)] + pub determinism: RedactionDeterminism, + #[serde(default)] + pub classes: Vec, + /// Explicit human approval for this exact policy configuration. + #[serde(default)] + pub reviewed: bool, +} + +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RightsHygienePolicy { + #[serde(default)] + pub required: bool, + #[serde(default)] + pub license: Option, + #[serde(default)] + pub rights_holder: Option, + #[serde(default)] + pub approval_reference: Option, + /// Records an explicit rights review; it is not a legal conclusion by Renderflow. + #[serde(default)] + pub reviewed: bool, +} + +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HygienePolicy { + #[serde(default)] + pub audience: PublicationAudience, + #[serde(default)] + pub metadata: MetadataHygienePolicy, + #[serde(default)] + pub secrets: SecretHygienePolicy, + #[serde(default)] + pub protected_references: ProtectedReferencePolicy, + #[serde(default)] + pub redaction: Option, + #[serde(default)] + pub rights: RightsHygienePolicy, } #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] @@ -283,6 +417,9 @@ pub struct ExecutionPolicy { pub publication_policy: Option, #[serde(default)] pub redaction_policy: Option, + /// Named hygiene policy applied to the complete selected publication bundle. + #[serde(default)] + pub hygiene_policy: Option, } impl Default for ExecutionPolicy { @@ -303,6 +440,7 @@ impl Default for ExecutionPolicy { reject_loss_classes: Vec::new(), publication_policy: None, redaction_policy: None, + hygiene_policy: None, } } } @@ -344,6 +482,8 @@ pub struct SpecV2 { pub sources: Vec, #[serde(default)] pub profiles: BTreeMap, + #[serde(default)] + pub hygiene: BTreeMap, pub targets: TargetSelection, #[serde(default)] pub execution: ExecutionPolicy, @@ -540,6 +680,22 @@ impl SpecV2 { )); } } + if self.execution.hygiene_policy.is_none() { + let selected_hygiene = self + .targets + .profiles + .iter() + .filter_map(|profile_name| self.profiles.get(profile_name)) + .filter_map(|profile| profile.hygiene_policy.as_deref()) + .collect::>(); + if selected_hygiene.len() > 1 { + diagnostics.push(SpecDiagnostic::new( + "$.targets.profiles", + "hygiene.policy.conflict", + "selected profiles use different hygiene policies; choose one with execution.hygiene_policy", + )); + } + } for (profile_name, profile) in &self.profiles { let base = format!("$.profiles.{profile_name}"); @@ -572,6 +728,70 @@ impl SpecV2 { &format!("{base}.exclude"), &mut diagnostics, ); + if let Some(policy) = &profile.hygiene_policy { + if !self.hygiene.contains_key(policy) { + diagnostics.push(SpecDiagnostic::new( + format!("{base}.hygiene_policy"), + "hygiene.policy.unknown", + format!("hygiene policy '{policy}' is not declared in $.hygiene"), + )); + } + } + } + + if let Some(policy) = &self.execution.hygiene_policy { + if !self.hygiene.contains_key(policy) { + diagnostics.push(SpecDiagnostic::new( + "$.execution.hygiene_policy", + "hygiene.policy.unknown", + format!("hygiene policy '{policy}' is not declared in $.hygiene"), + )); + } + } + + for (policy_name, policy) in &self.hygiene { + let base = format!("$.hygiene.{policy_name}"); + if !is_stable_id(policy_name) { + diagnostics.push(SpecDiagnostic::new( + base.clone(), + "hygiene.id.invalid", + "hygiene policy names must use only ASCII letters, digits, '.', '_', or '-'", + )); + } + validate_non_empty_values( + &policy.metadata.allow, + &format!("{base}.metadata.allow"), + &mut diagnostics, + ); + validate_non_empty_values( + &policy.metadata.deny, + &format!("{base}.metadata.deny"), + &mut diagnostics, + ); + validate_non_empty_values( + &policy.secrets.markers, + &format!("{base}.secrets.markers"), + &mut diagnostics, + ); + validate_non_empty_values( + &policy.protected_references.terms, + &format!("{base}.protected_references.terms"), + &mut diagnostics, + ); + if let Some(redaction) = &policy.redaction { + if redaction.provider.trim().is_empty() { + diagnostics.push(SpecDiagnostic::new( + format!("{base}.redaction.provider"), + "hygiene.redaction.provider_empty", + "a configured redaction policy requires a provider id", + )); + } + validate_non_empty_values( + &redaction.classes, + &format!("{base}.redaction.classes"), + &mut diagnostics, + ); + } } if self.execution.max_parallel == 0 { @@ -727,6 +947,18 @@ fn validate_allow_deny( } } +fn validate_non_empty_values(values: &[String], path: &str, diagnostics: &mut Vec) { + for (index, value) in values.iter().enumerate() { + if value.trim().is_empty() { + diagnostics.push(SpecDiagnostic::new( + format!("{path}[{index}]"), + "hygiene.value.empty", + "hygiene policy values must not be empty", + )); + } + } +} + fn is_stable_id(value: &str) -> bool { !value.is_empty() && value @@ -952,6 +1184,7 @@ pub(crate) fn migrate_v1_config(config: &Config) -> SpecV2 { immutable: true, }], profiles: BTreeMap::new(), + hygiene: BTreeMap::new(), targets: TargetSelection { exact, profiles: Vec::new(), @@ -990,6 +1223,11 @@ pub fn json_schema() -> Value { "additionalProperties": {"$ref": "#/$defs/profile"}, "default": {} }, + "hygiene": { + "type": "object", + "additionalProperties": {"$ref": "#/$defs/hygienePolicy"}, + "default": {} + }, "targets": {"$ref": "#/$defs/targetSelection"}, "execution": {"$ref": "#/$defs/executionPolicy"}, "output": {"$ref": "#/$defs/outputLayout"}, @@ -1071,7 +1309,63 @@ pub fn json_schema() -> Value { "description": {"type": ["string", "null"]}, "targets": {"type": "array", "items": {"$ref": "#/$defs/target"}, "default": []}, "include": {"$ref": "#/$defs/selectorSet"}, - "exclude": {"$ref": "#/$defs/selectorSet"} + "exclude": {"$ref": "#/$defs/selectorSet"}, + "hygiene_policy": {"anyOf": [{"$ref": "#/$defs/stableId"}, {"type": "null"}]} + } + }, + "hygienePolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "audience": {"enum": ["candidate", "private", "public", "commercial"], "default": "candidate"}, + "metadata": {"$ref": "#/$defs/metadataHygiene"}, + "secrets": {"$ref": "#/$defs/secretHygiene"}, + "protected_references": {"$ref": "#/$defs/protectedReferences"}, + "redaction": {"anyOf": [{"$ref": "#/$defs/contentRedaction"}, {"type": "null"}]}, + "rights": {"$ref": "#/$defs/rightsHygiene"} + } + }, + "metadataHygiene": { + "type": "object", "additionalProperties": false, + "properties": { + "allow": {"type": "array", "items": {"type": "string", "minLength": 1}, "default": []}, + "deny": {"type": "array", "items": {"type": "string", "minLength": 1}, "default": []}, + "allowlist_only": {"type": "boolean", "default": false} + } + }, + "secretHygiene": { + "type": "object", "additionalProperties": false, + "properties": { + "enabled": {"type": "boolean", "default": true}, + "block": {"type": "boolean", "default": true}, + "markers": {"type": "array", "items": {"type": "string", "minLength": 1}, "default": []} + } + }, + "protectedReferences": { + "type": "object", "additionalProperties": false, + "properties": { + "terms": {"type": "array", "items": {"type": "string", "minLength": 1}, "default": []}, + "block": {"type": "boolean", "default": true}, + "case_sensitive": {"type": "boolean", "default": false} + } + }, + "contentRedaction": { + "type": "object", "additionalProperties": false, "required": ["provider"], + "properties": { + "provider": {"type": "string", "minLength": 1}, + "determinism": {"enum": ["deterministic", "probabilistic"], "default": "deterministic"}, + "classes": {"type": "array", "items": {"type": "string", "minLength": 1}, "default": []}, + "reviewed": {"type": "boolean", "default": false} + } + }, + "rightsHygiene": { + "type": "object", "additionalProperties": false, + "properties": { + "required": {"type": "boolean", "default": false}, + "license": {"type": ["string", "null"]}, + "rights_holder": {"type": ["string", "null"]}, + "approval_reference": {"type": ["string", "null"]}, + "reviewed": {"type": "boolean", "default": false} } }, "allowDeny": { @@ -1134,7 +1428,8 @@ pub fn json_schema() -> Value { "default": [] }, "publication_policy": {"type": ["string", "null"]}, - "redaction_policy": {"type": ["string", "null"]} + "redaction_policy": {"type": ["string", "null"]}, + "hygiene_policy": {"anyOf": [{"$ref": "#/$defs/stableId"}, {"type": "null"}]} } }, "outputLayout": { diff --git a/crates/renderflow-core/src/super_resolution.rs b/crates/renderflow-core/src/super_resolution.rs index d60cefe..ced2d7e 100644 --- a/crates/renderflow-core/src/super_resolution.rs +++ b/crates/renderflow-core/src/super_resolution.rs @@ -1121,6 +1121,7 @@ mod tests { immutable: true, }], profiles: BTreeMap::new(), + hygiene: BTreeMap::new(), targets: TargetSelection::default(), execution: ExecutionPolicy { ai: AiPolicy::LocalOnly, diff --git a/docs/execution-evidence.md b/docs/execution-evidence.md index b82b282..9c7aa68 100644 --- a/docs/execution-evidence.md +++ b/docs/execution-evidence.md @@ -24,6 +24,8 @@ The artifact manifest contains source, retained intermediate, and terminal artif Terminal artifacts are validated before materialization. A required invalid artifact is never published; unavailable validation also blocks publication unless the spec explicitly allows it. When validation is disabled, the terminal state is recorded as `skipped` rather than inferred as valid. +When a publication-hygiene policy is selected, the generated candidate passes through a `publication.hygiene` step before terminal validation and materialization. Artifact evidence embeds the policy/provider identity, source and sanitized artifact IDs, changed metadata field classes, safe findings, and a `passed`, `review_required`, or `blocked` decision. Blocked candidates retain an `artifact-store:` locator but never receive a `bundle:` locator. See [Publication hygiene](publication-hygiene.md). + Each executed DAG edge produces step evidence with transform/capability/provider identity, input and output artifact IDs, a configuration digest, timestamps, duration, cache disposition, validation and fidelity states, and structured diagnostics. Cache hits use `state: reused`; transforms blocked by a failed dependency use `state: skipped` with a reason. The machine-readable contract is [`schemas/renderflow-run-v1.schema.json`](https://github.com/egohygiene/renderflow/blob/main/schemas/renderflow-run-v1.schema.json). @@ -36,4 +38,4 @@ Native IDs such as `artifact:sha256:` are mapped deterministically to Fl ## Sensitive data boundary -Run manifests contain digests of the resolved plan and source spec, not serialized configuration or environment variables. Step configuration is represented only by a SHA-256 digest. Artifact locators are relative `artifact-store:` or `bundle:` locators. Provider diagnostics are retained for operability, so provider implementations must not place credentials or secret values in error messages. +Run manifests contain digests of the resolved plan and source spec, not serialized configuration or environment variables. Step configuration is represented only by a SHA-256 digest. Artifact locators are relative `artifact-store:` or `bundle:` locators. Provider diagnostics are retained for operability, so provider implementations must not place credentials or secret values in error messages. Hygiene secret findings identify only a safe credential class; the matched value is never written to evidence. diff --git a/docs/provider-contract.md b/docs/provider-contract.md index 65c1c62..f818a4a 100644 --- a/docs/provider-contract.md +++ b/docs/provider-contract.md @@ -2,7 +2,7 @@ Renderflow exposes a versioned Rust provider seam for orchestration systems such as Flow. Renderflow owns transform execution and artifact validation; the caller remains responsible for cross-provider orchestration and recovery policy. -`RenderflowProvider` exposes structured operations for capability inspection, planning, execution, checkpoint assessment, and resume. `Engine::inspect_artifact` adds arbitrary-file inspection and provider-driven extraction through `renderflow.intake/v1`; the capability response advertises that schema and the `inspect_artifact` / `extract_artifacts` operations. The provider contract uses `renderflow.provider/v1`; progress callbacks use `renderflow.progress/v1`. Consumers must use these serialized models rather than human stdout or stderr. +`RenderflowProvider` exposes structured operations for capability inspection, planning, execution, checkpoint assessment, and resume. `Engine::inspect_artifact` adds arbitrary-file inspection and provider-driven extraction through `renderflow.intake/v1`; the capability response advertises that schema and the `inspect_artifact` / `extract_artifacts` operations. Publication-aware providers also advertise `publication_hygiene` and `renderflow.hygiene/v1`. The provider contract uses `renderflow.provider/v1`; progress callbacks use `renderflow.progress/v1`. Consumers must use these serialized models rather than human stdout or stderr. ```rust use renderflow::{EngineBuilder, ExecutionRequest, RenderflowProvider}; @@ -39,5 +39,6 @@ Progress events include their schema version and, when available, run ID, step I - `schemas/renderflow-provider-v1.schema.json` describes provider results and events. - `schemas/renderflow-checkpoints-v1.schema.json` describes durable checkpoint state. - `schemas/renderflow-intake-v1.schema.json` describes universal input identity, detection, inspection, and extracted-child evidence. +- `schemas/renderflow-hygiene-v1.schema.json` describes non-destructive publication-hygiene decisions and safe findings. - `schemas/renderflow-run-v1.schema.json` describes authoritative run evidence. - `RunManifest::flow_artifacts_v1()` projects outputs into `flow.artifact/v1` without importing Flow source. diff --git a/docs/publication-hygiene.md b/docs/publication-hygiene.md new file mode 100644 index 0000000..20f9516 --- /dev/null +++ b/docs/publication-hygiene.md @@ -0,0 +1,86 @@ +# Publication hygiene + +Renderflow can apply a named publication-hygiene policy after a derivative is generated and before it is materialized into a release bundle. The stage is non-destructive: canonical sources remain immutable, and sanitation creates a new content-addressed artifact whose evidence points to the unsanitized candidate. + +Hygiene is opt-in. A policy can be selected for a complete execution or attached to a derivative profile: + +```yaml +schema: renderflow/v2 +sources: + - id: source.comic + path: comic.md +profiles: + publication.comic: + hygiene_policy: public-comic + targets: + - role: web + format: html + - role: print + format: pdf +hygiene: + public-comic: + audience: commercial + metadata: + allow: + - dc.title + - dc.creator + deny: + - exif + - xmp + - iptc + - geolocation + - filesystem.* + secrets: + enabled: true + block: true + protected_references: + terms: + - Example Game Studio + - Example Post-Apocalypse Franchise + block: true + rights: + required: true + license: All rights reserved + rights_holder: Example Publisher LLC + approval_reference: rights-review-2026-09 + reviewed: true +targets: + profiles: + - publication.comic +``` + +Set `execution.hygiene_policy` to override profile policy selection for the whole requested bundle. If several selected profiles name different policies, Renderflow requires an explicit execution-level choice. + +## Four separate controls + +| Control | Purpose | Decision model | +| --- | --- | --- | +| Metadata sanitization | Removes configured record metadata and supported embedded JPEG/PNG metadata classes while preserving allowlisted publication fields. | Deterministic transform | +| Secret scanning | Detects likely private keys, tokens, authorization values, credential assignments, and configured secret markers. Findings never reproduce the detected value. | Deterministic publication gate | +| Content redaction | Delegates configured PII or content classes to a replaceable provider. | Deterministic or probabilistic transform | +| Rights review | Requires explicit license and approval evidence for public/commercial publication. | Human policy gate, not a legal conclusion | + +Protected-reference scanning is an additional deterministic gate for configured brands, franchises, companies, creators, or works. It is useful for catching a named influence that leaked from an internal prompt into generated text or metadata. A match means “remove or review this configured reference”; it does not mean Renderflow has detected copyright infringement or made a legal determination. + +## Metadata behavior + +`metadata.allow` and `metadata.deny` accept exact field names and namespace wildcards such as `filesystem.*`. With `allowlist_only: true`, all artifact-record metadata outside the allowlist is removed. Embedded metadata sanitation currently supports these field classes where structurally safe: + +- JPEG EXIF, XMP, IPTC, and comment segments; +- PNG EXIF and textual chunks. + +Other formats remain provider-extensible. Unsupported embedded formats are preserved rather than rewritten unsafely. + +Evidence records only changed field classes such as `exif` or `geolocation`; removed values are never copied into the run manifest. + +## Blocking and candidate preservation + +Secret findings, configured protected references, unavailable redaction providers, and incomplete public/commercial rights evidence can block materialization. The candidate remains in Renderflow's local content-addressed artifact store and appears in the run manifest with an `artifact-store:` locator and `blocked` hygiene evidence. It is not copied into the publication bundle. + +Probabilistic redaction always produces `review_required` status for the resulting candidate. Configuring or reviewing a provider is not treated as approval of a particular probabilistically redacted output. + +## Provider contract + +SDK integrations implement `ContentRedactionProvider` and declare a stable provider ID, version, and determinism class. Providers receive bytes plus configured content classes and return new bytes, changed classes, and safe findings. They must not mutate source files or include sensitive values in diagnostics. + +Hygiene evidence uses [`renderflow.hygiene/v1`](../schemas/renderflow-hygiene-v1.schema.json) and is embedded in terminal artifact evidence. A `publication.hygiene` step records the policy digest, provider, input candidate, sanitized output, duration, and fidelity. diff --git a/docs/user-guide/spec-v2-reference.md b/docs/user-guide/spec-v2-reference.md index c3cfa87..37b31e6 100644 --- a/docs/user-guide/spec-v2-reference.md +++ b/docs/user-guide/spec-v2-reference.md @@ -13,6 +13,7 @@ Spec v2 describes source intent, derivative selection, execution policy, and det | Field | Type | Required | Default | | --- | --- | --- | --- | | `execution` | `executionPolicy` | no | — | +| `hygiene` | `object` | no | `{}` | | `output` | `outputLayout` | no | — | | `profiles` | `object` | no | `{}` | | `schema` | `"renderflow/v2"` | yes | — | @@ -59,6 +60,7 @@ Spec v2 describes source intent, derivative selection, execution policy, and det | `optimization` | `speed` / `quality` / `balanced` / `pareto` | no | `"balanced"` | | `publication_policy` | `string` / `null` | no | — | | `redaction_policy` | `string` / `null` | no | — | +| `hygiene_policy` | `object` | no | — | | `reject_loss_classes` | `array` | no | `[]` | | `requirements` | `requirements` | no | — | | `retry_policy` | `string` / `null` | no | — | diff --git a/examples/publication-hygiene.yaml b/examples/publication-hygiene.yaml new file mode 100644 index 0000000..60e1aa7 --- /dev/null +++ b/examples/publication-hygiene.yaml @@ -0,0 +1,45 @@ +schema: renderflow/v2 +sources: + - id: source.manuscript + role: manuscript + path: manuscript.md + format: markdown +profiles: + publication.reviewed: + description: Public derivatives with deterministic hygiene and rights gates + hygiene_policy: public-release + targets: + - id: target.web + role: web + format: html +hygiene: + public-release: + audience: public + metadata: + allow: + - dc.title + - dc.creator + deny: + - exif + - xmp + - iptc + - geolocation + - filesystem.* + secrets: + enabled: true + block: true + protected_references: + terms: + - Example Franchise + block: true + rights: + required: true + license: All rights reserved + rights_holder: Example Publisher LLC + approval_reference: rights-review-example + reviewed: true +targets: + profiles: + - publication.reviewed +output: + bundle_root: dist diff --git a/mkdocs.yml b/mkdocs.yml index 2c98c5f..1dd5660 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -82,6 +82,7 @@ nav: - FAQ: user-guide/faq.md - Architecture: - Universal Intake: universal-intake.md + - Publication Hygiene: publication-hygiene.md - Overview: architecture/overview.md - Knowledge Base: - Identity: diff --git a/schemas/renderflow-hygiene-v1.schema.json b/schemas/renderflow-hygiene-v1.schema.json new file mode 100644 index 0000000..95f2514 --- /dev/null +++ b/schemas/renderflow-hygiene-v1.schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://egohygiene.github.io/renderflow/schemas/renderflow-hygiene-v1.schema.json", + "title": "Renderflow publication hygiene evidence v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "policy_id", + "provider_id", + "provider_version", + "source_artifact_id", + "output_artifact_id", + "status" + ], + "properties": { + "schema_version": { "const": "renderflow.hygiene/v1" }, + "policy_id": { "type": "string", "minLength": 1 }, + "provider_id": { "type": "string", "minLength": 1 }, + "provider_version": { "type": "string", "minLength": 1 }, + "source_artifact_id": { "type": "string", "minLength": 1 }, + "output_artifact_id": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "review_required", "blocked"] }, + "redaction": { + "type": "object", + "additionalProperties": false, + "required": ["provider_id", "determinism", "policy_reviewed"], + "properties": { + "provider_id": { "type": "string", "minLength": 1 }, + "provider_version": { "type": "string", "minLength": 1 }, + "determinism": { "enum": ["deterministic", "probabilistic"] }, + "policy_reviewed": { "type": "boolean" } + } + }, + "changed_field_classes": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "kind", "class", "message", "blocking"], + "properties": { + "code": { "type": "string", "minLength": 1 }, + "kind": { + "enum": ["metadata", "secret", "protected_reference", "redaction", "rights"] + }, + "class": { "type": "string", "minLength": 1 }, + "message": { "type": "string", "minLength": 1 }, + "blocking": { "type": "boolean" } + } + } + } + } +} diff --git a/schemas/renderflow-provider-v1.schema.json b/schemas/renderflow-provider-v1.schema.json index 5075544..1be4133 100644 --- a/schemas/renderflow-provider-v1.schema.json +++ b/schemas/renderflow-provider-v1.schema.json @@ -24,7 +24,7 @@ "capabilities": { "type": "object", "additionalProperties": false, - "required": ["schema_version", "provider_id", "provider_version", "operations", "progress_event_schema", "checkpoint_schema", "artifact_contract", "intake_schema"], + "required": ["schema_version", "provider_id", "provider_version", "operations", "progress_event_schema", "checkpoint_schema", "artifact_contract", "intake_schema", "hygiene_schema"], "properties": { "schema_version": { "const": "renderflow.provider/v1" }, "provider_id": { "const": "provider.renderflow" }, @@ -33,7 +33,8 @@ "progress_event_schema": { "const": "renderflow.progress/v1" }, "checkpoint_schema": { "const": "renderflow.checkpoints/v1" }, "artifact_contract": { "const": "flow.artifact/v1" }, - "intake_schema": { "const": "renderflow.intake/v1" } + "intake_schema": { "const": "renderflow.intake/v1" }, + "hygiene_schema": { "const": "renderflow.hygiene/v1" } } }, "plan": { diff --git a/schemas/renderflow-run-v1.schema.json b/schemas/renderflow-run-v1.schema.json index 12ba026..c3dd1ed 100644 --- a/schemas/renderflow-run-v1.schema.json +++ b/schemas/renderflow-run-v1.schema.json @@ -112,6 +112,7 @@ "fidelity": { "enum": ["lossless", "partial", "lossy", "path_dependent", "unknown"] }, + "hygiene": { "$ref": "#/$defs/hygiene_evidence" }, "warnings": { "type": "array", "items": { "type": "string" } }, "metadata": { "type": "object" } } @@ -137,6 +138,46 @@ } } }, + "hygiene_evidence": { + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "policy_id", "provider_id", "provider_version", "source_artifact_id", "output_artifact_id", "status"], + "properties": { + "schema_version": { "const": "renderflow.hygiene/v1" }, + "policy_id": { "type": "string", "minLength": 1 }, + "provider_id": { "type": "string", "minLength": 1 }, + "provider_version": { "type": "string", "minLength": 1 }, + "source_artifact_id": { "type": "string", "minLength": 1 }, + "output_artifact_id": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "review_required", "blocked"] }, + "redaction": { "$ref": "#/$defs/redaction_evidence" }, + "changed_field_classes": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } }, + "findings": { "type": "array", "items": { "$ref": "#/$defs/hygiene_finding" } } + } + }, + "hygiene_finding": { + "type": "object", + "additionalProperties": false, + "required": ["code", "kind", "class", "message", "blocking"], + "properties": { + "code": { "type": "string", "minLength": 1 }, + "kind": { "enum": ["metadata", "secret", "protected_reference", "redaction", "rights"] }, + "class": { "type": "string", "minLength": 1 }, + "message": { "type": "string", "minLength": 1 }, + "blocking": { "type": "boolean" } + } + }, + "redaction_evidence": { + "type": "object", + "additionalProperties": false, + "required": ["provider_id", "determinism", "policy_reviewed"], + "properties": { + "provider_id": { "type": "string", "minLength": 1 }, + "provider_version": { "type": "string", "minLength": 1 }, + "determinism": { "enum": ["deterministic", "probabilistic"] }, + "policy_reviewed": { "type": "boolean" } + } + }, "validation_diagnostic": { "type": "object", "additionalProperties": false, diff --git a/schemas/renderflow-v2.schema.json b/schemas/renderflow-v2.schema.json index af13b41..fef0f94 100644 --- a/schemas/renderflow-v2.schema.json +++ b/schemas/renderflow-v2.schema.json @@ -109,6 +109,12 @@ "null" ] }, + "hygiene_policy": { + "anyOf": [ + { "$ref": "#/$defs/stableId" }, + { "type": "null" } + ] + }, "reject_loss_classes": { "default": [], "items": { @@ -174,6 +180,75 @@ }, "type": "object" }, + "hygienePolicy": { + "additionalProperties": false, + "properties": { + "audience": { + "default": "candidate", + "enum": ["candidate", "private", "public", "commercial"] + }, + "metadata": { "$ref": "#/$defs/metadataHygiene" }, + "secrets": { "$ref": "#/$defs/secretHygiene" }, + "protected_references": { "$ref": "#/$defs/protectedReferences" }, + "redaction": { + "anyOf": [ + { "$ref": "#/$defs/contentRedaction" }, + { "type": "null" } + ] + }, + "rights": { "$ref": "#/$defs/rightsHygiene" } + }, + "type": "object" + }, + "metadataHygiene": { + "additionalProperties": false, + "properties": { + "allow": { "default": [], "items": { "minLength": 1, "type": "string" }, "type": "array" }, + "deny": { "default": [], "items": { "minLength": 1, "type": "string" }, "type": "array" }, + "allowlist_only": { "default": false, "type": "boolean" } + }, + "type": "object" + }, + "secretHygiene": { + "additionalProperties": false, + "properties": { + "enabled": { "default": true, "type": "boolean" }, + "block": { "default": true, "type": "boolean" }, + "markers": { "default": [], "items": { "minLength": 1, "type": "string" }, "type": "array" } + }, + "type": "object" + }, + "protectedReferences": { + "additionalProperties": false, + "properties": { + "terms": { "default": [], "items": { "minLength": 1, "type": "string" }, "type": "array" }, + "block": { "default": true, "type": "boolean" }, + "case_sensitive": { "default": false, "type": "boolean" } + }, + "type": "object" + }, + "contentRedaction": { + "additionalProperties": false, + "properties": { + "provider": { "minLength": 1, "type": "string" }, + "determinism": { "default": "deterministic", "enum": ["deterministic", "probabilistic"] }, + "classes": { "default": [], "items": { "minLength": 1, "type": "string" }, "type": "array" }, + "reviewed": { "default": false, "type": "boolean" } + }, + "required": ["provider"], + "type": "object" + }, + "rightsHygiene": { + "additionalProperties": false, + "properties": { + "required": { "default": false, "type": "boolean" }, + "license": { "type": ["string", "null"] }, + "rights_holder": { "type": ["string", "null"] }, + "approval_reference": { "type": ["string", "null"] }, + "reviewed": { "default": false, "type": "boolean" } + }, + "type": "object" + }, "profile": { "additionalProperties": false, "properties": { @@ -189,6 +264,12 @@ "include": { "$ref": "#/$defs/selectorSet" }, + "hygiene_policy": { + "anyOf": [ + { "$ref": "#/$defs/stableId" }, + { "type": "null" } + ] + }, "targets": { "default": [], "items": { @@ -498,6 +579,11 @@ "execution": { "$ref": "#/$defs/executionPolicy" }, + "hygiene": { + "additionalProperties": { "$ref": "#/$defs/hygienePolicy" }, + "default": {}, + "type": "object" + }, "output": { "$ref": "#/$defs/outputLayout" }, diff --git a/tests/fixtures/publication-hygiene/blocked.yaml b/tests/fixtures/publication-hygiene/blocked.yaml new file mode 100644 index 0000000..d3950a7 --- /dev/null +++ b/tests/fixtures/publication-hygiene/blocked.yaml @@ -0,0 +1,25 @@ +schema: renderflow/v2 +sources: + - id: source.blocked + path: source-blocked.md + format: markdown +hygiene: + fixture-policy: + audience: commercial + secrets: + markers: + - FIXTURE-CREDENTIAL-MARKER + protected_references: + terms: + - Example Franchise + rights: + required: true +targets: + exact: + - id: target.blocked + role: blocked + format: markdown +execution: + hygiene_policy: fixture-policy +output: + bundle_root: /tmp/renderflow-hygiene-blocked diff --git a/tests/fixtures/publication-hygiene/safe.yaml b/tests/fixtures/publication-hygiene/safe.yaml new file mode 100644 index 0000000..8605403 --- /dev/null +++ b/tests/fixtures/publication-hygiene/safe.yaml @@ -0,0 +1,29 @@ +schema: renderflow/v2 +sources: + - id: source.safe + path: source-safe.md + format: markdown +hygiene: + fixture-policy: + audience: public + metadata: + allowlist_only: true + allow: + - renderflow.source_id + protected_references: + terms: + - Example Franchise + rights: + required: true + license: All rights reserved + approval_reference: fixture-review + reviewed: true +targets: + exact: + - id: target.safe + role: reviewed + format: markdown +execution: + hygiene_policy: fixture-policy +output: + bundle_root: /tmp/renderflow-hygiene-safe diff --git a/tests/fixtures/publication-hygiene/source-blocked.md b/tests/fixtures/publication-hygiene/source-blocked.md new file mode 100644 index 0000000..82b890b --- /dev/null +++ b/tests/fixtures/publication-hygiene/source-blocked.md @@ -0,0 +1,5 @@ +# Internal generation note + +Use Example Franchise as a named visual reference. + +Configured fixture marker: FIXTURE-CREDENTIAL-MARKER diff --git a/tests/fixtures/publication-hygiene/source-safe.md b/tests/fixtures/publication-hygiene/source-safe.md new file mode 100644 index 0000000..76cbeda --- /dev/null +++ b/tests/fixtures/publication-hygiene/source-safe.md @@ -0,0 +1,3 @@ +# Original world + +A weathered roadside settlement glows beneath an amber sky.