diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..807c68a --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,71 @@ +name: Artifact conformance + +on: + pull_request: + release: + types: [published] + schedule: + - cron: "17 4 * * 2" + workflow_dispatch: + +permissions: + contents: read + +jobs: + fast: + name: Fast hermetic corpus + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.94" + + - name: Run focused conformance harness + env: + RENDERFLOW_CONFORMANCE_TIER: "fast" + RENDERFLOW_CONFORMANCE_REPORT: "${{ runner.temp }}/renderflow-conformance-fast.json" + run: cargo test --package renderflow --test golden_conformance --locked + + - name: Upload conformance report + if: always() + uses: actions/upload-artifact@v6 + with: + name: "renderflow-conformance-fast-${{ github.run_id }}" + path: "${{ runner.temp }}/renderflow-conformance-fast.json" + + maximal: + name: Maximal tool-backed corpus + if: github.event_name == 'schedule' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.94" + + - name: Install optional conformance providers + run: | + sudo apt-get update + sudo apt-get install --yes ffmpeg pandoc + + - name: Run maximal conformance harness + env: + RENDERFLOW_CONFORMANCE_TIER: "maximal" + RENDERFLOW_CONFORMANCE_REPORT: "${{ runner.temp }}/renderflow-conformance-maximal.json" + run: cargo test --package renderflow --test golden_conformance --locked + + - name: Upload conformance report + if: always() + uses: actions/upload-artifact@v6 + with: + name: "renderflow-conformance-maximal-${{ github.run_id }}" + path: "${{ runner.temp }}/renderflow-conformance-maximal.json" + retention-days: 90 diff --git a/crates/renderflow-core/src/detect.rs b/crates/renderflow-core/src/detect.rs index c85b3c3..62a22ba 100644 --- a/crates/renderflow-core/src/detect.rs +++ b/crates/renderflow-core/src/detect.rs @@ -83,6 +83,21 @@ pub struct DetectionConflict { /// The `buf` slice should contain at least the first 16 bytes of the file for /// reliable results, but longer slices are handled correctly. pub fn detect_from_bytes(buf: &[u8]) -> Option { + if buf.starts_with(b"RIFF") && buf.len() >= 12 { + match &buf[8..12] { + b"WAVE" => return Some(Format::Wav), + b"AVI " => return Some(Format::Avi), + b"WEBP" => return Some(Format::Webp), + _ => {} + } + } + if buf.len() >= 12 && buf.get(4..8) == Some(b"ftyp") { + return Some(match &buf[8..12] { + b"avif" | b"avis" => Format::Avif, + _ => Format::Mp4, + }); + } + // Iterate in a deterministic order so that ties between formats sharing // a prefix are resolved consistently. let registry = FormatCapabilityRegistry::global(); @@ -104,7 +119,10 @@ pub fn detect_from_bytes(buf: &[u8]) -> Option { // Prefer the match with the longest (most specific) signature to avoid // false positives when a shorter signature is a prefix of another. - candidates.sort_by(|a, b| b.1.cmp(&a.1)); + candidates.sort_by(|a, b| { + b.1.cmp(&a.1) + .then_with(|| a.0.to_string().cmp(&b.0.to_string())) + }); candidates.into_iter().next().map(|(f, _)| f) } @@ -198,6 +216,25 @@ mod tests { assert_eq!(result, Some(Format::Flac)); } + #[test] + fn detect_riff_formats_from_form_type() { + assert_eq!(detect_from_bytes(b"RIFF\0\0\0\0WAVE"), Some(Format::Wav)); + assert_eq!(detect_from_bytes(b"RIFF\0\0\0\0AVI "), Some(Format::Avi)); + assert_eq!(detect_from_bytes(b"RIFF\0\0\0\0WEBP"), Some(Format::Webp)); + } + + #[test] + fn detect_iso_base_media_from_major_brand() { + assert_eq!( + detect_from_bytes(b"\0\0\0\x18ftypisom\0\0\0\0"), + Some(Format::Mp4) + ); + assert_eq!( + detect_from_bytes(b"\0\0\0\x18ftypavif\0\0\0\0"), + Some(Format::Avif) + ); + } + #[test] fn detect_mp3_from_id3_tag() { let buf = b"ID3\x03\x00\x00\x00\x00\x00\x00"; diff --git a/crates/renderflow-core/src/intake.rs b/crates/renderflow-core/src/intake.rs index 6f4c235..d7853db 100644 --- a/crates/renderflow-core/src/intake.rs +++ b/crates/renderflow-core/src/intake.rs @@ -425,7 +425,9 @@ impl IntakeEngine { .find(|descriptor| descriptor.id == "zip") .copied() } else { - magic_matches.first().copied() + crate::detect::detect_from_bytes(bytes) + .and_then(|format| registry.get(format)) + .or_else(|| magic_matches.first().copied()) }; if let Some(descriptor) = magic_match { signals.push(IntakeSignal { diff --git a/crates/renderflow-core/src/validation.rs b/crates/renderflow-core/src/validation.rs index 706917f..e9254dc 100644 --- a/crates/renderflow-core/src/validation.rs +++ b/crates/renderflow-core/src/validation.rs @@ -821,10 +821,7 @@ fn conformance_row( .map(|tool| tool.stable_id().to_string()) .collect(), validator_ids, - fixture_ids: match descriptor.id { - "png" => vec!["fixture.corrupt.png.truncated".to_string()], - _ => Vec::new(), - }, + fixture_ids: golden_fixture_ids(descriptor.id), supported_platforms: vec![ "linux".to_string(), "macos".to_string(), @@ -836,6 +833,22 @@ fn conformance_row( } } +fn golden_fixture_ids(format: &str) -> Vec { + let ids: &[&str] = match format { + "markdown" => &["fixture.document.markdown"], + "pdf" => &["fixture.document.pdf"], + "png" => &["fixture.image.png", "fixture.mismatch.jpeg-png"], + "svg" => &["fixture.image.svg"], + "wav" => &["fixture.audio.wav"], + "mp4" => &["fixture.video.mp4"], + "zip" => &["fixture.archive.zip", "fixture.corrupt.zip"], + "json" => &["fixture.data.json"], + "srt" => &["fixture.subtitle.srt"], + _ => &[], + }; + ids.iter().map(|id| (*id).to_string()).collect() +} + #[cfg(test)] mod tests { use super::*; @@ -862,5 +875,14 @@ mod tests { let second = CapabilityConformanceMatrix::builtins(); assert_eq!(first, second); assert!(!first.formats.is_empty()); + assert_eq!( + first + .formats + .iter() + .find(|row| row.format == "wav") + .unwrap() + .fixture_ids, + vec!["fixture.audio.wav"] + ); } } diff --git a/crates/renderflow-core/tests/golden_conformance.rs b/crates/renderflow-core/tests/golden_conformance.rs new file mode 100644 index 0000000..873df3b --- /dev/null +++ b/crates/renderflow-core/tests/golden_conformance.rs @@ -0,0 +1,1003 @@ +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::fs; +use std::io::{Cursor, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use renderflow::artifact::{ + Artifact, ArtifactCollection, ArtifactDescriptor, ArtifactStorageClass, ArtifactStore, + ArtifactTransform, +}; +use renderflow::checkpoint::{CheckpointContext, RecoveryAction}; +use renderflow::evidence::{ + sha256_serialized, sha256_text, ArtifactEvidence, ArtifactManifest, ArtifactRole, + CacheDisposition, FidelityDeclaration, RunManifest, RunState, StepState, ValidationState, + ARTIFACT_MANIFEST_SCHEMA_V1, RUN_MANIFEST_SCHEMA_V1, +}; +use renderflow::graph::{ + DagExecutor, ExecutionPlan, Format, InputKind, TransformEdge, TransformGraph, +}; +use renderflow::hygiene::{HygieneEngine, HygieneFindingKind, HygieneStatus}; +use renderflow::optimization::OptimizationMode; +use renderflow::spec::{ + DerivativeProfile, HygienePolicy, ProtectedReferencePolicy, SecretHygienePolicy, +}; +use renderflow::transforms::aggregation::AggregationTransform; +use renderflow::transforms::Transform; +use renderflow::{IntakeBudgets, IntakeEngine, IntakeRequest}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +const CORPUS_SCHEMA: &str = "renderflow.golden-corpus/v1"; +const REPORT_SCHEMA: &str = "renderflow.conformance-report/v1"; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Corpus { + schema_version: String, + corpus_version: String, + license: String, + redistribution: String, + fixtures: Vec, + collections: Vec, + scenarios: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Fixture { + id: String, + filename: String, + encoding: String, + payload: String, + expected_format: Option, + expected_media_type: String, + family: String, + expected_outcome: String, + expected_diagnostic: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct FixtureCollection { + id: String, + members: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Scenario { + id: String, + tier: String, + fixtures: Vec, +} + +#[derive(Debug, Serialize)] +struct ConformanceReport { + schema_version: &'static str, + corpus_version: String, + corpus_digest: ReportDigest, + engine_version: &'static str, + tier: String, + status: &'static str, + fixtures: Vec, + scenarios: Vec, +} + +#[derive(Debug, Serialize)] +struct ReportDigest { + algorithm: &'static str, + value: String, +} + +#[derive(Debug, Serialize)] +struct FixtureEvidence { + id: String, + artifact_id: String, + digest: ReportDigest, + format: Option, + media_type: String, + outcome: String, +} + +#[derive(Debug, Serialize)] +struct ScenarioEvidence { + id: String, + status: String, + assertions: Vec, + providers: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option, +} + +#[derive(Default)] +struct ScenarioResults(BTreeMap); + +impl ScenarioResults { + fn insert(&mut self, entry: (String, ScenarioEvidence)) { + self.0.insert(entry.0, entry.1); + } + + fn keys(&self) -> impl Iterator { + self.0.keys() + } + + fn into_values(self) -> impl Iterator { + self.0.into_values() + } +} + +struct StaticTransform { + name: &'static str, + output: &'static str, + executions: Option>, +} + +impl Transform for StaticTransform { + fn name(&self) -> &str { + self.name + } + + fn apply(&self, _input: String) -> Result { + if let Some(executions) = &self.executions { + executions.fetch_add(1, Ordering::SeqCst); + } + Ok(self.output.to_string()) + } +} + +struct AlwaysFails; + +impl ArtifactTransform for AlwaysFails { + fn name(&self) -> &str { + "fixture.always-fails" + } + + fn version(&self) -> &str { + "1" + } + + fn apply(&self, _: &Artifact, _: Format, _: &ArtifactStore) -> Result { + anyhow::bail!("synthetic branch-local failure") + } +} + +struct WrongFormat; + +impl ArtifactTransform for WrongFormat { + fn name(&self) -> &str { + "fixture.wrong-format" + } + + fn apply(&self, input: &Artifact, _: Format, store: &ArtifactStore) -> Result { + store.put_bytes( + br#"{"unexpected":true}"#, + ArtifactDescriptor::for_format(Format::Json, ArtifactStorageClass::Intermediate) + .with_source(input.id().clone()), + ) + } +} + +struct OrderedPdf; + +impl AggregationTransform for OrderedPdf { + fn name(&self) -> &str { + "fixture.ordered-pdf" + } + + fn aggregate(&self, inputs: &[&str], output_path: &str) -> Result<()> { + let mut output = fs::File::create(output_path)?; + output.write_all(b"%PDF-1.4\n% ordered synthetic inputs\n")?; + for input in inputs { + let mut bytes = Vec::new(); + fs::File::open(input)?.read_to_end(&mut bytes)?; + output.write_all(&bytes)?; + } + output.write_all(b"\n%%EOF\n")?; + Ok(()) + } +} + +fn corpus_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/fixtures/golden-artifacts/v1/corpus.json") +} + +fn decode_hex(value: &str) -> Result> { + if !value.len().is_multiple_of(2) { + anyhow::bail!("hex fixture has an odd number of digits"); + } + (0..value.len()) + .step_by(2) + .map(|offset| { + u8::from_str_radix(&value[offset..offset + 2], 16) + .with_context(|| format!("invalid hex fixture at byte {}", offset / 2)) + }) + .collect() +} + +fn materialize_fixture(root: &Path, fixture: &Fixture) -> Result { + let bytes = match fixture.encoding.as_str() { + "utf8" => fixture.payload.as_bytes().to_vec(), + "hex" => decode_hex(&fixture.payload)?, + other => anyhow::bail!("unsupported fixture encoding '{other}'"), + }; + let path = root.join(&fixture.filename); + fs::write(&path, bytes)?; + Ok(path) +} + +fn source_artifact(store: &ArtifactStore, path: &Path) -> Result { + store.import_path( + path, + ArtifactDescriptor::for_format(Format::Markdown, ArtifactStorageClass::Source), + ) +} + +fn one_edge(from: Format, to: Format) -> renderflow::graph::MultiTargetDag { + let mut graph = TransformGraph::new(); + graph.add_transform( + TransformEdge::with_input_kind(from, to, 1.0, 1.0, InputKind::Single) + .with_provider("provider.fixture.local", format!("{from}.to.{to}")) + .with_variant("fixture-v1"), + ); + graph + .build_multi_target_dag(from, &[to]) + .expect("fixture edge is reachable") +} + +fn checkpoint_context(toolchain: &str) -> CheckpointContext { + CheckpointContext { + execution_plan_digest: sha256_text("fixture-plan-v1"), + source_spec_digest: sha256_text("fixture-source-spec-v1"), + toolchain_fingerprint: Some(toolchain.to_string()), + } +} + +fn scenario(id: &str, assertions: &[&str], providers: &[&str]) -> (String, ScenarioEvidence) { + ( + id.to_string(), + ScenarioEvidence { + id: id.to_string(), + status: "passed".to_string(), + assertions: assertions + .iter() + .map(|value| (*value).to_string()) + .collect(), + providers: providers.iter().map(|value| (*value).to_string()).collect(), + reason: None, + }, + ) +} + +fn tool_version(program: &str) -> Option { + let argument = if program == "ffmpeg" { + "-version" + } else { + "--version" + }; + let output = std::process::Command::new(program) + .arg(argument) + .output() + .ok()?; + output.status.success().then(|| { + String::from_utf8_lossy(&output.stdout) + .lines() + .next() + .unwrap_or("version unavailable") + .trim() + .to_string() + }) +} + +#[test] +fn golden_artifact_forest_conformance() -> Result<()> { + let corpus_bytes = fs::read(corpus_path())?; + let corpus: Corpus = serde_json::from_slice(&corpus_bytes)?; + assert_eq!(corpus.schema_version, CORPUS_SCHEMA); + assert_eq!(corpus.license, "CC0-1.0"); + assert!(corpus.redistribution.contains("Synthetic")); + + let fixture_ids = corpus + .fixtures + .iter() + .map(|fixture| fixture.id.as_str()) + .collect::>(); + assert_eq!(fixture_ids.len(), corpus.fixtures.len()); + assert_eq!(corpus.fixtures.len(), 12); + assert_eq!(corpus.collections.len(), 1); + for collection in &corpus.collections { + assert!(collection.id.starts_with("fixture.collection.")); + assert!(collection + .members + .iter() + .all(|member| fixture_ids.contains(member.as_str()))); + } + for declared in &corpus.scenarios { + assert!(!declared.fixtures.is_empty()); + assert!(matches!( + declared.tier.as_str(), + "fast" | "tool_backed" | "maximal" + )); + assert!(declared.fixtures.iter().all(|fixture| { + fixture_ids.contains(fixture.as_str()) + || corpus.collections.iter().any(|value| value.id == *fixture) + })); + } + + let work = tempfile::tempdir()?; + let fixture_root = work.path().join("fixtures"); + fs::create_dir_all(&fixture_root)?; + let intake_store = ArtifactStore::new(work.path().join("intake-store"))?; + let mut paths = BTreeMap::new(); + let mut fixture_evidence = Vec::new(); + let mut observed_families = BTreeSet::new(); + for fixture in &corpus.fixtures { + let path = materialize_fixture(&fixture_root, fixture)?; + let request = IntakeRequest::from_path(&path) + .with_extraction(true) + .with_budgets(IntakeBudgets { + max_depth: 1, + max_artifacts: 8, + max_extracted_bytes: 64 * 1024, + max_expansion_ratio: 16.0, + }); + let report = IntakeEngine::new().intake(&request, &intake_store)?; + assert_eq!(report.profile.format, fixture.expected_format); + assert_eq!(report.profile.media_type, fixture.expected_media_type); + match fixture.expected_outcome.as_str() { + "accepted" => assert!( + report.profile.conflicts.is_empty(), + "{} unexpectedly reported conflicts: {:?}", + fixture.id, + report.profile.conflicts + ), + "conflict" => assert!(!report.profile.conflicts.is_empty()), + "rejected" => { + let code = fixture + .expected_diagnostic + .as_deref() + .context("rejected fixture must name its diagnostic")?; + assert!(report.diagnostics.iter().any(|item| item.code == code)); + } + other => anyhow::bail!("unknown expected fixture outcome '{other}'"), + } + intake_store.verify(&report.source)?; + observed_families.insert(fixture.family.as_str()); + fixture_evidence.push(FixtureEvidence { + id: fixture.id.clone(), + artifact_id: report.source.id().to_string(), + digest: ReportDigest { + algorithm: "sha256", + value: report.source.digest().value().to_string(), + }, + format: report.profile.format, + media_type: report.profile.media_type, + outcome: fixture.expected_outcome.clone(), + }); + paths.insert(fixture.id.clone(), path); + } + assert_eq!( + observed_families, + BTreeSet::from([ + "archive", "audio", "data", "document", "image", "subtitle", "unknown", "video" + ]) + ); + + let mut results = ScenarioResults::default(); + results.insert(scenario( + "inspect_only", + &["all fixture identities, formats, media types, conflicts, and diagnostics matched"], + &["provider.renderflow.native-inspection"], + )); + + let execution_store = ArtifactStore::new(work.path().join("execution-store"))?; + let markdown_path = &paths["fixture.document.markdown"]; + let source = source_artifact(&execution_store, markdown_path)?; + let direct_dag = one_edge(Format::Markdown, Format::Html); + let mut direct = DagExecutor::new(); + direct.register_single_with_identity( + Format::Markdown, + Format::Html, + Arc::new(StaticTransform { + name: "fixture.markdown-html", + output: "

Synthetic artifact

\n", + executions: None, + }), + "fixture.markdown-html/v1", + ); + let direct_report = direct.execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert_eq!(direct_report.steps.len(), 1); + assert_eq!(direct_report.steps[0].state, StepState::Complete); + assert!(direct_report.artifacts.contains_key(&Format::Html)); + results.insert(scenario( + "direct_deterministic_conversion", + &["production DAG executor produced one validated HTML derivative"], + &["provider.fixture.local"], + )); + + let mut graph = TransformGraph::new(); + graph.add_transform(TransformEdge::new(Format::Markdown, Format::Html, 1.0, 1.0)); + graph.add_transform(TransformEdge::new(Format::Html, Format::Pdf, 1.0, 1.0)); + graph.add_transform(TransformEdge::new(Format::Html, Format::Json, 1.0, 1.0)); + let multi_dag = graph + .build_multi_target_dag(Format::Markdown, &[Format::Pdf, Format::Json]) + .context("multi-target fixture graph must resolve")?; + assert_eq!(multi_dag.edge_count(), 3); + let first_plan = ExecutionPlan::from_dag( + &multi_dag, + Format::Markdown, + &[Format::Pdf, Format::Json], + OptimizationMode::Balanced, + ); + let second_plan = ExecutionPlan::from_dag( + &multi_dag, + Format::Markdown, + &[Format::Pdf, Format::Json], + OptimizationMode::Balanced, + ); + assert_eq!( + sha256_serialized(&first_plan)?, + sha256_serialized(&second_plan)? + ); + let mut multi = DagExecutor::new(); + multi.register_single( + Format::Markdown, + Format::Html, + Arc::new(StaticTransform { + name: "fixture.markdown-html", + output: "

Synthetic artifact

\n", + executions: None, + }), + ); + multi.register_single( + Format::Html, + Format::Pdf, + Arc::new(StaticTransform { + name: "fixture.html-pdf", + output: "%PDF-1.4\n%%EOF\n", + executions: None, + }), + ); + multi.register_single( + Format::Html, + Format::Json, + Arc::new(StaticTransform { + name: "fixture.html-json", + output: "{\"kind\":\"synthetic-derivative\"}\n", + executions: None, + }), + ); + let multi_report = multi.execute_artifact_with_evidence( + &multi_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert_eq!(multi_report.steps.len(), 3); + assert!(multi_report.artifacts.contains_key(&Format::Pdf)); + assert!(multi_report.artifacts.contains_key(&Format::Json)); + results.insert(scenario( + "multi_step_conversion", + &["topological execution produced the two-edge PDF path"], + &["provider.fixture.local"], + )); + results.insert(scenario( + "multi_target_shared_intermediate", + &["one HTML intermediate served PDF and JSON target branches"], + &["provider.fixture.local"], + )); + results.insert(scenario( + "cross_family_derivative", + &["document input produced a validated structured-data derivative"], + &["provider.fixture.local"], + )); + + let first_page = source.clone(); + let second_page = execution_store.put_bytes( + b"second synthetic page", + ArtifactDescriptor::for_format(Format::Markdown, ArtifactStorageClass::Source), + )?; + let mut collection_graph = TransformGraph::new(); + collection_graph.add_collection_transform(Format::Markdown, Format::Pdf, 1.0, 1.0); + let collection_dag = collection_graph + .build_multi_target_dag(Format::Markdown, &[Format::Pdf]) + .context("collection graph must resolve")?; + let mut collection_executor = DagExecutor::new(); + collection_executor.register_aggregation(Format::Markdown, Format::Pdf, Arc::new(OrderedPdf)); + let collection_output = collection_executor.execute_artifacts( + &collection_dag, + Format::Markdown, + ArtifactCollection::new(vec![first_page.clone(), second_page.clone()]), + &execution_store, + )?; + let aggregate = collection_output[&Format::Pdf].clone().into_one()?; + assert_eq!( + aggregate.sources(), + &[first_page.id().clone(), second_page.id().clone()] + ); + results.insert(scenario( + "collection_aggregation", + &["ordered collection membership and output source lineage were preserved"], + &["provider.fixture.local"], + )); + + let executions = Arc::new(AtomicUsize::new(0)); + let cache_path = work.path().join("dag-cache.json"); + let mut cached = DagExecutor::new().with_cache(&cache_path); + cached.register_single_with_identity( + Format::Markdown, + Format::Html, + Arc::new(StaticTransform { + name: "fixture.counting-html", + output: "

cached

\n", + executions: Some(Arc::clone(&executions)), + }), + "fixture.counting-html/v1", + ); + let cold = cached.execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + let warm = cached.execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert_eq!(cold.steps[0].cache, CacheDisposition::Miss); + assert_eq!(warm.steps[0].cache, CacheDisposition::Hit); + let mut changed = DagExecutor::new().with_cache(&cache_path); + changed.register_single_with_identity( + Format::Markdown, + Format::Html, + Arc::new(StaticTransform { + name: "fixture.counting-html", + output: "

cache identity changed

\n", + executions: Some(Arc::clone(&executions)), + }), + "fixture.counting-html/v2", + ); + let invalidated = changed.execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert_eq!(invalidated.steps[0].cache, CacheDisposition::Miss); + assert_eq!(executions.load(Ordering::SeqCst), 2); + results.insert(scenario( + "cache_cold_warm_partial_invalidation", + &["cold miss, warm hit, and transform-identity invalidation were observable"], + &["provider.fixture.local"], + )); + + let mut provider_graph = TransformGraph::new(); + provider_graph.add_transform( + TransformEdge::new(Format::Markdown, Format::Html, 0.1, 1.0) + .with_provider("provider.fixture.remote", "document.convert"), + ); + provider_graph.add_transform( + TransformEdge::new(Format::Markdown, Format::Html, 1.0, 1.0) + .with_provider("provider.fixture.local", "document.convert"), + ); + let available = HashSet::from(["provider.fixture.local".to_string()]); + let filtered = provider_graph.filtered_by_available_providers(&available); + let fallback = filtered + .build_multi_target_dag(Format::Markdown, &[Format::Html]) + .context("local fallback must remain reachable")?; + assert_eq!( + fallback.all_edges()[0].provider_id.as_deref(), + Some("provider.fixture.local") + ); + results.insert(scenario( + "missing_provider_fallback", + &["unavailable remote edge was excluded while the local edge remained selectable"], + &["provider.fixture.local"], + )); + + let mut wrong = DagExecutor::new(); + wrong.register_artifact(Format::Markdown, Format::Html, Arc::new(WrongFormat)); + let wrong_report = wrong.execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert_eq!(wrong_report.steps[0].state, StepState::Failed); + assert_eq!(wrong_report.steps[0].validation, ValidationState::Invalid); + results.insert(scenario( + "validation_failure", + &["wrong-format output was classified invalid despite a successful transform return"], + &["provider.fixture.local"], + )); + + let mut partial_graph = TransformGraph::new(); + partial_graph.add_transform(TransformEdge::new(Format::Markdown, Format::Html, 1.0, 1.0)); + partial_graph.add_transform(TransformEdge::new(Format::Markdown, Format::Pdf, 1.0, 1.0)); + partial_graph.add_transform(TransformEdge::new(Format::Pdf, Format::Json, 1.0, 1.0)); + let partial_dag = partial_graph + .build_multi_target_dag(Format::Markdown, &[Format::Html, Format::Json]) + .context("partial graph must resolve")?; + let mut partial = DagExecutor::new(); + partial.register_single( + Format::Markdown, + Format::Html, + Arc::new(StaticTransform { + name: "fixture.good-branch", + output: "

good branch

\n", + executions: None, + }), + ); + partial.register_artifact(Format::Markdown, Format::Pdf, Arc::new(AlwaysFails)); + let partial_report = partial.execute_artifact_with_evidence( + &partial_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert!(partial_report.artifacts.contains_key(&Format::Html)); + assert!(partial_report + .steps + .iter() + .any(|step| step.state == StepState::Failed)); + assert!(partial_report + .steps + .iter() + .any(|step| step.state == StepState::Skipped)); + results.insert(scenario( + "branch_local_failure_partial_result", + &["successful sibling output survived a failed branch and downstream skip"], + &["provider.fixture.local"], + )); + + let cancelled = Arc::new(AtomicBool::new(true)); + let cancellation_report = DagExecutor::new() + .with_cancellation_flag(cancelled) + .execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert_eq!(cancellation_report.steps[0].state, StepState::Cancelled); + results.insert(scenario( + "cancellation", + &["pre-wave cancellation produced explicit cancelled step evidence"], + &[], + )); + + let checkpoint_path = work.path().join("checkpoints.json"); + let checkpoint_runs = Arc::new(AtomicUsize::new(0)); + let build_checkpoint_executor = |resume: bool, identity: &'static str, toolchain: &str| { + let mut executor = DagExecutor::new() + .with_toolchain_fingerprint(toolchain) + .with_checkpoints(&checkpoint_path, checkpoint_context(toolchain), resume); + executor.register_single_with_identity( + Format::Markdown, + Format::Html, + Arc::new(StaticTransform { + name: "fixture.checkpoint-html", + output: "

checkpointed

\n", + executions: Some(Arc::clone(&checkpoint_runs)), + }), + identity, + ); + executor + }; + build_checkpoint_executor(false, "fixture.checkpoint-html/v1", "toolchain-v1") + .execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + let resumed = build_checkpoint_executor(true, "fixture.checkpoint-html/v1", "toolchain-v1") + .execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert_eq!(resumed.steps[0].state, StepState::Reused); + assert_eq!(checkpoint_runs.load(Ordering::SeqCst), 1); + results.insert(scenario( + "checkpoint_resume", + &["compatible validated checkpoint resumed without repeating work"], + &["provider.fixture.local"], + )); + let changed_context = checkpoint_context("toolchain-v2"); + let previous_context = checkpoint_context("toolchain-v1"); + let decision = + renderflow::checkpoint::CheckpointStore::open(&checkpoint_path, previous_context)? + .context_decision(&changed_context); + assert_eq!(decision.action, RecoveryAction::Recompute); + let changed_tool = + build_checkpoint_executor(true, "fixture.checkpoint-html/v1", "toolchain-v2") + .execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + source.clone(), + &execution_store, + )?; + assert_eq!(changed_tool.steps[0].state, StepState::Complete); + let changed_source = execution_store.put_bytes( + b"# Changed synthetic source\n", + ArtifactDescriptor::for_format(Format::Markdown, ArtifactStorageClass::Source), + )?; + let source_invalidated = + build_checkpoint_executor(true, "fixture.checkpoint-html/v1", "toolchain-v2") + .execute_artifact_with_evidence( + &direct_dag, + Format::Markdown, + changed_source, + &execution_store, + )?; + assert_eq!(source_invalidated.steps[0].state, StepState::Complete); + results.insert(scenario( + "version_invalidation", + &["transform identity, toolchain context, and source identity changes forced recomputation"], + &["provider.fixture.local"], + )); + + let gated_source = execution_store.put_bytes( + b"Protected Fixture with FIXTURE-CREDENTIAL-MARKER", + ArtifactDescriptor::for_format(Format::Markdown, ArtifactStorageClass::Source), + )?; + let policy = HygienePolicy { + secrets: SecretHygienePolicy { + markers: vec!["FIXTURE-CREDENTIAL-MARKER".to_string()], + ..SecretHygienePolicy::default() + }, + protected_references: ProtectedReferencePolicy { + terms: vec!["Protected Fixture".to_string()], + ..ProtectedReferencePolicy::default() + }, + ..HygienePolicy::default() + }; + let hygiene = HygieneEngine::new().apply( + "fixture.publication", + &policy, + &gated_source, + &execution_store, + )?; + assert_eq!(hygiene.evidence.status, HygieneStatus::Blocked); + assert!(hygiene + .evidence + .findings + .iter() + .any(|finding| finding.kind == HygieneFindingKind::ProtectedReference)); + assert!(!serde_json::to_string(&hygiene.evidence)?.contains("FIXTURE-CREDENTIAL-MARKER")); + results.insert(scenario( + "privacy_redaction_gate", + &["secret and protected-reference gates blocked release without leaking the marker"], + &["renderflow.core-hygiene"], + )); + + let inner = { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file("leaf.txt", zip::write::SimpleFileOptions::default())?; + writer.write_all(b"bounded leaf")?; + writer.finish()?.into_inner() + }; + let outer_path = fixture_root.join("nested.zip"); + { + let file = fs::File::create(&outer_path)?; + let mut writer = zip::ZipWriter::new(file); + writer.start_file("inner.zip", zip::write::SimpleFileOptions::default())?; + writer.write_all(&inner)?; + writer.finish()?; + } + let bounded = IntakeEngine::new().intake( + &IntakeRequest::from_path(&outer_path) + .with_extraction(true) + .with_budgets(IntakeBudgets { + max_depth: 1, + max_artifacts: 4, + max_extracted_bytes: 64 * 1024, + max_expansion_ratio: 32.0, + }), + &intake_store, + )?; + assert!(bounded + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "intake.budget.depth")); + results.insert(scenario( + "bounded_recursive_extraction", + &["nested archive extraction stopped at the declared depth budget"], + &["provider.renderflow.zip"], + )); + + let everything: DerivativeProfile = + serde_yaml_ng::from_str(include_str!("../data/profiles/everything-v1.yaml"))?; + assert!(everything.all_reachable); + assert!(everything.targets.is_empty()); + results.insert(scenario( + "everything_profile", + &["bundled profile selects all policy-allowed reachable branches"], + &[], + )); + let magazine: DerivativeProfile = + serde_yaml_ng::from_str(include_str!("../data/profiles/magazine-v1.yaml"))?; + let magazine_roles = magazine + .targets + .iter() + .filter_map(|target| target.role.as_deref()) + .collect::>(); + assert!(magazine_roles.contains("print/press")); + assert!(magazine_roles.contains("digital/web/index")); + assert!(magazine_roles.contains("assets/previews/cover")); + assert!(magazine.policy.validation.is_some()); + results.insert(scenario( + "magazine_release_bundle", + &["versioned magazine roles and release validation gate remain present"], + &[], + )); + + let intermediate = &multi_report.artifacts[&Format::Html]; + let output = &multi_report.artifacts[&Format::Json]; + let source_evidence = ArtifactEvidence::from_artifact( + &source, + "source", + ArtifactRole::Source, + markdown_path.display().to_string(), + renderflow::evidence::ProducerEvidence::source(), + ValidationState::Valid, + FidelityDeclaration::Lossless, + ); + let output_evidence = ArtifactEvidence::from_artifact( + output, + "data", + ArtifactRole::Terminal, + "dist/data.json", + renderflow::evidence::ProducerEvidence { + system: "renderflow".to_string(), + transform: Some("fixture.html-json".to_string()), + capability: Some("data.generate".to_string()), + provider: Some("provider.fixture.local".to_string()), + version: Some("1".to_string()), + }, + ValidationState::Valid, + FidelityDeclaration::Lossless, + ); + let intermediate_evidence = ArtifactEvidence::from_artifact( + intermediate, + "web-intermediate", + ArtifactRole::Intermediate, + "cache/intermediate.html", + renderflow::evidence::ProducerEvidence { + system: "renderflow".to_string(), + transform: Some("fixture.markdown-html".to_string()), + capability: Some("document.convert".to_string()), + provider: Some("provider.fixture.local".to_string()), + version: Some("1".to_string()), + }, + ValidationState::Valid, + FidelityDeclaration::Lossless, + ); + let run_id = format!("run:sha256:{}", "0".repeat(64)); + let manifest = RunManifest { + schema_version: RUN_MANIFEST_SCHEMA_V1.to_string(), + run_id: run_id.clone(), + execution_plan_digest: sha256_serialized(&first_plan)?, + source_spec_digest: sha256_text("fixture-source-spec-v1"), + engine_version: env!("CARGO_PKG_VERSION").to_string(), + started_at_unix_ms: 0, + completed_at_unix_ms: 0, + state: RunState::Complete, + artifact_manifest: ArtifactManifest { + schema_version: ARTIFACT_MANIFEST_SCHEMA_V1.to_string(), + run_id, + output_dir: "dist".to_string(), + outputs: vec!["dist/data.json".to_string()], + artifacts: vec![source_evidence, intermediate_evidence, output_evidence], + }, + steps: multi_report.steps, + diagnostics: multi_report.diagnostics, + toolchain: None, + artifact_forest: None, + }; + let flow = manifest.flow_artifacts_v1(); + assert_eq!(flow.len(), 3); + assert!(flow + .iter() + .all(|artifact| artifact.schema_version == "flow.artifact/v1")); + assert!(flow + .iter() + .all(|artifact| artifact.artifact_id.starts_with("artifact:sha256-"))); + assert!(flow + .iter() + .filter(|artifact| artifact.role != "source") + .all(|artifact| !artifact.sources.is_empty())); + + let tier = std::env::var("RENDERFLOW_CONFORMANCE_TIER").unwrap_or_else(|_| "fast".to_string()); + anyhow::ensure!(matches!(tier.as_str(), "fast" | "tool_backed" | "maximal")); + let requested_rank = match tier.as_str() { + "fast" => 0, + "tool_backed" => 1, + _ => 2, + }; + let tools = ["pandoc", "ffmpeg"]; + let available_tools = tools + .iter() + .filter_map(|tool| tool_version(tool).map(|version| format!("tool.{tool}@{version}"))) + .collect::>(); + let all_tools_available = available_tools.len() == tools.len(); + results.insert(( + "optional_tool_providers".to_string(), + ScenarioEvidence { + id: "optional_tool_providers".to_string(), + status: if requested_rank == 0 { + "excluded" + } else if all_tools_available { + "passed" + } else { + "unavailable" + } + .to_string(), + assertions: vec![ + "optional provider availability is explicit and never a silent pass".to_string(), + ], + providers: available_tools, + reason: (!all_tools_available) + .then(|| "one or more optional tool providers were not installed".to_string()), + }, + )); + results.insert(( + "maximal_release_matrix".to_string(), + ScenarioEvidence { + id: "maximal_release_matrix".to_string(), + status: if requested_rank == 2 { + "passed" + } else { + "excluded" + } + .to_string(), + assertions: vec![ + "maximal tier records every corpus family and scenario state".to_string(), + ], + providers: Vec::new(), + reason: (requested_rank < 2).then(|| "maximal tier was not requested".to_string()), + }, + )); + + let declared_scenarios = corpus + .scenarios + .iter() + .map(|item| item.id.as_str()) + .collect::>(); + let observed_scenarios = results.keys().map(String::as_str).collect::>(); + assert_eq!(observed_scenarios, declared_scenarios); + let scenarios = results.into_values().collect::>(); + let report = ConformanceReport { + schema_version: REPORT_SCHEMA, + corpus_version: corpus.corpus_version, + corpus_digest: ReportDigest { + algorithm: "sha256", + value: format!("{:x}", Sha256::digest(&corpus_bytes)), + }, + engine_version: env!("CARGO_PKG_VERSION"), + tier, + status: "passed", + fixtures: fixture_evidence, + scenarios, + }; + let encoded = serde_json::to_vec_pretty(&report)?; + let decoded: serde_json::Value = serde_json::from_slice(&encoded)?; + assert_eq!(decoded["schema_version"], REPORT_SCHEMA); + assert_eq!(decoded["fixtures"].as_array().map(Vec::len), Some(12)); + if let Some(path) = std::env::var_os("RENDERFLOW_CONFORMANCE_REPORT") { + fs::write(path, encoded)?; + } + Ok(()) +} diff --git a/docs/conformance-corpus.md b/docs/conformance-corpus.md new file mode 100644 index 0000000..08130ea --- /dev/null +++ b/docs/conformance-corpus.md @@ -0,0 +1,59 @@ +# Golden artifact conformance + +Renderflow's versioned golden corpus binds its artifact claims to a small, +redistribution-safe acceptance harness. The corpus contains synthetic payloads +only. It does not contain publication, comic, customer, or third-party content, +and no fixture requires network or AI access. + +## Contracts + +The canonical manifest is +`tests/fixtures/golden-artifacts/v1/corpus.json`. Binary payloads are stored as +lowercase hexadecimal inside the manifest so review, reproduction, and source +control remain transparent. The manifest conforms to +`schemas/renderflow-golden-corpus-v1.schema.json`. + +The focused Rust harness materializes those payloads in a temporary directory +and exercises production intake, graph planning, execution, validation, cache, +checkpoint, hygiene, profile, and Flow-projection APIs. It can emit a report +conforming to `schemas/renderflow-conformance-report-v1.schema.json`: + +```bash +RENDERFLOW_CONFORMANCE_TIER="fast" \ +RENDERFLOW_CONFORMANCE_REPORT="/tmp/renderflow-conformance.json" \ +cargo test --package renderflow --test golden_conformance --locked +``` + +The report pins the corpus digest and engine version. Each materialized fixture +records its artifact identity, payload digest, detected format, media type, and +expected outcome. Each scenario records its status, assertions, and selected +providers. Tool-backed runs include observed provider version strings; missing +optional providers are `unavailable`, never silently successful. + +## Tiers + +| Tier | Purpose | Dependency policy | +| --- | --- | --- | +| `fast` | Pull-request acceptance | Hermetic; Rust toolchain only | +| `tool_backed` | Adapter verification | Optional tools are probed and versioned | +| `maximal` | Scheduled and release evidence | All corpus families and scenario states are reported | + +The dedicated conformance workflow runs the fast tier for pull requests and +the maximal tier on its schedule, on published releases, and when manually +dispatched. Both publish the machine-readable report even when a job fails. + +## Adding a fixture + +1. Use an original synthetic payload small enough to inspect in review. +2. Add its payload, family, expected identity, and outcome to the versioned + corpus manifest. Never add secrets, protected references, or production + content merely to exercise a gate; use an unmistakably synthetic marker. +3. Map at least one scenario to the fixture and add an assertion through a + public production API. +4. Validate both schemas and run the focused harness locally. +5. Increment `corpus_version` when fixture bytes or expected behavior changes. + +A capability graduates from `unavailable` or `experimental` only when the +corpus proves its success path, failure classification, validation evidence, +and provider behavior on every supported platform. Nondeterministic formats +must assert structural and semantic invariants rather than byte identity. diff --git a/mkdocs.yml b/mkdocs.yml index 5c5072b..3f9d7b0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -123,6 +123,7 @@ nav: - Execution Evidence: execution-evidence.md - Resumable Provider Contract: provider-contract.md - Artifact Validation: user-guide/artifact-validation.md + - Golden Conformance Corpus: conformance-corpus.md - Plugin Architecture: architecture/plugin-architecture.md - Execution Plans: architecture/execution-plans.md - CLI Reference: diff --git a/schemas/renderflow-conformance-report-v1.schema.json b/schemas/renderflow-conformance-report-v1.schema.json new file mode 100644 index 0000000..8e80f68 --- /dev/null +++ b/schemas/renderflow-conformance-report-v1.schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://egohygiene.github.io/renderflow/schemas/renderflow-conformance-report-v1.schema.json", + "title": "Renderflow golden conformance report v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "corpus_version", "corpus_digest", "engine_version", "tier", "status", "fixtures", "scenarios"], + "properties": { + "schema_version": { "const": "renderflow.conformance-report/v1" }, + "corpus_version": { "type": "string", "minLength": 1 }, + "corpus_digest": { "$ref": "#/$defs/digest" }, + "engine_version": { "type": "string", "minLength": 1 }, + "tier": { "enum": ["fast", "tool_backed", "maximal"] }, + "status": { "enum": ["passed", "failed"] }, + "fixtures": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "artifact_id", "digest", "format", "media_type", "outcome"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "artifact_id": { "type": "string", "pattern": "^artifact:sha256:[a-f0-9]{64}$" }, + "digest": { "$ref": "#/$defs/digest" }, + "format": { "type": ["string", "null"] }, + "media_type": { "type": "string", "pattern": "^[^/]+/[^/]+$" }, + "outcome": { "enum": ["accepted", "conflict", "rejected"] } + } + } + }, + "scenarios": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "status", "assertions", "providers"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "unavailable", "excluded", "failed"] }, + "assertions": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "providers": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "reason": { "type": "string", "minLength": 1 } + } + } + } + }, + "$defs": { + "digest": { + "type": "object", + "additionalProperties": false, + "required": ["algorithm", "value"], + "properties": { + "algorithm": { "const": "sha256" }, + "value": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + } + } + } +} diff --git a/schemas/renderflow-golden-corpus-v1.schema.json b/schemas/renderflow-golden-corpus-v1.schema.json new file mode 100644 index 0000000..afafd71 --- /dev/null +++ b/schemas/renderflow-golden-corpus-v1.schema.json @@ -0,0 +1,65 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://egohygiene.github.io/renderflow/schemas/renderflow-golden-corpus-v1.schema.json", + "title": "Renderflow golden artifact corpus v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "corpus_version", "license", "redistribution", "fixtures", "collections", "scenarios"], + "properties": { + "schema_version": { "const": "renderflow.golden-corpus/v1" }, + "corpus_version": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" }, + "license": { "const": "CC0-1.0" }, + "redistribution": { "type": "string", "minLength": 1 }, + "fixtures": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/fixture" } + }, + "collections": { + "type": "array", + "items": { "$ref": "#/$defs/collection" } + }, + "scenarios": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/scenario" } + } + }, + "$defs": { + "fixture": { + "type": "object", + "additionalProperties": false, + "required": ["id", "filename", "encoding", "payload", "expected_media_type", "family", "expected_outcome"], + "properties": { + "id": { "type": "string", "pattern": "^fixture\\.[a-z0-9.-]+$" }, + "filename": { "type": "string", "minLength": 1 }, + "encoding": { "enum": ["utf8", "hex"] }, + "payload": { "type": "string" }, + "expected_format": { "type": "string", "minLength": 1 }, + "expected_media_type": { "type": "string", "pattern": "^[^/]+/[^/]+$" }, + "family": { "type": "string", "minLength": 1 }, + "expected_outcome": { "enum": ["accepted", "conflict", "rejected"] }, + "expected_diagnostic": { "type": "string", "minLength": 1 } + } + }, + "collection": { + "type": "object", + "additionalProperties": false, + "required": ["id", "members"], + "properties": { + "id": { "type": "string", "pattern": "^fixture\\.collection\\.[a-z0-9.-]+$" }, + "members": { "type": "array", "minItems": 1, "items": { "type": "string" } } + } + }, + "scenario": { + "type": "object", + "additionalProperties": false, + "required": ["id", "tier", "fixtures"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9_]+$" }, + "tier": { "enum": ["fast", "tool_backed", "maximal"] }, + "fixtures": { "type": "array", "minItems": 1, "items": { "type": "string" } } + } + } + } +} diff --git a/tests/fixtures/golden-artifacts/v1/corpus.json b/tests/fixtures/golden-artifacts/v1/corpus.json new file mode 100644 index 0000000..0f8d595 --- /dev/null +++ b/tests/fixtures/golden-artifacts/v1/corpus.json @@ -0,0 +1,155 @@ +{ + "schema_version": "renderflow.golden-corpus/v1", + "corpus_version": "1.0.0", + "license": "CC0-1.0", + "redistribution": "Synthetic fixtures created for Renderflow conformance; no publication or third-party content.", + "fixtures": [ + { + "id": "fixture.document.markdown", + "filename": "synthetic.md", + "encoding": "utf8", + "payload": "# Synthetic artifact\n\nA bounded, redistribution-safe Renderflow fixture.\n", + "expected_format": "markdown", + "expected_media_type": "text/markdown", + "family": "document", + "expected_outcome": "accepted" + }, + { + "id": "fixture.document.pdf", + "filename": "synthetic.pdf", + "encoding": "utf8", + "payload": "%PDF-1.4\n1 0 obj<>endobj\ntrailer<>\n%%EOF\n", + "expected_format": "pdf", + "expected_media_type": "application/pdf", + "family": "document", + "expected_outcome": "accepted" + }, + { + "id": "fixture.image.png", + "filename": "pixel.png", + "encoding": "hex", + "payload": "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154089963000100000500010d0a2db40000000049454e44ae426082", + "expected_format": "png", + "expected_media_type": "image/png", + "family": "image", + "expected_outcome": "accepted" + }, + { + "id": "fixture.image.svg", + "filename": "shapes.svg", + "encoding": "utf8", + "payload": "\n", + "expected_format": "svg", + "expected_media_type": "image/svg+xml", + "family": "image", + "expected_outcome": "accepted" + }, + { + "id": "fixture.audio.wav", + "filename": "silence.wav", + "encoding": "hex", + "payload": "524946462400000057415645666d74201000000001000100401f0000803e0000020010006461746100000000", + "expected_format": "wav", + "expected_media_type": "audio/wav", + "family": "audio", + "expected_outcome": "accepted" + }, + { + "id": "fixture.video.mp4", + "filename": "container.mp4", + "encoding": "hex", + "payload": "000000186674797069736f6d0000020069736f6d6d703431", + "expected_format": "mp4", + "expected_media_type": "video/mp4", + "family": "video", + "expected_outcome": "accepted" + }, + { + "id": "fixture.archive.zip", + "filename": "empty.zip", + "encoding": "hex", + "payload": "504b0506000000000000000000000000000000000000", + "expected_format": "zip", + "expected_media_type": "application/zip", + "family": "archive", + "expected_outcome": "accepted" + }, + { + "id": "fixture.data.json", + "filename": "records.json", + "encoding": "utf8", + "payload": "[{\"id\":1,\"kind\":\"synthetic\"},{\"id\":2,\"kind\":\"synthetic\"}]\n", + "expected_format": "json", + "expected_media_type": "application/json", + "family": "data", + "expected_outcome": "accepted" + }, + { + "id": "fixture.subtitle.srt", + "filename": "captions.srt", + "encoding": "utf8", + "payload": "1\n00:00:00,000 --> 00:00:01,000\nSynthetic caption.\n", + "expected_format": "srt", + "expected_media_type": "application/x-subrip", + "family": "subtitle", + "expected_outcome": "accepted" + }, + { + "id": "fixture.corrupt.zip", + "filename": "corrupt.zip", + "encoding": "hex", + "payload": "504b03047472756e6361746564", + "expected_format": "zip", + "expected_media_type": "application/zip", + "family": "archive", + "expected_outcome": "rejected", + "expected_diagnostic": "intake.archive.malformed" + }, + { + "id": "fixture.mismatch.jpeg-png", + "filename": "not-a-jpeg.jpg", + "encoding": "hex", + "payload": "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154089963000100000500010d0a2db40000000049454e44ae426082", + "expected_format": "png", + "expected_media_type": "image/png", + "family": "image", + "expected_outcome": "conflict" + }, + { + "id": "fixture.unknown.binary", + "filename": "unknown.bin", + "encoding": "hex", + "payload": "deadbeef001337ff", + "expected_media_type": "application/octet-stream", + "family": "unknown", + "expected_outcome": "accepted" + } + ], + "collections": [ + { + "id": "fixture.collection.ordered-pages", + "members": ["fixture.document.markdown", "fixture.data.json"] + } + ], + "scenarios": [ + { "id": "inspect_only", "tier": "fast", "fixtures": ["fixture.document.markdown", "fixture.document.pdf", "fixture.image.png", "fixture.image.svg", "fixture.audio.wav", "fixture.video.mp4", "fixture.archive.zip", "fixture.data.json", "fixture.subtitle.srt", "fixture.corrupt.zip", "fixture.mismatch.jpeg-png", "fixture.unknown.binary"] }, + { "id": "direct_deterministic_conversion", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "multi_step_conversion", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "multi_target_shared_intermediate", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "cross_family_derivative", "tier": "fast", "fixtures": ["fixture.document.markdown", "fixture.data.json"] }, + { "id": "collection_aggregation", "tier": "fast", "fixtures": ["fixture.collection.ordered-pages"] }, + { "id": "cache_cold_warm_partial_invalidation", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "missing_provider_fallback", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "validation_failure", "tier": "fast", "fixtures": ["fixture.corrupt.zip"] }, + { "id": "branch_local_failure_partial_result", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "cancellation", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "checkpoint_resume", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "version_invalidation", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "privacy_redaction_gate", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "bounded_recursive_extraction", "tier": "fast", "fixtures": ["fixture.archive.zip", "fixture.corrupt.zip"] }, + { "id": "everything_profile", "tier": "fast", "fixtures": ["fixture.document.markdown"] }, + { "id": "magazine_release_bundle", "tier": "fast", "fixtures": ["fixture.document.markdown", "fixture.image.png"] }, + { "id": "optional_tool_providers", "tier": "tool_backed", "fixtures": ["fixture.document.markdown", "fixture.video.mp4"] }, + { "id": "maximal_release_matrix", "tier": "maximal", "fixtures": ["fixture.document.markdown", "fixture.image.png", "fixture.audio.wav", "fixture.video.mp4"] } + ] +}