Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions crates/renderflow-core/src/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,11 @@ pub fn run_cli(cli: Cli) -> Result<()> {
format,
} => commands::tools::run_variants(&id, models_dir.as_deref(), &format)?,
},
Some(Commands::Capabilities { format, transforms }) => {
commands::tools::run_capabilities(transforms.as_deref(), &format)?
}
Some(Commands::Capabilities {
format,
transforms,
matrix,
}) => commands::tools::run_capabilities(transforms.as_deref(), &format, matrix)?,
Some(Commands::Spec { subcommand }) => match subcommand {
SpecCommands::Validate { config, format } => {
commands::spec::run_validate(&config, &format)?
Expand Down
3 changes: 3 additions & 0 deletions crates/renderflow-core/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,9 @@ pub enum Commands {
/// Optional transform YAML whose dynamic providers should be included.
#[arg(long, value_name = "FILE")]
transforms: Option<String>,
/// Emit the generated artifact capability conformance matrix.
#[arg(long)]
matrix: bool,
},

/// Validate, migrate, and export the Renderflow execution specification.
Expand Down
35 changes: 33 additions & 2 deletions crates/renderflow-core/src/commands/tools.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use serde::Serialize;
use crate::super_resolution::{UpscaylModelCatalog, UPSCAYL_TOOL_ID};
use crate::toolchain::{ToolAvailability, ToolDescriptor, ToolRegistry};
use crate::transforms::yaml_loader::load_tool_registry_from_yaml;
use crate::validation::CapabilityConformanceMatrix;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum StructuredFormat {
Expand Down Expand Up @@ -184,10 +185,40 @@ pub fn run_variants(id: &str, models_dir: Option<&str>, format: &str) -> Result<
Ok(())
}

pub fn run_capabilities(transforms: Option<&str>, format: &str) -> Result<()> {
pub fn run_capabilities(transforms: Option<&str>, format: &str, matrix: bool) -> Result<()> {
let format = StructuredFormat::parse(format)?;
if matrix {
let matrix = CapabilityConformanceMatrix::builtins();
if format != StructuredFormat::Text {
return emit_serialized(&matrix, format);
}
println!("Renderflow Artifact Capability Conformance");
println!("==========================================");
println!(
"{:<12} {:<14} {:<9} Validators",
"Format", "Status", "Executor"
);
for row in matrix.formats {
let status = serde_json::to_value(row.support_status)?
.as_str()
.unwrap_or("unknown")
.to_string();
println!(
"{:<12} {:<14} {:<9} {}",
row.format,
status,
if row.executor_implemented {
"yes"
} else {
"no"
},
row.validator_ids.join(", ")
);
}
return Ok(());
}
let registry = load_registry(transforms)?;
let capabilities: BTreeMap<String, Vec<String>> = registry.capabilities();
let format = StructuredFormat::parse(format)?;

if format != StructuredFormat::Text {
return emit_serialized(&capabilities, format);
Expand Down
22 changes: 22 additions & 0 deletions crates/renderflow-core/src/evidence.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,24 @@ pub enum ValidationState {
NotRequested,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct ValidationDiagnostic {
pub code: String,
pub message: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct ValidatorEvidence {
pub validator_id: String,
pub validator_version: String,
pub provider: String,
pub state: ValidationState,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub diagnostics: Vec<ValidationDiagnostic>,
}

#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CacheDisposition {
Expand Down Expand Up @@ -139,6 +157,8 @@ pub struct ArtifactEvidence {
pub sources: Vec<String>,
pub cache: CacheDisposition,
pub validation: ValidationState,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub validation_evidence: Vec<ValidatorEvidence>,
pub fidelity: FidelityDeclaration,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub warnings: Vec<String>,
Expand Down Expand Up @@ -179,6 +199,7 @@ impl ArtifactEvidence {
sources: artifact.sources().iter().map(ToString::to_string).collect(),
cache,
validation,
validation_evidence: Vec::new(),
fidelity,
warnings: Vec::new(),
metadata: artifact
Expand Down Expand Up @@ -529,6 +550,7 @@ mod tests {
sources: vec!["artifact:sha256:SOURCE123".to_string()],
cache: CacheDisposition::Miss,
validation: ValidationState::Valid,
validation_evidence: Vec::new(),
fidelity: FidelityDeclaration::Lossless,
warnings: Vec::new(),
metadata: BTreeMap::new(),
Expand Down
3 changes: 3 additions & 0 deletions crates/renderflow-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
//! Exposes the core subsystems for use by benchmarks, tests, and external
//! integrations. The binary entrypoint lives in `main.rs`.

#![recursion_limit = "256"]

mod adapters;
pub mod ai;
pub mod app;
Expand Down Expand Up @@ -30,6 +32,7 @@ pub mod strategies;
pub mod super_resolution;
pub mod toolchain;
pub mod transforms;
pub mod validation;

pub use evidence::{ArtifactManifest, RunManifest};
pub use sdk::{
Expand Down
152 changes: 130 additions & 22 deletions crates/renderflow-core/src/planning.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,16 @@ use crate::graph::{
};
use crate::optimization::OptimizationMode;
use crate::spec::{
load_spec, AiPolicy, CollisionPolicy, SelectorSet, SourceKind, SourceSpec, SourceSpecVersion,
SpecV2, TargetSelection, TargetSpec,
load_spec, AiPolicy, CollisionPolicy, RejectedLossClass, SelectorSet, SourceKind, SourceSpec,
SourceSpecVersion, SpecV2, TargetSelection, TargetSpec, ValidationFailureMode,
};
use crate::super_resolution::{select_upscayl_variants, UpscaylModelCatalog};
use crate::toolchain::{
transform_capability_id, ToolDeterminism, ToolId, ToolLocality, ToolRegistry,
ToolRuntimeContext, ToolchainSnapshot,
};
use crate::transforms::yaml_loader::build_graph_executor_and_tools_from_yaml;
use crate::validation::{ArtifactValidationOutcome, ValidationRegistry};

const BUILTIN_ADAPTER_EVIDENCE: &str = "builtin.strategy";

Expand Down Expand Up @@ -484,8 +485,11 @@ pub fn execute(mut resolved: ResolvedExecution, dry_run: bool) -> Result<Canonic
let mut diagnostics = plan_diagnostics(&resolved);
diagnostics.append(&mut report.diagnostics);
let mut output_locators = HashMap::<String, String>::new();
let mut validation_outcomes = HashMap::<String, ArtifactValidationOutcome>::new();
let mut blocked_artifacts = HashSet::<String>::new();
let mut actual_outputs = Vec::new();
let mut target_failures = false;
let mut fatal_validation_failure = false;

if let Err(error) = validate_post_execution_budgets(&resolved, &report.artifacts) {
target_failures = true;
Expand All @@ -497,7 +501,8 @@ pub fn execute(mut resolved: ResolvedExecution, dry_run: bool) -> Result<Canonic
});
}

for (target, destination) in resolved.targets.iter().zip(predicted.iter()) {
let validation_registry = ValidationRegistry::builtins();
for target in &resolved.targets {
let Some(artifact) = report.artifacts.get(&target.format) else {
target_failures = true;
diagnostics.push(ExecutionDiagnostic {
Expand All @@ -511,14 +516,95 @@ pub fn execute(mut resolved: ResolvedExecution, dry_run: bool) -> Result<Canonic
});
continue;
};
if resolved.spec.execution.validation.required && artifact.size_bytes() == 0 {
target_failures = true;

let outcome = if resolved.spec.execution.validation.required {
validation_registry.validate_in_store(
artifact,
target.format,
&resolved.spec.execution.validation.validators,
&store,
)
} else {
ArtifactValidationOutcome {
state: ValidationState::Skipped,
validators: Vec::new(),
}
};
let step_id = producing_step_id(artifact, &report.steps);
if let Some(step) = report.steps.iter_mut().find(|step| {
step.output_artifacts
.iter()
.any(|artifact_id| artifact_id == artifact.id().as_str())
}) {
step.validation = outcome.state;
}
for validator in &outcome.validators {
for validator_diagnostic in &validator.diagnostics {
let blocking = validator.state == ValidationState::Invalid
|| (validator.state == ValidationState::Unavailable
&& !resolved.spec.execution.validation.allow_unavailable);
diagnostics.push(ExecutionDiagnostic {
severity: if blocking {
match resolved.spec.execution.validation.failure_mode {
ValidationFailureMode::Fatal => DiagnosticSeverity::FatalFailure,
ValidationFailureMode::BranchLocal => {
DiagnosticSeverity::RecoverableFailure
}
}
} else {
DiagnosticSeverity::Warning
},
code: validator_diagnostic.code.clone(),
message: format!(
"{} (validator {}@{}, provider {})",
validator_diagnostic.message,
validator.validator_id,
validator.validator_version,
validator.provider
),
step_id: step_id.clone(),
});
}
}
let validation_blocked = outcome.state == ValidationState::Invalid
|| (outcome.state == ValidationState::Unavailable
&& !resolved.spec.execution.validation.allow_unavailable);
let fidelity = producing_fidelity(artifact, &report.steps);
let fidelity_blocked = resolved
.spec
.execution
.reject_loss_classes
.iter()
.any(|class| rejects_fidelity(*class, fidelity));
if fidelity_blocked {
diagnostics.push(ExecutionDiagnostic {
severity: DiagnosticSeverity::FatalFailure,
code: "validation.empty_artifact".to_string(),
message: format!("Target '{}' produced an empty artifact", target.format),
step_id: producing_step_id(artifact, &report.steps),
severity: match resolved.spec.execution.validation.failure_mode {
ValidationFailureMode::Fatal => DiagnosticSeverity::FatalFailure,
ValidationFailureMode::BranchLocal => DiagnosticSeverity::RecoverableFailure,
},
code: "fidelity.rejected_loss_class".to_string(),
message: format!(
"Target '{}' has rejected fidelity class '{:?}'",
target.format, fidelity
)
.to_lowercase(),
step_id,
});
}
if validation_blocked || fidelity_blocked {
target_failures = true;
blocked_artifacts.insert(artifact.id().to_string());
fatal_validation_failure |=
resolved.spec.execution.validation.failure_mode == ValidationFailureMode::Fatal;
}
validation_outcomes.insert(artifact.id().to_string(), outcome);
}

for (target, destination) in resolved.targets.iter().zip(predicted.iter()) {
let Some(artifact) = report.artifacts.get(&target.format) else {
continue;
};
if fatal_validation_failure || blocked_artifacts.contains(artifact.id().as_str()) {
continue;
}
if target_failures
Expand Down Expand Up @@ -549,6 +635,7 @@ pub fn execute(mut resolved: ResolvedExecution, dry_run: bool) -> Result<Canonic
&report,
&output_locators,
&diagnostics,
&validation_outcomes,
);
let has_failed_step = report
.steps
Expand Down Expand Up @@ -812,6 +899,7 @@ fn artifact_evidence(
report: &DagExecutionReport,
output_locators: &HashMap<String, String>,
diagnostics: &[ExecutionDiagnostic],
validation_outcomes: &HashMap<String, ArtifactValidationOutcome>,
) -> Vec<ArtifactEvidence> {
let mut evidence = vec![source_artifact_evidence(resolved, source)];
let mut artifacts = report.artifacts.iter().collect::<Vec<_>>();
Expand Down Expand Up @@ -842,19 +930,10 @@ fn artifact_evidence(
.iter()
.any(|artifact_id| artifact_id == artifact.id().as_str())
});
let invalid = producing_step.is_some_and(|step| {
diagnostics.iter().any(|diagnostic| {
diagnostic.code.starts_with("validation.")
&& diagnostic.step_id.as_deref() == Some(step.step_id.as_str())
})
});
let validation = if invalid {
ValidationState::Invalid
} else if resolved.spec.execution.validation.required && target.is_some() {
ValidationState::Valid
} else {
ValidationState::NotRequested
};
let outcome = validation_outcomes.get(artifact.id().as_str());
let validation = outcome
.map(|outcome| outcome.state)
.unwrap_or(ValidationState::NotRequested);
let producer = producing_step
.map(|step| ProducerEvidence {
system: "renderflow".to_string(),
Expand All @@ -874,6 +953,9 @@ fn artifact_evidence(
let mut artifact_evidence = ArtifactEvidence::from_artifact(
artifact, role, lifecycle, locator, producer, validation, fidelity,
);
artifact_evidence.validation_evidence = outcome
.map(|outcome| outcome.validators.clone())
.unwrap_or_default();
if let Some(step) = producing_step {
artifact_evidence.warnings = diagnostics
.iter()
Expand Down Expand Up @@ -916,6 +998,32 @@ fn producing_step_id(artifact: &Artifact, steps: &[StepEvidence]) -> Option<Stri
.map(|step| step.step_id.clone())
}

fn producing_fidelity(artifact: &Artifact, steps: &[StepEvidence]) -> FidelityDeclaration {
steps
.iter()
.find(|step| {
step.output_artifacts
.iter()
.any(|artifact_id| artifact_id == artifact.id().as_str())
})
.map(|step| step.fidelity)
.unwrap_or(FidelityDeclaration::Unknown)
}

fn rejects_fidelity(class: RejectedLossClass, fidelity: FidelityDeclaration) -> bool {
matches!(
(class, fidelity),
(RejectedLossClass::Lossless, FidelityDeclaration::Lossless)
| (RejectedLossClass::Partial, FidelityDeclaration::Partial)
| (RejectedLossClass::Lossy, FidelityDeclaration::Lossy)
| (
RejectedLossClass::PathDependent,
FidelityDeclaration::PathDependent
)
| (RejectedLossClass::Unknown, FidelityDeclaration::Unknown)
)
}

fn enrich_step_versions(steps: &mut [StepEvidence], toolchain: Option<&ToolchainSnapshot>) {
for step in steps {
let version = step.provider.as_deref().and_then(|provider| {
Expand Down
Loading
Loading