From cced91ec35a64cd218d710810fceb99dafd93c81 Mon Sep 17 00:00:00 2001 From: Ha Phan Tran Date: Thu, 12 Nov 2026 22:01:29 -0500 Subject: [PATCH 1/2] Add a twin loop that lifts simulator observations back onto the model Generation fidelity compares the model with the generated files. The twin loop compares the model with what the generated scene does when it runs. - The Smart Warehouse Omniverse templates stamp every layout prim and layout record with the ID of its model element (spatialDsl:elementId, id). - observe.py records the world pose of every stamped prim as a simulator-neutral observation log. The Isaac Sim bridge writes it with --observe-out; run_standin_sim.py writes the same log with no simulator. - benchmarks/twin-loop.ts lifts each sample onto a copy of the model (design pose plus the residual between observed and authored scene pose) and evaluates the design-time spatial rules on it. It reports realization offsets, runtime drifts, rule verdicts that differ between model and run, and a reconciliation proposal, each traced to the model element, prim, generated file, and template. --- benchmarks/README.md | 69 ++- benchmarks/lib/twin.ts | 469 ++++++++++++++++++ benchmarks/package.json | 5 +- benchmarks/twin-loop.ts | 200 ++++++++ docs/user-guide/omniverse-code-generation.md | 4 + .../smart-warehouse-omniverse-project.json | 4 +- .../omniverse-assets/warehouse_sim/README.md | 25 +- .../omniverse-assets/warehouse_sim/observe.py | 159 ++++++ .../warehouse_sim/run_standin_sim.py | 128 +++++ .../warehouse_sim/run_warehouse_sim.py | 30 +- .../warehouse_sim/test_observe.py | 127 +++++ .../smart-warehouse-omniverse-project.json | 4 +- 12 files changed, 1213 insertions(+), 11 deletions(-) create mode 100644 benchmarks/lib/twin.ts create mode 100644 benchmarks/twin-loop.ts create mode 100644 examples/omniverse-assets/warehouse_sim/observe.py create mode 100644 examples/omniverse-assets/warehouse_sim/run_standin_sim.py create mode 100644 examples/omniverse-assets/warehouse_sim/test_observe.py diff --git a/benchmarks/README.md b/benchmarks/README.md index 23f8052..e94a4c5 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,7 +1,7 @@ # Evaluation scripts -Two unattended scripts measure SpatialDSL Studio without a browser, an HTTP -server, or a database. Both call the backend engines in-process and write their +Three unattended scripts measure SpatialDSL Studio without a browser, an HTTP +server, or a database. All call the backend engines in-process and write their results to `benchmarks/results/` (git-ignored). ## Prerequisites @@ -9,7 +9,7 @@ results to `benchmarks/results/` (git-ignored). ```bash cd backend && npm ci # the scripts reuse the backend's ts-node and engines cd ../benchmarks -python3 -m venv .venv && .venv/bin/pip install usd-core # fidelity only; provides pxr +python3 -m venv .venv && .venv/bin/pip install usd-core # fidelity and twin loop; provides pxr ``` ## Scalability @@ -53,3 +53,66 @@ error or warning. Outputs: `results/fidelity.json`, `results/fidelity.md`, and the generated bundles in `results/fidelity/`. Expected runtime: under 10 seconds. + +## Twin loop + +```bash +cd benchmarks && npm run twin-loop +``` + +Generation fidelity compares the model with the generated files. The twin loop +continues from there: it compares the model with what the scene does when it +runs, and brings the result back in the model's own terms. + +1. The bundled Smart Warehouse model is generated into + `results/twin-loop/bundle/`. Every generated prim and layout record carries + the ID of its model element (`spatialDsl:elementId`, `id`). +2. `observe.py snapshot` reads the composed stage with pxr and records the + authored pose of every stamped prim, with the layer that authors it. +3. A simulator runs the bundle and records an observation log: the world pose + of every stamped prim over time, plus which elements the simulator drives. +4. Each sample is lifted onto a copy of the model: the observed model pose is + the design pose plus the difference between the observed and the authored + scene pose. The design-time spatial rules are then evaluated, unchanged, on + the lifted model. + +The report lists, per run: + +- **realization offsets**: undriven elements that never sat at their authored + pose; +- **runtime drifts**: undriven elements that moved during the run, with the + onset time; +- **rule verdicts that differ** between the model and the run, with the time + span in which they held; +- a **reconciliation proposal**: the model changes that accepting the observed + state would make (`reconciled-model.json`, importable as a model), and the + rule issues that model would gain or lose. + +Each finding names the model element, the prim, the generated file, and the +template that produced that file. + +Without arguments the script runs three scenarios on the simulator stand-in +(`run_standin_sim.py`: the MAS brain drives the robots kinematically, with no +physics engine): an undisturbed run, a storage rack pushed into the aisle at +t = 20 s, and a dock resting 150 mm above its authored pose. It exits non-zero +if the findings differ from what was injected. The stand-in checks the loop, +not the physics: deviations exist only where a scenario injects them. + +To analyse a run recorded in Isaac Sim, run the bundle from +`results/twin-loop/bundle/` with the brain and the bridge (see +`examples/omniverse-assets/warehouse_sim/README.md`) and pass the log back: + +```bash +python3 warehouse_brain.py --layout /warehouse_layout.json # terminal 1 +/isaac-sim/python.sh run_warehouse_sim.py --layout /warehouse_layout.json \ + --scene /warehouse_simulation.usda \ + --observe-out observations.jsonl --headless --max-seconds 60 # terminal 2 +npm run twin-loop -- --observations observations.jsonl +``` + +The log records the hash of the layout it ran against; the script refuses a log +recorded against a different generation of the model. + +Outputs: `results/twin-loop.json`, `results/twin-loop.md`, and per-scenario +`report.json`, `report.md`, `observations.jsonl`, and `reconciled-model.json` +in `results/twin-loop/`. Expected runtime: about 5 seconds. diff --git a/benchmarks/lib/twin.ts b/benchmarks/lib/twin.ts new file mode 100644 index 0000000..55c4344 --- /dev/null +++ b/benchmarks/lib/twin.ts @@ -0,0 +1,469 @@ +/** + * Twin loop analysis: bring what a simulator observed back into the model's + * own terms, then reuse the design-time rule engine on it. + * + * Three states of every placed element are compared: + * design the pose in the model (millimetres, degrees); + * authored the pose of the generated prim before the simulation starts; + * observed the pose of the same prim while the simulation runs. + * + * The generator stamps each prim with the ID of its model element, so the + * three states line up without matching names. Observations are lifted by + * residual: the observed model pose is the design pose plus the difference + * between the observed and the authored scene pose. This needs the scene's + * unit (metres) and nothing else about how a template placed the prim. + */ +import fs from 'fs'; +import { Metamodel, Model, ModelElement, ValidationIssue } from '../../shared/types'; +import { evaluateSpatialRules } from '../../shared/spatial-rules'; + +export const OBSERVATION_SCHEMA = 'spatialdsl.observations/1'; +const MM_PER_M = 1000; + +export interface ObservedPose { + elementId: string; + prim: string; + layer?: string | null; + x: number; + y: number; + z: number; + yaw: number; +} + +export interface ObservationHeader { + schema: string; + source: string; + layoutSha256?: string | null; + scene?: string | null; + controlledElementIds: string[]; +} + +export interface ObservationSample { + t: number; + poses: Map; +} + +export interface ObservationLog { + header: ObservationHeader; + samples: ObservationSample[]; +} + +export interface GeneratedFileTrace { + filename: string; + contentHash: string; + templateId?: string; + templateName: string; +} + +/** Where a finding comes from: model element, generated prim, generated file, generating template. */ +export interface TraceChain { + elementId: string; + elementName: string; + className: string; + prim: string; + generatedFile?: string; + template?: { id?: string; name: string }; +} + +export interface PoseMm { + x: number; + y: number; + z: number; + rotationZ: number; +} + +export interface DeviationFinding { + /** realization-offset: never at its authored pose. runtime-drift: moved during the run. */ + kind: 'realization-offset' | 'runtime-drift'; + trace: TraceChain; + onsetT: number; + deltaMm: { x: number; y: number; z: number }; + deltaYawDeg: number; + designPose: PoseMm; + observedPose: PoseMm; +} + +export interface RuleFinding { + /** runtime-rule-violation: holds at runtime only. design-violation-not-observed: holds in the model only. */ + kind: 'runtime-rule-violation' | 'design-violation-not-observed'; + ruleId: string; + ruleName: string; + severity: ValidationIssue['severity']; + message: string; + firstT?: number; + lastT?: number; + samples: number; + elements: TraceChain[]; +} + +export interface ModelPatchEntry { + elementId: string; + elementName: string; + path: 'presentation.position3D' | 'presentation.rotationZ'; + from: unknown; + to: unknown; +} + +export interface TwinReport { + provenance: { + modelId: string; + modelHash: string; + metamodelHash: string; + generatedFiles: GeneratedFileTrace[]; + observation: { + source: string; + scene?: string | null; + layoutSha256?: string | null; + samples: number; + durationS: number; + tracedElements: number; + controlledElements: number; + }; + tolerances: TwinTolerances; + }; + summary: { + realizationOffsets: number; + runtimeDrifts: number; + runtimeRuleViolations: number; + designViolationsNotObserved: number; + designRuleIssues: number; + }; + deviations: DeviationFinding[]; + ruleFindings: RuleFinding[]; + motion: Array<{ trace: TraceChain; pathLengthMm: number }>; + reconciliation: { + patch: ModelPatchEntry[]; + /** Rule issues the model would gain or lose if the patch were accepted. */ + ruleImpact: { introduced: string[]; resolved: string[] }; + }; +} + +export interface TwinTolerances { + positionMm: number; + angleDeg: number; +} + +export const DEFAULT_TOLERANCES: TwinTolerances = { positionMm: 1, angleDeg: 0.1 }; + +const round = (value: number, digits: number): number => { + const factor = 10 ** digits; + return Math.round(value * factor) / factor; +}; +const wrapDegrees = (value: number): number => (((value + 180) % 360) + 360) % 360 - 180; + +export function readObservationLog(file: string): ObservationLog { + const records = fs.readFileSync(file, 'utf8').split('\n').filter(line => line.trim()).map(line => JSON.parse(line)); + const header = records[0] as ObservationHeader | undefined; + if (!header || header.schema !== OBSERVATION_SCHEMA) { + throw new Error(`${file} is not a ${OBSERVATION_SCHEMA} log`); + } + const samples: ObservationSample[] = []; + records.slice(1).forEach(record => { + const { t, ...pose } = record; + if (samples.length === 0 || samples[samples.length - 1].t !== t) samples.push({ t, poses: new Map() }); + samples[samples.length - 1].poses.set(pose.elementId, pose as ObservedPose); + }); + if (samples.length === 0) throw new Error(`${file} holds no samples`); + return { header: { ...header, controlledElementIds: header.controlledElementIds || [] }, samples }; +} + +const designPose = (element: ModelElement): PoseMm => ({ + x: element.presentation!.position3D!.x, + y: element.presentation!.position3D!.y, + z: element.presentation!.position3D!.z, + rotationZ: element.presentation!.rotationZ ?? 0, +}); + +/** Residual between two scene poses, in model units. */ +const residual = (from: ObservedPose, to: ObservedPose) => ({ + x: round((to.x - from.x) * MM_PER_M, 3), + y: round((to.y - from.y) * MM_PER_M, 3), + z: round((to.z - from.z) * MM_PER_M, 3), + yaw: round(wrapDegrees(to.yaw - from.yaw), 6), +}); + +const exceeds = (delta: ReturnType, tolerances: TwinTolerances): boolean => ( + Math.hypot(delta.x, delta.y, delta.z) > tolerances.positionMm || Math.abs(delta.yaw) > tolerances.angleDeg +); + +/** The model pose an observation implies; components with no residual keep the design value exactly. */ +export function liftPose(element: ModelElement, authored: ObservedPose, observed: ObservedPose): PoseMm { + const design = designPose(element); + const delta = residual(authored, observed); + const shifted = (value: number, by: number): number => by === 0 ? value : round(value + by, 6); + return { + x: shifted(design.x, delta.x), + y: shifted(design.y, delta.y), + z: shifted(design.z, delta.z), + rotationZ: delta.yaw === 0 ? design.rotationZ : round(wrapDegrees(design.rotationZ + delta.yaw), 6), + }; +} + +/** A copy of the model in which every observed element sits where the simulator saw it. */ +export function liftModel( + model: Model, + authored: Map, + observed: Map, + skip: Set = new Set() +): Model { + return { + ...model, + elements: model.elements.map(element => { + const from = authored.get(element.id); + const to = observed.get(element.id); + if (!from || !to || skip.has(element.id) || !element.presentation?.position3D) return element; + const pose = liftPose(element, from, to); + return { + ...element, + presentation: { + ...element.presentation, + position3D: { ...element.presentation.position3D, x: pose.x, y: pose.y, z: pose.z }, + rotationZ: pose.rotationZ, + }, + }; + }), + }; +} + +const issueKey = (issue: ValidationIssue): string => ( + [issue.constraintId, issue.elementId, ...(issue.relatedElementIds || [])].join('|') +); + +export function analyzeRun(input: { + model: Model; + metamodel: Metamodel; + modelHash: string; + metamodelHash: string; + generatedFiles: GeneratedFileTrace[]; + authored: ObservationLog; + observations: ObservationLog; + tolerances?: TwinTolerances; +}): { report: TwinReport; reconciledModel: Model } { + const { model, metamodel, observations } = input; + const tolerances = input.tolerances || DEFAULT_TOLERANCES; + const authored = input.authored.samples[0].poses; + const first = observations.samples[0]; + const last = observations.samples[observations.samples.length - 1]; + const controlled = new Set(observations.header.controlledElementIds); + const classNames = new Map(metamodel.classes.map(metaClass => [metaClass.id, metaClass.name])); + const ruleNames = new Map((metamodel.spatialRules || []).map(rule => [rule.id, rule.name])); + const elements = new Map(model.elements.map(element => [element.id, element])); + const filesByName = new Map(input.generatedFiles.map(file => [file.filename, file])); + + const traceOf = (elementId: string): TraceChain => { + const element = elements.get(elementId); + const pose = authored.get(elementId); + const file = pose?.layer ? filesByName.get(pose.layer) : undefined; + return { + elementId, + elementName: String(element?.name || element?.style?.name || elementId), + className: element ? classNames.get(element.modelElementId) || element.modelElementId : 'unknown', + prim: pose?.prim || 'unknown', + ...(pose?.layer ? { generatedFile: pose.layer } : {}), + ...(file ? { template: { id: file.templateId, name: file.templateName } } : {}), + }; + }; + + const traced = model.elements.filter(element => ( + element.presentation?.position3D && authored.has(element.id) && first.poses.has(element.id) + )); + + // Elements the simulator does not drive are expected to stay where the model put them. + const deviations: DeviationFinding[] = []; + const finding = ( + kind: DeviationFinding['kind'], element: ModelElement, onsetT: number, from: ObservedPose, to: ObservedPose + ): DeviationFinding => { + const delta = residual(from, to); + return { + kind, + trace: traceOf(element.id), + onsetT, + deltaMm: { x: delta.x, y: delta.y, z: delta.z }, + deltaYawDeg: delta.yaw, + designPose: designPose(element), + observedPose: liftPose(element, authored.get(element.id)!, to), + }; + }; + traced.filter(element => !controlled.has(element.id)).forEach(element => { + const start = first.poses.get(element.id)!; + if (exceeds(residual(authored.get(element.id)!, start), tolerances)) { + deviations.push(finding('realization-offset', element, first.t, authored.get(element.id)!, start)); + } + const onset = observations.samples.find(sample => { + const pose = sample.poses.get(element.id); + return pose !== undefined && exceeds(residual(start, pose), tolerances); + }); + const end = last.poses.get(element.id); + if (onset && end) deviations.push(finding('runtime-drift', element, onset.t, start, end)); + }); + + const motion = traced.filter(element => controlled.has(element.id)).map(element => { + let pathLengthMm = 0; + let previous = first.poses.get(element.id)!; + observations.samples.forEach(sample => { + const pose = sample.poses.get(element.id); + if (!pose) return; + pathLengthMm += Math.hypot(pose.x - previous.x, pose.y - previous.y, pose.z - previous.z) * MM_PER_M; + previous = pose; + }); + return { trace: traceOf(element.id), pathLengthMm: round(pathLengthMm, 1) }; + }); + + // The design-time rules, evaluated unchanged on the lifted model of every sample. + const designIssues = new Map(evaluateSpatialRules(metamodel, model).map(issue => [issueKey(issue), issue])); + const seen = new Map(); + observations.samples.forEach(sample => { + evaluateSpatialRules(metamodel, liftModel(model, authored, sample.poses)).forEach(issue => { + const entry = seen.get(issueKey(issue)); + if (entry) { + entry.lastT = sample.t; + entry.samples += 1; + } else { + seen.set(issueKey(issue), { issue, firstT: sample.t, lastT: sample.t, samples: 1 }); + } + }); + }); + const ruleFinding = (kind: RuleFinding['kind'], issue: ValidationIssue, span?: { + firstT: number; lastT: number; samples: number; + }): RuleFinding => ({ + kind, + ruleId: issue.constraintId || 'unknown', + ruleName: ruleNames.get(issue.constraintId || '') || issue.constraintId || 'unknown', + severity: issue.severity, + message: issue.message, + ...(span ? { firstT: span.firstT, lastT: span.lastT } : {}), + samples: span?.samples || 0, + elements: [issue.elementId, ...(issue.relatedElementIds || [])] + .filter((id): id is string => Boolean(id)).map(traceOf), + }); + const ruleFindings: RuleFinding[] = [ + ...[...seen.entries()].filter(([key]) => !designIssues.has(key)) + .map(([, entry]) => ruleFinding('runtime-rule-violation', entry.issue, entry)), + ...[...designIssues.entries()].filter(([key]) => !seen.has(key)) + .map(([, issue]) => ruleFinding('design-violation-not-observed', issue)), + ]; + + // Reconciliation proposal: adopt the final observed pose of every element that + // deviated. Driven elements keep their design pose, which is where they start. + const deviated = new Set(deviations.map(item => item.trace.elementId)); + const keep = new Set(model.elements.filter(element => !deviated.has(element.id)).map(element => element.id)); + const reconciledModel = liftModel(model, authored, last.poses, keep); + const patch: ModelPatchEntry[] = []; + reconciledModel.elements.forEach(element => { + const before = elements.get(element.id)!; + if (element === before) return; + const elementName = traceOf(element.id).elementName; + const from = before.presentation!.position3D!; + const to = element.presentation!.position3D!; + if (from.x !== to.x || from.y !== to.y || from.z !== to.z) { + patch.push({ elementId: element.id, elementName, path: 'presentation.position3D', from, to }); + } + if ((before.presentation!.rotationZ ?? 0) !== element.presentation!.rotationZ) { + patch.push({ + elementId: element.id, elementName, path: 'presentation.rotationZ', + from: before.presentation!.rotationZ ?? 0, to: element.presentation!.rotationZ, + }); + } + }); + const reconciledIssues = new Map(evaluateSpatialRules(metamodel, reconciledModel).map(issue => [issueKey(issue), issue])); + + const report: TwinReport = { + provenance: { + modelId: model.id, + modelHash: input.modelHash, + metamodelHash: input.metamodelHash, + generatedFiles: input.generatedFiles, + observation: { + source: observations.header.source, + scene: observations.header.scene, + layoutSha256: observations.header.layoutSha256, + samples: observations.samples.length, + durationS: round(last.t - first.t, 3), + tracedElements: traced.length, + controlledElements: motion.length, + }, + tolerances, + }, + summary: { + realizationOffsets: deviations.filter(item => item.kind === 'realization-offset').length, + runtimeDrifts: deviations.filter(item => item.kind === 'runtime-drift').length, + runtimeRuleViolations: ruleFindings.filter(item => item.kind === 'runtime-rule-violation').length, + designViolationsNotObserved: ruleFindings.filter(item => item.kind === 'design-violation-not-observed').length, + designRuleIssues: designIssues.size, + }, + deviations, + ruleFindings, + motion, + reconciliation: { + patch, + ruleImpact: { + introduced: [...reconciledIssues.entries()].filter(([key]) => !designIssues.has(key)).map(([, issue]) => issue.message), + resolved: [...designIssues.entries()].filter(([key]) => !reconciledIssues.has(key)).map(([, issue]) => issue.message), + }, + }, + }; + return { report, reconciledModel }; +} + +export const shortHash = (hash: string): string => hash.replace(/^sha256:/, '').slice(0, 12); + +const traceCell = (trace: TraceChain): string => ( + `${trace.elementName} (${trace.className})
\`${trace.prim}\`
${trace.generatedFile || 'unknown file'}` + + ` from template "${trace.template?.name || 'unknown'}"` +); + +export function renderReport(title: string, report: TwinReport): string { + const { provenance, summary } = report; + const lines = [ + `# Twin loop report: ${title}`, + '', + `Observation source **${provenance.observation.source}**: ${provenance.observation.samples} samples over ` + + `${provenance.observation.durationS} s, ${provenance.observation.tracedElements} traced elements ` + + `(${provenance.observation.controlledElements} driven by the simulator). Model \`${shortHash(provenance.modelHash)}\`, ` + + `tolerance ${provenance.tolerances.positionMm} mm / ${provenance.tolerances.angleDeg} deg.`, + '', + '| Finding | Count |', + '| --- | ---: |', + `| Realization offsets (authored pose never reached) | ${summary.realizationOffsets} |`, + `| Runtime drifts (undriven element moved) | ${summary.runtimeDrifts} |`, + `| Rule violations at runtime only | ${summary.runtimeRuleViolations} |`, + `| Design-time rule issues not observed at runtime | ${summary.designViolationsNotObserved} |`, + `| Design-time rule issues | ${summary.designRuleIssues} |`, + '', + ]; + if (report.deviations.length > 0) { + lines.push('## Deviations', '', '| Kind | Onset (s) | Delta x, y, z (mm) / yaw (deg) | Traced to |', '| --- | ---: | --- | --- |'); + report.deviations.forEach(item => lines.push( + `| ${item.kind} | ${item.onsetT} | ${item.deltaMm.x}, ${item.deltaMm.y}, ${item.deltaMm.z} / ${item.deltaYawDeg} | ${traceCell(item.trace)} |` + )); + lines.push(''); + } + if (report.ruleFindings.length > 0) { + lines.push('## Rule verdicts that differ between model and run', '', '| Kind | Rule | First to last (s) | Message | Elements |', '| --- | --- | --- | --- | --- |'); + report.ruleFindings.forEach(item => lines.push( + `| ${item.kind} | ${item.ruleName} (${item.severity}) | ${item.firstT === undefined ? 'n/a' : `${item.firstT} to ${item.lastT}`} | ${item.message} | ${item.elements.map(trace => trace.elementName).join(', ')} |` + )); + lines.push(''); + } + if (report.motion.length > 0) { + lines.push('## Driven elements', '', '| Element | Path length (mm) |', '| --- | ---: |'); + report.motion.forEach(item => lines.push(`| ${item.trace.elementName} | ${item.pathLengthMm} |`)); + lines.push(''); + } + lines.push('## Reconciliation proposal', ''); + if (report.reconciliation.patch.length === 0) { + lines.push('The run agrees with the model; nothing to reconcile.', ''); + } else { + lines.push('Accepting the observed state would change the model as follows (`reconciled-model.json`):', '', + '| Element | Attribute | From | To |', '| --- | --- | --- | --- |'); + report.reconciliation.patch.forEach(entry => lines.push( + `| ${entry.elementName} | ${entry.path} | \`${JSON.stringify(entry.from)}\` | \`${JSON.stringify(entry.to)}\` |` + )); + lines.push(''); + const { introduced, resolved } = report.reconciliation.ruleImpact; + if (introduced.length > 0) lines.push('Rule issues the accepted model would gain:', '', ...introduced.map(text => `- ${text}`), ''); + if (resolved.length > 0) lines.push('Rule issues the accepted model would lose:', '', ...resolved.map(text => `- ${text}`), ''); + if (introduced.length === 0 && resolved.length === 0) lines.push('The accepted model has the same rule verdicts as the design model.', ''); + } + return lines.join('\n'); +} diff --git a/benchmarks/package.json b/benchmarks/package.json index 30ef112..196aacd 100644 --- a/benchmarks/package.json +++ b/benchmarks/package.json @@ -1,10 +1,11 @@ { "name": "spatialdsl-benchmarks", "private": true, - "description": "Unattended evaluation scripts for SpatialDSL Studio (scalability and generation fidelity).", + "description": "Unattended evaluation scripts for SpatialDSL Studio (scalability, generation fidelity, and the twin loop).", "scripts": { "scalability": "../backend/node_modules/.bin/ts-node --transpile-only --project ../backend/tsconfig.json scalability.ts", "fidelity:generate": "../backend/node_modules/.bin/ts-node --transpile-only --project ../backend/tsconfig.json fidelity-generate.ts", - "fidelity": ".venv/bin/python fidelity.py" + "fidelity": ".venv/bin/python fidelity.py", + "twin-loop": "../backend/node_modules/.bin/ts-node --transpile-only --project ../backend/tsconfig.json twin-loop.ts" } } diff --git a/benchmarks/twin-loop.ts b/benchmarks/twin-loop.ts new file mode 100644 index 0000000..3789696 --- /dev/null +++ b/benchmarks/twin-loop.ts @@ -0,0 +1,200 @@ +/** + * Twin loop on the Smart Warehouse example: generate the scene, run it, record + * what happens, lift the observations back onto the model, and re-evaluate the + * design-time spatial rules on what was observed. + * + * npm run twin-loop # three stand-in scenarios, self-checking + * npm run twin-loop -- --observations # analyse a log recorded by the Isaac Sim bridge + * + * The stand-in scenarios need no simulator. A log recorded in Isaac Sim + * (run_warehouse_sim.py --scene ... --observe-out ...) must come from the + * bundle this script generates in results/twin-loop/bundle; the layout hash in + * the log is checked against it. + */ +import './lib/offline-env'; +import { execFileSync } from 'child_process'; +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; +import { CodeGenerationTemplate, Metamodel, Model } from '../shared/types'; +import { codeGenerationEngine } from '../backend/src/services/pipeline/code-generation.engine'; +import { contentHash } from '../backend/src/services/lifecycle/canonical'; +import { + GeneratedFileTrace, ObservationLog, TwinReport, analyzeRun, readObservationLog, renderReport, shortHash, +} from './lib/twin'; + +const repoRoot = path.resolve(__dirname, '..'); +const resultsDir = path.join(__dirname, 'results'); +const outDir = path.join(resultsDir, 'twin-loop'); +const bundleDir = path.join(outDir, 'bundle'); +const simDir = path.join(repoRoot, 'examples', 'omniverse-assets', 'warehouse_sim'); +const python = path.join(__dirname, '.venv', 'bin', 'python'); +const BUNDLE_FILES = [ + 'warehouse_layout.usda', 'warehouse_assets.usda', 'warehouse_physics.usda', 'warehouse_simulation.usda', + 'warehouse_layout.json', +]; +const RACK_AISLE_RULE = '10000000-0000-4000-8000-000000000302'; + +const readJson = (relativePath: string): any => JSON.parse(fs.readFileSync(path.join(repoRoot, relativePath), 'utf8')); +const metamodel = readJson('frontend/src/examples/data/smart-warehouse-metamodel.json') as Metamodel; +const model = readJson('frontend/src/examples/data/smart-warehouse-model.json') as Model; +const templates = readJson('examples/codegen-projects/smart-warehouse-omniverse-project.json').templates as CodeGenerationTemplate[]; + +interface Scenario { + name: string; + description: string; + disturbances: string[]; + /** Returns the reasons the report differs from what the scenario injected. */ + check: (report: TwinReport) => string[]; +} + +const scenarios: Scenario[] = [ + { + name: 'clean', + description: 'Nothing disturbs the scene; only the robots move.', + disturbances: [], + check: report => [ + ...(report.deviations.length === 0 ? [] : [`expected no deviations, found ${report.deviations.length}`]), + ...(report.summary.runtimeRuleViolations === 0 ? [] : ['expected no runtime-only rule violations']), + ...(report.reconciliation.patch.length === 0 ? [] : ['expected an empty reconciliation patch']), + ...(report.motion.length > 0 && report.motion.every(item => item.pathLengthMm > 0) ? [] : ['expected every robot to move']), + ], + }, + { + name: 'runtime-drift', + description: 'Storage Rack #2 is pushed 1.2 m towards Storage Rack at t = 20 s, closing the aisle.', + disturbances: ['Storage Rack #2:-1.2,0@20'], + check: report => { + const failures: string[] = []; + const [deviation] = report.deviations; + if (report.deviations.length !== 1 || deviation.kind !== 'runtime-drift' + || deviation.trace.elementName !== 'Storage Rack #2' || deviation.onsetT !== 20 || deviation.deltaMm.x !== -1200) { + failures.push('expected one runtime drift of Storage Rack #2 by -1200 mm in x from t = 20 s'); + } else if (deviation.trace.generatedFile !== 'warehouse_layout.usda' || !deviation.trace.template?.id) { + failures.push('expected the drift to trace to the layout layer and its template'); + } + const aisle = report.ruleFindings.find(item => item.kind === 'runtime-rule-violation' && item.ruleId === RACK_AISLE_RULE); + if (!aisle || aisle.firstT !== 20) failures.push('expected the rack aisle rule to fail at runtime from t = 20 s'); + if (!report.reconciliation.ruleImpact.introduced.some(text => text.startsWith('Storage rack aisle width'))) { + failures.push('expected accepting the observed layout to introduce the rack aisle issue'); + } + return failures; + }, + }, + { + name: 'realization-offset', + description: 'Loading Dock rests 150 mm above its authored pose from the first sample on.', + disturbances: ['Loading Dock:0,0,0.15@0'], + check: report => { + const [deviation] = report.deviations; + return report.deviations.length === 1 && deviation.kind === 'realization-offset' + && deviation.trace.elementName === 'Loading Dock' && deviation.deltaMm.z === 150 && deviation.onsetT === 0 + ? [] + : ['expected one realization offset of Loading Dock by 150 mm in z from t = 0 s']; + }, + }, +]; + +const runPython = (args: string[]): void => { + if (!fs.existsSync(python)) throw new Error(`Missing ${python}; see benchmarks/README.md for the venv setup`); + execFileSync(python, args, { cwd: simDir, stdio: ['ignore', 'ignore', 'inherit'] }); +}; + +// Generate the bundle once; every scenario runs against the same generation. +const generated = codeGenerationEngine.generate(model, metamodel, templates); +const generatedFiles: GeneratedFileTrace[] = generated.map((file, index) => ({ + filename: file.filename, + contentHash: file.contentHash, + templateId: templates[index].id, + templateName: templates[index].name, +})); +fs.rmSync(outDir, { recursive: true, force: true }); +fs.mkdirSync(bundleDir, { recursive: true }); +BUNDLE_FILES.forEach(name => { + fs.writeFileSync(path.join(bundleDir, name), generated.find(file => file.filename === name)!.content); +}); +// Layers reference assets as @omniverse-assets/...@ relative to the bundle root. +fs.symlinkSync(path.join(repoRoot, 'examples', 'omniverse-assets'), path.join(bundleDir, 'omniverse-assets'), 'dir'); +const layoutPath = path.join(bundleDir, 'warehouse_layout.json'); +const layoutSha256 = crypto.createHash('sha256').update(fs.readFileSync(layoutPath)).digest('hex'); +const authoredPath = path.join(bundleDir, 'authored.jsonl'); +runPython(['observe.py', 'snapshot', '--scene', path.join(bundleDir, 'warehouse_simulation.usda'), '--out', authoredPath]); +const authored = readObservationLog(authoredPath); + +const analyse = (name: string, title: string, observations: ObservationLog): TwinReport => { + if (observations.header.layoutSha256 !== layoutSha256) { + throw new Error(`${name}: the log was recorded against a different generated layout ` + + `(${observations.header.layoutSha256}) than this model generates (${layoutSha256})`); + } + const { report, reconciledModel } = analyzeRun({ + model, metamodel, modelHash: contentHash(model), metamodelHash: contentHash(metamodel), + generatedFiles, authored, observations, + }); + const directory = path.join(outDir, name); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(path.join(directory, 'report.json'), `${JSON.stringify(report, null, 2)}\n`); + fs.writeFileSync(path.join(directory, 'report.md'), renderReport(title, report)); + if (report.reconciliation.patch.length > 0) { + fs.writeFileSync(path.join(directory, 'reconciled-model.json'), `${JSON.stringify(reconciledModel, null, 2)}\n`); + } + return report; +}; + +const externalIndex = process.argv.indexOf('--observations'); +if (externalIndex >= 0) { + const file = path.resolve(process.argv[externalIndex + 1] || ''); + const observations = readObservationLog(file); + const report = analyse('external', path.basename(file), observations); + process.stdout.write(`${renderReport(path.basename(file), report)}\nReport written to ${path.join(outDir, 'external')}\n`); + process.exit(0); +} + +const started = Date.now(); +const rows = scenarios.map(scenario => { + const directory = path.join(outDir, scenario.name); + fs.mkdirSync(directory, { recursive: true }); + const observationsPath = path.join(directory, 'observations.jsonl'); + runPython([ + 'run_standin_sim.py', '--layout', layoutPath, '--authored', authoredPath, '--observe-out', observationsPath, + ...scenario.disturbances.flatMap(disturbance => ['--disturb', disturbance]), + ]); + const report = analyse(scenario.name, scenario.name, readObservationLog(observationsPath)); + return { scenario, report, failures: scenario.check(report) }; +}); + +const summary = { + scenarios: rows.map(row => ({ + name: row.scenario.name, + description: row.scenario.description, + disturbances: row.scenario.disturbances, + summary: row.report.summary, + reconciliationPatchEntries: row.report.reconciliation.patch.length, + passed: row.failures.length === 0, + failures: row.failures, + })), + tracedElements: rows[0].report.provenance.observation.tracedElements, + samplesPerRun: rows[0].report.provenance.observation.samples, + modelHash: rows[0].report.provenance.modelHash, + layoutSha256, + seconds: Math.round((Date.now() - started) / 100) / 10, +}; +fs.writeFileSync(path.join(resultsDir, 'twin-loop.json'), `${JSON.stringify(summary, null, 2)}\n`); +const lines = [ + '# Twin loop results', + '', + `Smart Warehouse model \`${shortHash(summary.modelHash)}\`, ${summary.tracedElements} traced elements, ` + + `${summary.samplesPerRun} samples per run, observation source: kinematic stand-in (no physics engine).`, + '', + '| Scenario | Injected | Offsets | Drifts | Runtime-only rule violations | Patch entries | Result |', + '| --- | --- | ---: | ---: | ---: | ---: | --- |', + ...summary.scenarios.map(row => ( + `| ${row.name} | ${row.disturbances.join('; ') || 'nothing'} | ${row.summary.realizationOffsets} | ${row.summary.runtimeDrifts} | ` + + `${row.summary.runtimeRuleViolations} | ${row.reconciliationPatchEntries} | ${row.passed ? 'as injected' : `MISMATCH: ${row.failures.join('; ')}`} |` + )), + '', + `Per-scenario reports are in \`results/twin-loop//report.md\`. Runtime ${summary.seconds} s.`, + '', +]; +fs.writeFileSync(path.join(resultsDir, 'twin-loop.md'), lines.join('\n')); +process.stdout.write(`${lines.join('\n')}\n`); +process.exit(summary.scenarios.every(row => row.passed) ? 0 : 1); diff --git a/docs/user-guide/omniverse-code-generation.md b/docs/user-guide/omniverse-code-generation.md index 862ca6c..b7a8329 100644 --- a/docs/user-guide/omniverse-code-generation.md +++ b/docs/user-guide/omniverse-code-generation.md @@ -74,6 +74,10 @@ The generated Python script creates a `.usda` USD stage with: - a placeholder cube only for unmapped classes or unavailable asset files - source metadata attributes such as `spatialDsl:sourceName`, `spatialDsl:className`, and source coordinates in millimeters +- in the layered layout layer and `warehouse_layout.json`, the ID of the model + element behind each prim and record (`spatialDsl:elementId`, `id`), so a + recorded simulation run can be attributed to model elements (see the Twin + loop section of `benchmarks/README.md`) The legacy Python builder remains useful for its explicit missing-asset cube fallback. The layered root adds a real physics scene, proxy colliders, robot diff --git a/examples/codegen-projects/smart-warehouse-omniverse-project.json b/examples/codegen-projects/smart-warehouse-omniverse-project.json index 4660d1e..f127172 100644 --- a/examples/codegen-projects/smart-warehouse-omniverse-project.json +++ b/examples/codegen-projects/smart-warehouse-omniverse-project.json @@ -15,14 +15,14 @@ "id": "a2c1e7d0-0000-4000-8000-0000000000f1", "name": "Generate Warehouse Layout", "language": "json", - "templateContent": "{\n \"units\": \"mm\",\n \"robots\": [{{#each elementsByClassName.MobileRobot}}{{commaIfNeeded \"wh_robots\"}}\n {\"name\": \"{{name}}\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"batteryLevel\": {{#if BatteryLevel}}{{BatteryLevel}}{{else}}100{{/if}}, \"lowBattery\": {{#if LowBatteryThreshold}}{{LowBatteryThreshold}}{{else}}20{{/if}}, \"chargeRate\": {{#if ChargeRate}}{{ChargeRate}}{{else}}10{{/if}} }{{/each}}\n ],\n \"obstacles\": [{{#each elementsByClassName.StorageRack}}{{commaIfNeeded \"wh_obs\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/storage_rack.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}{{#each elementsByClassName.Dock}}{{commaIfNeeded \"wh_obs\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/dock.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"pickups\": [{{#each elementsByClassName.Conveyor}}{{commaIfNeeded \"wh_pick\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/conveyor.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"dropoffs\": [{{#each elementsByClassName.OutputLocation}}{{commaIfNeeded \"wh_drop\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/output_location.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"chargers\": [{{#each elementsByClassName.ChargingStation}}{{commaIfNeeded \"wh_charge\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/charging_station.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"drones\": [{{#each elementsByClassName.InspectionDrone}}{{commaIfNeeded \"wh_drones\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/inspection_drone.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"yaw\": {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}}, \"kinematic\": true }{{/each}}\n ]\n}\n", + "templateContent": "{\n \"units\": \"mm\",\n \"robots\": [{{#each elementsByClassName.MobileRobot}}{{commaIfNeeded \"wh_robots\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"batteryLevel\": {{#if BatteryLevel}}{{BatteryLevel}}{{else}}100{{/if}}, \"lowBattery\": {{#if LowBatteryThreshold}}{{LowBatteryThreshold}}{{else}}20{{/if}}, \"chargeRate\": {{#if ChargeRate}}{{ChargeRate}}{{else}}10{{/if}} }{{/each}}\n ],\n \"obstacles\": [{{#each elementsByClassName.StorageRack}}{{commaIfNeeded \"wh_obs\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/storage_rack.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}{{#each elementsByClassName.Dock}}{{commaIfNeeded \"wh_obs\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/dock.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"pickups\": [{{#each elementsByClassName.Conveyor}}{{commaIfNeeded \"wh_pick\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/conveyor.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"dropoffs\": [{{#each elementsByClassName.OutputLocation}}{{commaIfNeeded \"wh_drop\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/output_location.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"chargers\": [{{#each elementsByClassName.ChargingStation}}{{commaIfNeeded \"wh_charge\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/charging_station.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"drones\": [{{#each elementsByClassName.InspectionDrone}}{{commaIfNeeded \"wh_drones\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/inspection_drone.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"yaw\": {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}}, \"kinematic\": true }{{/each}}\n ]\n}\n", "outputPattern": "warehouse_layout.json" }, { "id": "a2c1e7d0-0000-4000-8000-0000000000f2", "name": "Generate Layered USD Layout", "language": "plaintext", - "templateContent": "#usda 1.0\n(\n defaultPrim = \"World\"\n documentation = \"SpatialDSL model layout only; asset and physics opinions live in sibling layers.\"\n metersPerUnit = 1\n timeCodesPerSecond = 60\n upAxis = \"Z\"\n)\n\ndef Xform \"World\"\n{\n custom string spatialDsl:layerRole = \"layout\"\n\n def Cube \"Floor\"\n {\n double size = 1\n color3f[] primvars:displayColor = [(0.18, 0.20, 0.23)]\n double3 xformOp:scale = (80, 60, 0.1)\n double3 xformOp:translate = (5, 0, -0.05)\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:scale\"]\n }\n\n def Xform \"MobileRobot\"\n {\n{{#each elementsByClassName.MobileRobot}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"MobileRobot\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{meters Z}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\"]\n }\n{{/each}}\n }\n\n def Xform \"Conveyor\"\n {\n{{#each elementsByClassName.Conveyor}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"Conveyor\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"OutputLocation\"\n {\n{{#each elementsByClassName.OutputLocation}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"OutputLocation\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"ChargingStation\"\n {\n{{#each elementsByClassName.ChargingStation}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"ChargingStation\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"PathwayArea\"\n {\n{{#each elementsByClassName.PathwayArea}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"PathwayArea\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"StorageRack\"\n {\n{{#each elementsByClassName.StorageRack}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"StorageRack\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"Dock\"\n {\n{{#each elementsByClassName.Dock}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"Dock\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"InspectionDrone\"\n {\n{{#each elementsByClassName.InspectionDrone}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"InspectionDrone\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{meters Z}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\"]\n }\n{{/each}}\n }\n}\n", + "templateContent": "#usda 1.0\n(\n defaultPrim = \"World\"\n documentation = \"SpatialDSL model layout only; asset and physics opinions live in sibling layers.\"\n metersPerUnit = 1\n timeCodesPerSecond = 60\n upAxis = \"Z\"\n)\n\ndef Xform \"World\"\n{\n custom string spatialDsl:layerRole = \"layout\"\n\n def Cube \"Floor\"\n {\n double size = 1\n color3f[] primvars:displayColor = [(0.18, 0.20, 0.23)]\n double3 xformOp:scale = (80, 60, 0.1)\n double3 xformOp:translate = (5, 0, -0.05)\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:scale\"]\n }\n\n def Xform \"MobileRobot\"\n {\n{{#each elementsByClassName.MobileRobot}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"MobileRobot\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{meters Z}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\"]\n }\n{{/each}}\n }\n\n def Xform \"Conveyor\"\n {\n{{#each elementsByClassName.Conveyor}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"Conveyor\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"OutputLocation\"\n {\n{{#each elementsByClassName.OutputLocation}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"OutputLocation\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"ChargingStation\"\n {\n{{#each elementsByClassName.ChargingStation}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"ChargingStation\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"PathwayArea\"\n {\n{{#each elementsByClassName.PathwayArea}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"PathwayArea\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"StorageRack\"\n {\n{{#each elementsByClassName.StorageRack}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"StorageRack\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"Dock\"\n {\n{{#each elementsByClassName.Dock}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"Dock\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"InspectionDrone\"\n {\n{{#each elementsByClassName.InspectionDrone}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"InspectionDrone\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{meters Z}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\"]\n }\n{{/each}}\n }\n}\n", "outputPattern": "warehouse_layout.usda" }, { diff --git a/examples/omniverse-assets/warehouse_sim/README.md b/examples/omniverse-assets/warehouse_sim/README.md index 2b6cd25..d2c8b84 100644 --- a/examples/omniverse-assets/warehouse_sim/README.md +++ b/examples/omniverse-assets/warehouse_sim/README.md @@ -99,12 +99,32 @@ Python can install it with `./python.sh -m pip install asyncua`. The robots plan paths around the racks and dock and yield to each other at crossings; the brain reassigns pickup/drop-off goals as they arrive. +## Recording a run + +Every prim of a generated layered scene carries the ID of the model element it +was generated from (`spatialDsl:elementId`), and so does every record of a +generated `warehouse_layout.json` (`id`). Add `--observe-out observations.jsonl` +(and optionally `--observe-hz`, default 2) to the simulator command to record +the world pose of every stamped prim while the scene runs. It needs `--scene`, +because only generated prims carry the stamp. + +The log is JSON Lines: a header naming the source, the hash of the layout the +run used, and the elements the simulator drives, then one record per element +per sample (`t`, `elementId`, `prim`, `x`, `y`, `z` in metres, `yaw` in +degrees). `observe.py` documents the format. + +`run_standin_sim.py` writes the same log without Isaac Sim, a GPU, or OPC UA: +the brain drives the robots kinematically, everything else stays at its +authored pose, and `--disturb "NAME:DX,DY[,DZ[,DYAW]]@T"` displaces an element +at a chosen time. `benchmarks/README.md` (Twin loop) describes how a log is +lifted back onto the model and checked against its spatial rules. + ## Tests Pure-Python, no Isaac Sim or OPC UA needed: ```bash -python3 -m unittest test_nav test_brain test_runtime_modes test_layout +python3 -m unittest test_nav test_brain test_runtime_modes test_layout test_observe ``` - `test_nav.py`: A* routes around obstacles / returns None when walled off; @@ -115,6 +135,9 @@ python3 -m unittest test_nav test_brain test_runtime_modes test_layout joint geometry. - `test_layout.py`: millimetre-to-metre XYZ scaling, exact 4.5 m/0 m drone base elevations, and separation from the ground obstacle set. +- `test_observe.py`: the observation log format, the stand-in (only robots + move; a disturbance displaces its element from its time on), and reading the + stamp, pose, and authoring layer from a USD stage (skipped without pxr). ## Swapping the transport diff --git a/examples/omniverse-assets/warehouse_sim/observe.py b/examples/omniverse-assets/warehouse_sim/observe.py new file mode 100644 index 0000000..e3fbc94 --- /dev/null +++ b/examples/omniverse-assets/warehouse_sim/observe.py @@ -0,0 +1,159 @@ +"""Record what a running scene does, keyed by the model element behind each prim. + +The code generator stamps every generated prim with `spatialDsl:elementId`, the +ID of the model element it was generated from. This module reads that stamp +back, so a pose observed in the simulator can be attributed to a model element +without matching names or paths. + +The observation log is JSON Lines and simulator-neutral: the Isaac Sim bridge +(run_warehouse_sim.py --observe-out) and the stand-in (run_standin_sim.py) +write the same records. + + line 1 {"schema": "spatialdsl.observations/1", "source": ..., "units": ..., + "layoutSha256": ..., "scene": ..., "controlledElementIds": [...]} + line 2+ {"t": seconds, "elementId": ..., "prim": ..., "layer": ..., + "x": m, "y": m, "z": m, "yaw": deg} + +`layoutSha256` binds the log to the generated bundle it was recorded against. +`controlledElementIds` lists the elements the simulator drives (the robots); +every other element is expected to stay where the model put it. + +Authored poses of a generated scene, with no simulator (needs pxr / usd-core): + python observe.py snapshot --scene warehouse_simulation.usda --out authored.jsonl +""" + +import argparse +import hashlib +import json +import math +import os + +SCHEMA = "spatialdsl.observations/1" +ELEMENT_ID_ATTRIBUTE = "spatialDsl:elementId" + + +def file_sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + digest.update(chunk) + return digest.hexdigest() + + +class ObservationRecorder: + """Append-only writer for one observation log.""" + + def __init__(self, path, *, source, layout_sha256=None, scene=None, controlled=()): + self.path = path + self.samples = 0 + self._handle = open(path, "w", encoding="utf-8") + self._write({ + "schema": SCHEMA, + "source": source, + "units": {"length": "m", "angle": "deg"}, + "frame": "world, right-handed, Z up", + "layoutSha256": layout_sha256, + "scene": scene, + "controlledElementIds": sorted(controlled), + }) + + def _write(self, record): + self._handle.write(json.dumps(record, sort_keys=True) + "\n") + + def sample(self, t, poses): + """poses: records with elementId, prim, x, y, z, yaw (and optionally layer).""" + for pose in poses: + self._write({"t": round(float(t), 6), **pose}) + self.samples += 1 + + def close(self): + self._handle.close() + + +def traced_prims(stage): + """Every prim that carries a generated element ID, in stage traversal order.""" + traced = [] + for prim in stage.Traverse(): + attribute = prim.GetAttribute(ELEMENT_ID_ATTRIBUTE) + if not attribute or not attribute.HasAuthoredValue(): + continue + # The layer that authors the stamp is the generated file the prim came from. + stack = attribute.GetPropertyStack() + traced.append({ + "elementId": attribute.Get(), + "prim": str(prim.GetPath()), + "layer": os.path.basename(stack[0].layer.identifier) if stack else None, + "handle": prim, + }) + return traced + + +def world_pose(prim): + """World position in stage units and yaw in degrees of a prim, read from the stage.""" + from pxr import Usd, UsdGeom + + matrix = UsdGeom.Xformable(prim).ComputeLocalToWorldTransform(Usd.TimeCode.Default()) + position = matrix.ExtractTranslation() + # Row 0 is the image of the local X axis, so this yaw is unaffected by scale. + yaw = math.degrees(math.atan2(matrix[0][1], matrix[0][0])) + return float(position[0]), float(position[1]), float(position[2]), yaw + + +def sample_stage(traced): + """One pose record per traced prim, as currently held by the stage.""" + poses = [] + for item in traced: + x, y, z, yaw = world_pose(item["handle"]) + poses.append({ + "elementId": item["elementId"], "prim": item["prim"], "layer": item["layer"], + "x": x, "y": y, "z": z, "yaw": yaw, + }) + return poses + + +def read_log(path): + """Return (header, samples); samples is a list of (t, {elementId: record}) in time order.""" + with open(path, "r", encoding="utf-8") as handle: + lines = [json.loads(line) for line in handle if line.strip()] + if not lines or lines[0].get("schema") != SCHEMA: + raise ValueError("%s is not a %s log" % (path, SCHEMA)) + samples = [] + for record in lines[1:]: + if not samples or samples[-1][0] != record["t"]: + samples.append((record["t"], {})) + samples[-1][1][record["elementId"]] = record + return lines[0], samples + + +def snapshot(scene_path, out_path): + """Write the authored (pre-simulation) pose of every traced prim as one sample.""" + from pxr import Usd + + stage = Usd.Stage.Open(scene_path) + if stage is None: + raise SystemExit("Could not open USD stage: " + scene_path) + traced = traced_prims(stage) + if not traced: + raise SystemExit( + "No prim in %s carries %s; regenerate the bundle with the current templates." + % (scene_path, ELEMENT_ID_ATTRIBUTE) + ) + recorder = ObservationRecorder(out_path, source="usd-authored", scene=os.path.basename(scene_path)) + recorder.sample(0.0, sample_stage(traced)) + recorder.close() + return len(traced) + + +def main(): + parser = argparse.ArgumentParser(description="Observation log utilities.") + commands = parser.add_subparsers(dest="command", required=True) + snap = commands.add_parser("snapshot", help="write the authored poses of a generated scene") + snap.add_argument("--scene", required=True) + snap.add_argument("--out", required=True) + args = parser.parse_args() + count = snapshot(os.path.abspath(args.scene), os.path.abspath(args.out)) + print("[observe] %d traced prims written to %s" % (count, args.out)) + + +if __name__ == "__main__": + main() diff --git a/examples/omniverse-assets/warehouse_sim/run_standin_sim.py b/examples/omniverse-assets/warehouse_sim/run_standin_sim.py new file mode 100644 index 0000000..0a17468 --- /dev/null +++ b/examples/omniverse-assets/warehouse_sim/run_standin_sim.py @@ -0,0 +1,128 @@ +"""Simulator stand-in for the twin loop: no Isaac Sim, GPU, or OPC UA. + +The MAS brain drives the robots kinematically, exactly as the Isaac Sim bridge +does in its default control mode, and the run is recorded in the same +observation log (see observe.py). Everything the brain does not drive stays at +its authored pose unless a disturbance moves it, which is what a physics engine +does when something pushes an object or an asset does not rest where it was +placed. + + python run_standin_sim.py --layout warehouse_layout.json \ + --authored authored.jsonl --observe-out observations.jsonl \ + --disturb "Storage Rack #2:-1.2,0@20" + +`--authored` is the output of `observe.py snapshot` for the generated scene. +`--disturb NAME:DX,DY[,DZ[,DYAW]]@T` displaces the named element by metres and +degrees at simulation time T (repeatable). T = 0 means the element never sits +at its authored pose. +""" + +import argparse +import math +import os + +from layout import load_layout +from observe import ObservationRecorder, file_sha256, read_log +from warehouse_brain import WarehouseMAS + +CONTROL_DT = 0.1 + + +def parse_disturbance(text): + try: + target, at = text.rsplit("@", 1) + name, deltas = target.rsplit(":", 1) + values = [float(value) for value in deltas.split(",")] + if not 2 <= len(values) <= 4: + raise ValueError + values += [0.0] * (4 - len(values)) + return {"name": name, "dx": values[0], "dy": values[1], "dz": values[2], "dyaw": values[3], "t": float(at)} + except ValueError: + raise argparse.ArgumentTypeError("expected NAME:DX,DY[,DZ[,DYAW]]@T, got %r" % text) + + +def parse_args(): + here = os.path.dirname(os.path.abspath(__file__)) + parser = argparse.ArgumentParser(description="Kinematic stand-in for the Isaac Sim bridge.") + parser.add_argument("--layout", default=os.path.join(here, "warehouse_layout.json")) + parser.add_argument("--authored", required=True, help="authored poses from `observe.py snapshot`") + parser.add_argument("--observe-out", required=True) + parser.add_argument("--observe-hz", type=float, default=2.0) + parser.add_argument("--seconds", type=float, default=60.0) + parser.add_argument("--disturb", type=parse_disturbance, action="append", default=[]) + return parser.parse_args() + + +def run(layout_path, authored_path, out_path, *, seconds=60.0, observe_hz=2.0, disturbances=()): + layout = load_layout(layout_path) + _, authored_samples = read_log(authored_path) + poses = {element_id: dict(record) for element_id, record in authored_samples[0][1].items()} + for pose in poses.values(): + pose.pop("t", None) + + names = {} + for group in ("robots", "obstacles", "pickups", "dropoffs", "chargers", "drones"): + for record in layout[group]: + if "id" not in record: + raise SystemExit("Layout records carry no element id; regenerate warehouse_layout.json.") + names[record["name"]] = record["id"] + missing = sorted(name for name, element_id in names.items() if element_id not in poses) + if missing: + raise SystemExit("Authored poses are missing layout elements: " + ", ".join(missing)) + for disturbance in disturbances: + if disturbance["name"] not in names: + raise SystemExit("Cannot disturb unknown element " + disturbance["name"]) + + mas = WarehouseMAS(layout, control_dt=CONTROL_DT) + robot_ids = {robot.name: names[robot.name] for robot in mas.robots} + reported = { + robot.name: (poses[robot_ids[robot.name]]["x"], poses[robot_ids[robot.name]]["y"], + poses[robot_ids[robot.name]]["yaw"]) + for robot in mas.robots + } + + recorder = ObservationRecorder( + out_path, source="standin", layout_sha256=file_sha256(layout_path), + scene=os.path.basename(authored_path), controlled=robot_ids.values(), + ) + pending = sorted(disturbances, key=lambda item: item["t"]) + steps = int(round(seconds / CONTROL_DT)) + sample_every = max(1, int(round(1.0 / (observe_hz * CONTROL_DT)))) + for step in range(steps + 1): + t = step * CONTROL_DT + while pending and pending[0]["t"] <= t + 1e-9: + disturbance = pending.pop(0) + pose = poses[names[disturbance["name"]]] + pose["x"] += disturbance["dx"] + pose["y"] += disturbance["dy"] + pose["z"] += disturbance["dz"] + pose["yaw"] += disturbance["dyaw"] + for name, (x, y, heading) in reported.items(): + poses[robot_ids[name]].update({"x": x, "y": y, "yaw": heading}) + if step % sample_every == 0: + recorder.sample(t, [poses[element_id] for element_id in sorted(poses)]) + outputs = mas.update(reported) + for robot in mas.robots: + tx, ty, speed = outputs[robot.name]["target"] + x, y, heading = reported[robot.name] + dx, dy = tx - x, ty - y + distance = math.hypot(dx, dy) + if distance > 1e-6 and speed > 1e-6: + move = min(distance, min(speed, robot.max_speed) * CONTROL_DT) + x += dx / distance * move + y += dy / distance * move + heading = math.degrees(math.atan2(dy, dx)) + reported[robot.name] = (x, y, heading) + recorder.close() + return recorder.samples + + +def main(): + args = parse_args() + samples = run(args.layout, args.authored, args.observe_out, seconds=args.seconds, + observe_hz=args.observe_hz, disturbances=args.disturb) + print("[standin] %d samples over %.1f s written to %s" % (samples, args.seconds, args.observe_out)) + + +if __name__ == "__main__": + main() diff --git a/examples/omniverse-assets/warehouse_sim/run_warehouse_sim.py b/examples/omniverse-assets/warehouse_sim/run_warehouse_sim.py index c834b9f..e771c08 100644 --- a/examples/omniverse-assets/warehouse_sim/run_warehouse_sim.py +++ b/examples/omniverse-assets/warehouse_sim/run_warehouse_sim.py @@ -22,6 +22,7 @@ import os import time +import observe from layout import load_layout, scene_bounds ROBOT_ASSET = "nvidia-f1tenth-amr/f1tenth_amr_collision.usda" @@ -60,6 +61,11 @@ def parse_args(): "--save-runtime-layer", help="export the non-destructive session-layer physics/control overrides on shutdown", ) + parser.add_argument( + "--observe-out", + help="record the world pose of every generated prim to this observation log (needs --scene)", + ) + parser.add_argument("--observe-hz", type=float, default=2.0) parser.add_argument("--headless", action="store_true") parser.add_argument("--max-seconds", type=float, default=0.0) return parser.parse_args() @@ -125,6 +131,8 @@ def main(): args = parse_args() layout = load_layout(args.layout) asset_root = os.path.abspath(os.path.expanduser(args.asset_root)) + if args.observe_out and not args.scene: + raise SystemExit("--observe-out needs --scene: only generated prims carry model element IDs") from isaacsim import SimulationApp simulation_app = SimulationApp({"headless": args.headless}) @@ -323,7 +331,7 @@ def add_prop(group, record): ) ) robots.append({ - "name": record["name"], "opc": "Robot%d" % (index + 1), + "name": record["name"], "id": record.get("id"), "opc": "Robot%d" % (index + 1), "x": record["x"], "y": record["y"], "heading": 0.0, "prim": prim, "body": body, "t_attr": t_attr, "rz_attr": rz_attr, "left_joint": left_joint, "right_joint": right_joint, @@ -342,6 +350,20 @@ def add_prop(group, record): dt = 1.0 / SIM_HZ sim_time = 0.0 + recorder = None + if args.observe_out: + traced = observe.traced_prims(stage) + if not traced or not all(r["id"] for r in robots): + raise SystemExit("Scene or layout carries no model element IDs; regenerate the bundle") + recorder = observe.ObservationRecorder( + os.path.abspath(os.path.expanduser(args.observe_out)), + source="isaac-sim", + layout_sha256=observe.file_sha256(args.layout), + scene=os.path.basename(args.scene), + controlled=[r["id"] for r in robots if r["id"]], + ) + recorder.sample(sim_time, observe.sample_stage(traced)) + next_observation = 1.0 / args.observe_hz try: while simulation_app.is_running(): for r in robots: @@ -412,9 +434,15 @@ def add_prop(group, record): ua.DataValue(ua.Variant([r["x"], r["y"], r["heading"]], ua.VariantType.Double))) simulation_app.update() sim_time += dt + if recorder and sim_time >= next_observation: + recorder.sample(sim_time, observe.sample_stage(traced)) + next_observation += 1.0 / args.observe_hz if args.max_seconds > 0.0 and sim_time >= args.max_seconds: break finally: + if recorder: + recorder.close() + print("[sim] wrote %d observation samples to %s" % (recorder.samples, recorder.path)) if args.save_runtime_layer: runtime_path = os.path.abspath(os.path.expanduser(args.save_runtime_layer)) stage.GetSessionLayer().Export(runtime_path) diff --git a/examples/omniverse-assets/warehouse_sim/test_observe.py b/examples/omniverse-assets/warehouse_sim/test_observe.py new file mode 100644 index 0000000..9666434 --- /dev/null +++ b/examples/omniverse-assets/warehouse_sim/test_observe.py @@ -0,0 +1,127 @@ +"""Tests for the observation log and the simulator stand-in (no Isaac Sim needed). + +The snapshot test reads a USD stage and is skipped when pxr is not installed. +""" + +import json +import os +import tempfile +import unittest + +import observe +import run_standin_sim + +try: + from pxr import Usd # noqa: F401 + HAVE_PXR = True +except ImportError: + HAVE_PXR = False + +LAYOUT = { + "units": "m", + "robots": [ + {"id": "robot-a", "name": "Robot A", "x": -18.0, "y": -8.0, "z": 0.0}, + {"id": "robot-b", "name": "Robot B", "x": 12.0, "y": -8.0, "z": 0.0}, + ], + "obstacles": [{"id": "rack", "name": "Rack", "x": 0.0, "y": 0.0, "z": 0.0, "length": 3.0, "width": 6.0, "height": 2.5}], + "pickups": [ + {"id": "conv-a", "name": "Conv A", "x": -18.0, "y": -9.0, "z": 0.0, "length": 6.0, "width": 2.0, "height": 3.0}, + {"id": "conv-b", "name": "Conv B", "x": 12.0, "y": -8.0, "z": 0.0, "length": 6.0, "width": 2.0, "height": 3.0}, + ], + "dropoffs": [{"id": "output", "name": "Output", "x": 0.0, "y": 8.0, "z": 0.0, "length": 3.0, "width": 3.0, "height": 0.5}], + "chargers": [], + "drones": [], +} + +STAMPED_SCENE = """#usda 1.0 +( + defaultPrim = "World" + metersPerUnit = 1 + upAxis = "Z" +) + +def Xform "World" +{ + def Xform "Rack_0" + { + custom string spatialDsl:elementId = "rack" + double3 xformOp:translate = (10, 12, 1.25) + float xformOp:rotateZ = 30 + double3 xformOp:scale = (2, 1, 2.5) + uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:rotateZ", "xformOp:scale"] + } + + def Xform "Unstamped" + { + } +} +""" + + +class ObservationTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.layout_path = self.path("warehouse_layout.json") + with open(self.layout_path, "w", encoding="utf-8") as handle: + json.dump(LAYOUT, handle) + self.authored_path = self.path("authored.jsonl") + recorder = observe.ObservationRecorder(self.authored_path, source="usd-authored") + recorder.sample(0.0, [ + {"elementId": record["id"], "prim": "/World/" + record["id"], "layer": "warehouse_layout.usda", + "x": record["x"], "y": record["y"], "z": record["z"] + record.get("height", 0.0) / 2.0, "yaw": 0.0} + for group in ("robots", "obstacles", "pickups", "dropoffs") for record in LAYOUT[group] + ]) + recorder.close() + + def path(self, name): + return os.path.join(self.directory.name, name) + + def run_standin(self, **options): + out = self.path("observations.jsonl") + run_standin_sim.run(self.layout_path, self.authored_path, out, seconds=10.0, **options) + return observe.read_log(out) + + def test_log_starts_with_a_header_and_groups_samples_by_time(self): + header, samples = observe.read_log(self.authored_path) + self.assertEqual(header["schema"], observe.SCHEMA) + self.assertEqual(header["units"], {"length": "m", "angle": "deg"}) + self.assertEqual(len(samples), 1) + self.assertEqual(samples[0][1]["rack"]["z"], 1.25) + + def test_standin_moves_only_the_robots_and_binds_the_log_to_the_layout(self): + header, samples = self.run_standin() + self.assertEqual(header["controlledElementIds"], ["robot-a", "robot-b"]) + self.assertEqual(header["layoutSha256"], observe.file_sha256(self.layout_path)) + self.assertEqual(len(samples), 21) # 2 Hz over 10 s, both ends included + first, last = samples[0][1], samples[-1][1] + self.assertEqual((first["rack"]["x"], first["rack"]["y"]), (last["rack"]["x"], last["rack"]["y"])) + self.assertNotEqual((first["robot-a"]["x"], first["robot-a"]["y"]), + (last["robot-a"]["x"], last["robot-a"]["y"])) + + def test_disturbance_displaces_the_element_from_its_time_on(self): + disturbance = run_standin_sim.parse_disturbance("Rack:-1.2,0.5,0,15@4") + _, samples = self.run_standin(disturbances=[disturbance]) + by_time = dict(samples) + self.assertEqual(by_time[3.5]["rack"]["x"], 0.0) + self.assertAlmostEqual(by_time[4.0]["rack"]["x"], -1.2) + self.assertAlmostEqual(by_time[10.0]["rack"]["y"], 0.5) + self.assertAlmostEqual(by_time[10.0]["rack"]["yaw"], 15.0) + + @unittest.skipUnless(HAVE_PXR, "pxr (usd-core) is not installed") + def test_snapshot_reads_the_stamp_pose_and_authoring_layer(self): + scene = self.path("scene.usda") + with open(scene, "w", encoding="utf-8") as handle: + handle.write(STAMPED_SCENE) + out = self.path("snapshot.jsonl") + self.assertEqual(observe.snapshot(scene, out), 1) + header, samples = observe.read_log(out) + record = samples[0][1]["rack"] + self.assertEqual(header["source"], "usd-authored") + self.assertEqual((record["prim"], record["layer"]), ("/World/Rack_0", "scene.usda")) + self.assertEqual((record["x"], record["y"], record["z"]), (10.0, 12.0, 1.25)) + self.assertAlmostEqual(record["yaw"], 30.0, places=4) + + +if __name__ == "__main__": + unittest.main() diff --git a/frontend/src/examples/data/smart-warehouse-omniverse-project.json b/frontend/src/examples/data/smart-warehouse-omniverse-project.json index 4660d1e..f127172 100644 --- a/frontend/src/examples/data/smart-warehouse-omniverse-project.json +++ b/frontend/src/examples/data/smart-warehouse-omniverse-project.json @@ -15,14 +15,14 @@ "id": "a2c1e7d0-0000-4000-8000-0000000000f1", "name": "Generate Warehouse Layout", "language": "json", - "templateContent": "{\n \"units\": \"mm\",\n \"robots\": [{{#each elementsByClassName.MobileRobot}}{{commaIfNeeded \"wh_robots\"}}\n {\"name\": \"{{name}}\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"batteryLevel\": {{#if BatteryLevel}}{{BatteryLevel}}{{else}}100{{/if}}, \"lowBattery\": {{#if LowBatteryThreshold}}{{LowBatteryThreshold}}{{else}}20{{/if}}, \"chargeRate\": {{#if ChargeRate}}{{ChargeRate}}{{else}}10{{/if}} }{{/each}}\n ],\n \"obstacles\": [{{#each elementsByClassName.StorageRack}}{{commaIfNeeded \"wh_obs\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/storage_rack.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}{{#each elementsByClassName.Dock}}{{commaIfNeeded \"wh_obs\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/dock.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"pickups\": [{{#each elementsByClassName.Conveyor}}{{commaIfNeeded \"wh_pick\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/conveyor.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"dropoffs\": [{{#each elementsByClassName.OutputLocation}}{{commaIfNeeded \"wh_drop\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/output_location.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"chargers\": [{{#each elementsByClassName.ChargingStation}}{{commaIfNeeded \"wh_charge\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/charging_station.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"drones\": [{{#each elementsByClassName.InspectionDrone}}{{commaIfNeeded \"wh_drones\"}}\n {\"name\": \"{{name}}\", \"asset\": \"warehouse-kit/inspection_drone.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"yaw\": {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}}, \"kinematic\": true }{{/each}}\n ]\n}\n", + "templateContent": "{\n \"units\": \"mm\",\n \"robots\": [{{#each elementsByClassName.MobileRobot}}{{commaIfNeeded \"wh_robots\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"batteryLevel\": {{#if BatteryLevel}}{{BatteryLevel}}{{else}}100{{/if}}, \"lowBattery\": {{#if LowBatteryThreshold}}{{LowBatteryThreshold}}{{else}}20{{/if}}, \"chargeRate\": {{#if ChargeRate}}{{ChargeRate}}{{else}}10{{/if}} }{{/each}}\n ],\n \"obstacles\": [{{#each elementsByClassName.StorageRack}}{{commaIfNeeded \"wh_obs\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/storage_rack.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}{{#each elementsByClassName.Dock}}{{commaIfNeeded \"wh_obs\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/dock.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"pickups\": [{{#each elementsByClassName.Conveyor}}{{commaIfNeeded \"wh_pick\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/conveyor.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"dropoffs\": [{{#each elementsByClassName.OutputLocation}}{{commaIfNeeded \"wh_drop\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/output_location.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"chargers\": [{{#each elementsByClassName.ChargingStation}}{{commaIfNeeded \"wh_charge\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/charging_station.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}} }{{/each}}\n ],\n \"drones\": [{{#each elementsByClassName.InspectionDrone}}{{commaIfNeeded \"wh_drones\"}}\n {\"id\": \"{{id}}\", \"name\": \"{{name}}\", \"asset\": \"warehouse-kit/inspection_drone.usda\", \"x\": {{X}}, \"y\": {{Y}}, \"z\": {{Z}}, \"yaw\": {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}, \"length\": {{Length}}, \"width\": {{Width}}, \"height\": {{Height}}, \"kinematic\": true }{{/each}}\n ]\n}\n", "outputPattern": "warehouse_layout.json" }, { "id": "a2c1e7d0-0000-4000-8000-0000000000f2", "name": "Generate Layered USD Layout", "language": "plaintext", - "templateContent": "#usda 1.0\n(\n defaultPrim = \"World\"\n documentation = \"SpatialDSL model layout only; asset and physics opinions live in sibling layers.\"\n metersPerUnit = 1\n timeCodesPerSecond = 60\n upAxis = \"Z\"\n)\n\ndef Xform \"World\"\n{\n custom string spatialDsl:layerRole = \"layout\"\n\n def Cube \"Floor\"\n {\n double size = 1\n color3f[] primvars:displayColor = [(0.18, 0.20, 0.23)]\n double3 xformOp:scale = (80, 60, 0.1)\n double3 xformOp:translate = (5, 0, -0.05)\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:scale\"]\n }\n\n def Xform \"MobileRobot\"\n {\n{{#each elementsByClassName.MobileRobot}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"MobileRobot\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{meters Z}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\"]\n }\n{{/each}}\n }\n\n def Xform \"Conveyor\"\n {\n{{#each elementsByClassName.Conveyor}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"Conveyor\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"OutputLocation\"\n {\n{{#each elementsByClassName.OutputLocation}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"OutputLocation\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"ChargingStation\"\n {\n{{#each elementsByClassName.ChargingStation}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"ChargingStation\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"PathwayArea\"\n {\n{{#each elementsByClassName.PathwayArea}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"PathwayArea\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"StorageRack\"\n {\n{{#each elementsByClassName.StorageRack}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"StorageRack\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"Dock\"\n {\n{{#each elementsByClassName.Dock}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"Dock\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"InspectionDrone\"\n {\n{{#each elementsByClassName.InspectionDrone}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"InspectionDrone\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{meters Z}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\"]\n }\n{{/each}}\n }\n}\n", + "templateContent": "#usda 1.0\n(\n defaultPrim = \"World\"\n documentation = \"SpatialDSL model layout only; asset and physics opinions live in sibling layers.\"\n metersPerUnit = 1\n timeCodesPerSecond = 60\n upAxis = \"Z\"\n)\n\ndef Xform \"World\"\n{\n custom string spatialDsl:layerRole = \"layout\"\n\n def Cube \"Floor\"\n {\n double size = 1\n color3f[] primvars:displayColor = [(0.18, 0.20, 0.23)]\n double3 xformOp:scale = (80, 60, 0.1)\n double3 xformOp:translate = (5, 0, -0.05)\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:scale\"]\n }\n\n def Xform \"MobileRobot\"\n {\n{{#each elementsByClassName.MobileRobot}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"MobileRobot\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{meters Z}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\"]\n }\n{{/each}}\n }\n\n def Xform \"Conveyor\"\n {\n{{#each elementsByClassName.Conveyor}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"Conveyor\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"OutputLocation\"\n {\n{{#each elementsByClassName.OutputLocation}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"OutputLocation\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"ChargingStation\"\n {\n{{#each elementsByClassName.ChargingStation}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"ChargingStation\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"PathwayArea\"\n {\n{{#each elementsByClassName.PathwayArea}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"PathwayArea\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"StorageRack\"\n {\n{{#each elementsByClassName.StorageRack}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"StorageRack\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"Dock\"\n {\n{{#each elementsByClassName.Dock}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"Dock\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{baseCenterMeters Z Height}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n double3 xformOp:scale = ({{meters Length}}, {{meters Width}}, {{meters Height}})\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\", \"xformOp:scale\"]\n }\n{{/each}}\n }\n\n def Xform \"InspectionDrone\"\n {\n{{#each elementsByClassName.InspectionDrone}}\n def Xform \"{{usdIdentifier name}}_{{@index}}\"\n {\n custom string spatialDsl:className = \"InspectionDrone\"\n custom string spatialDsl:sourceName = \"{{name}}\"\n custom string spatialDsl:elementId = \"{{id}}\"\n custom double spatialDsl:baseElevationMm = {{Z}}\n double3 xformOp:translate = ({{meters X}}, {{meters Y}}, {{meters Z}})\n float xformOp:rotateZ = {{#unless RZ}}0{{else}}{{RZ}}{{/unless}}\n uniform token[] xformOpOrder = [\"xformOp:translate\", \"xformOp:rotateZ\"]\n }\n{{/each}}\n }\n}\n", "outputPattern": "warehouse_layout.usda" }, { From fb0cd36cdbbcd8cd6a2aa3a68a9c21a42085ed4a Mon Sep 17 00:00:00 2001 From: Ha Phan Tran Date: Sun, 15 Nov 2026 21:10:39 -0500 Subject: [PATCH 2/2] Tighten twin loop findings and bind logs to the whole generated bundle - Reject observation logs whose samples miss authored elements, instead of reading the gap as no movement. - Report a realization offset only when no sample reaches the authored pose. - Report drifts with the displacement at onset and at the end, so a drift that comes back is not shown as zero. - Compare rule verdicts with the model sample by sample, so a design violation that disappears for part of a run is reported. - Fingerprint the layout JSON, the root scene layer, and its sublayers, instead of the layout JSON alone, and name the files that differ. - Read a log given with --observations before cleaning the results folder. - Apply stand-in disturbances of robots to the pose the brain sees. - Add a transient drift scenario. --- benchmarks/README.md | 33 +++-- benchmarks/lib/twin.ts | 140 ++++++++++++------ benchmarks/twin-loop.ts | 59 ++++++-- .../omniverse-assets/warehouse_sim/README.md | 9 +- .../omniverse-assets/warehouse_sim/observe.py | 54 ++++++- .../warehouse_sim/run_standin_sim.py | 18 ++- .../warehouse_sim/run_warehouse_sim.py | 2 +- .../warehouse_sim/test_observe.py | 46 +++++- 8 files changed, 262 insertions(+), 99 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index e94a4c5..c940376 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -68,9 +68,13 @@ runs, and brings the result back in the model's own terms. `results/twin-loop/bundle/`. Every generated prim and layout record carries the ID of its model element (`spatialDsl:elementId`, `id`). 2. `observe.py snapshot` reads the composed stage with pxr and records the - authored pose of every stamped prim, with the layer that authors it. + authored pose of every stamped prim, with the layer that authors it, and a + fingerprint of the generated bundle (the layout JSON, the root scene layer, + and its sublayers). 3. A simulator runs the bundle and records an observation log: the world pose of every stamped prim over time, plus which elements the simulator drives. + Every sample must cover every element of the snapshot; an incomplete log is + rejected rather than read as "nothing moved". 4. Each sample is lifted onto a copy of the model: the observed model pose is the design pose plus the difference between the observed and the authored scene pose. The design-time spatial rules are then evaluated, unchanged, on @@ -78,12 +82,14 @@ runs, and brings the result back in the model's own terms. The report lists, per run: -- **realization offsets**: undriven elements that never sat at their authored - pose; -- **runtime drifts**: undriven elements that moved during the run, with the - onset time; -- **rule verdicts that differ** between the model and the run, with the time - span in which they held; +- **realization offsets**: undriven elements that were not at their authored + pose in any sample; +- **runtime drifts**: undriven elements that moved away from where they + started, with the onset time and the displacement at onset and at the end + (zero at the end when the element came back); +- **rule verdicts that differ** between the model and the run, compared sample + by sample, with the first and last sample and the number of samples in + which they differ; - a **reconciliation proposal**: the model changes that accepting the observed state would make (`reconciled-model.json`, importable as a model), and the rule issues that model would gain or lose. @@ -91,11 +97,12 @@ The report lists, per run: Each finding names the model element, the prim, the generated file, and the template that produced that file. -Without arguments the script runs three scenarios on the simulator stand-in +Without arguments the script runs four scenarios on the simulator stand-in (`run_standin_sim.py`: the MAS brain drives the robots kinematically, with no physics engine): an undisturbed run, a storage rack pushed into the aisle at -t = 20 s, and a dock resting 150 mm above its authored pose. It exits non-zero -if the findings differ from what was injected. The stand-in checks the loop, +t = 20 s, the same rack pushed back at t = 40 s, and a dock resting 150 mm +above its authored pose. It exits non-zero if the findings differ from what was +injected. The stand-in checks the loop, not the physics: deviations exist only where a scenario injects them. To analyse a run recorded in Isaac Sim, run the bundle from @@ -110,8 +117,10 @@ python3 warehouse_brain.py --layout /warehouse_layout.json # npm run twin-loop -- --observations observations.jsonl ``` -The log records the hash of the layout it ran against; the script refuses a log -recorded against a different generation of the model. +The log records the fingerprint of the generated bundle it ran against; the +script refuses a log recorded against a different generation of the model and +names the generated files that differ. A log may be read from anywhere, +including `results/twin-loop/`, which the script cleans only after reading it. Outputs: `results/twin-loop.json`, `results/twin-loop.md`, and per-scenario `report.json`, `report.md`, `observations.jsonl`, and `reconciled-model.json` diff --git a/benchmarks/lib/twin.ts b/benchmarks/lib/twin.ts index 55c4344..f832354 100644 --- a/benchmarks/lib/twin.ts +++ b/benchmarks/lib/twin.ts @@ -33,7 +33,9 @@ export interface ObservedPose { export interface ObservationHeader { schema: string; source: string; - layoutSha256?: string | null; + /** Identifies the generated bundle the run used: the layout JSON, the root scene layer, and its sublayers. */ + bundleFingerprint?: string | null; + bundleFiles?: Record | null; scene?: string | null; controlledElementIds: string[]; } @@ -73,18 +75,32 @@ export interface PoseMm { } export interface DeviationFinding { - /** realization-offset: never at its authored pose. runtime-drift: moved during the run. */ + /** + * realization-offset: not at its authored pose in any sample; deltas are measured from the authored pose. + * runtime-drift: moved away from where it started; deltas are measured from the first sample. + */ kind: 'realization-offset' | 'runtime-drift'; trace: TraceChain; onsetT: number; + /** Displacement in the onset sample. */ deltaMm: { x: number; y: number; z: number }; deltaYawDeg: number; + /** Displacement in the last sample; zero when the element came back. */ + finalDeltaMm: { x: number; y: number; z: number }; + finalDeltaYawDeg: number; designPose: PoseMm; + /** The model pose implied by the onset sample. */ observedPose: PoseMm; + /** The model pose implied by the last sample. */ + finalPose: PoseMm; } export interface RuleFinding { - /** runtime-rule-violation: holds at runtime only. design-violation-not-observed: holds in the model only. */ + /** + * runtime-rule-violation: absent from the model, present in some samples. + * design-violation-not-observed: present in the model, absent from some samples. + * firstT, lastT, and samples describe the samples in which the verdict differs from the model. + */ kind: 'runtime-rule-violation' | 'design-violation-not-observed'; ruleId: string; ruleName: string; @@ -113,7 +129,7 @@ export interface TwinReport { observation: { source: string; scene?: string | null; - layoutSha256?: string | null; + bundleFingerprint?: string | null; samples: number; durationS: number; tracedElements: number; @@ -264,37 +280,52 @@ export function analyzeRun(input: { }; }; - const traced = model.elements.filter(element => ( - element.presentation?.position3D && authored.has(element.id) && first.poses.has(element.id) - )); + // A missing observation must not read as "unchanged": every sample has to cover every authored element. + const strangers = [...authored.keys()].filter(id => !elements.has(id)); + if (strangers.length > 0) { + throw new Error(`The authored snapshot holds ${strangers.length} element(s) this model does not have, ` + + `for example ${strangers[0]}; it was taken from a different generation of the model`); + } + observations.samples.forEach(sample => { + const missing = [...authored.keys()].filter(id => !sample.poses.has(id)); + if (missing.length > 0) { + throw new Error(`The observation sample at t = ${sample.t} s lacks ${missing.length} of ${authored.size} ` + + `authored elements, for example ${traceOf(missing[0]).elementName}; the log is incomplete`); + } + }); + const traced = model.elements.filter(element => element.presentation?.position3D && authored.has(element.id)); // Elements the simulator does not drive are expected to stay where the model put them. const deviations: DeviationFinding[] = []; const finding = ( - kind: DeviationFinding['kind'], element: ModelElement, onsetT: number, from: ObservedPose, to: ObservedPose + kind: DeviationFinding['kind'], element: ModelElement, onsetT: number, + reference: ObservedPose, atOnset: ObservedPose, atEnd: ObservedPose ): DeviationFinding => { - const delta = residual(from, to); + const delta = residual(reference, atOnset); + const finalDelta = residual(reference, atEnd); return { kind, trace: traceOf(element.id), onsetT, deltaMm: { x: delta.x, y: delta.y, z: delta.z }, deltaYawDeg: delta.yaw, + finalDeltaMm: { x: finalDelta.x, y: finalDelta.y, z: finalDelta.z }, + finalDeltaYawDeg: finalDelta.yaw, designPose: designPose(element), - observedPose: liftPose(element, authored.get(element.id)!, to), + observedPose: liftPose(element, authored.get(element.id)!, atOnset), + finalPose: liftPose(element, authored.get(element.id)!, atEnd), }; }; traced.filter(element => !controlled.has(element.id)).forEach(element => { + const authoredPose = authored.get(element.id)!; const start = first.poses.get(element.id)!; - if (exceeds(residual(authored.get(element.id)!, start), tolerances)) { - deviations.push(finding('realization-offset', element, first.t, authored.get(element.id)!, start)); - } - const onset = observations.samples.find(sample => { - const pose = sample.poses.get(element.id); - return pose !== undefined && exceeds(residual(start, pose), tolerances); - }); - const end = last.poses.get(element.id); - if (onset && end) deviations.push(finding('runtime-drift', element, onset.t, start, end)); + const end = last.poses.get(element.id)!; + const neverAuthored = observations.samples.every(sample => ( + exceeds(residual(authoredPose, sample.poses.get(element.id)!), tolerances) + )); + if (neverAuthored) deviations.push(finding('realization-offset', element, first.t, authoredPose, start, end)); + const onset = observations.samples.find(sample => exceeds(residual(start, sample.poses.get(element.id)!), tolerances)); + if (onset) deviations.push(finding('runtime-drift', element, onset.t, start, onset.poses.get(element.id)!, end)); }); const motion = traced.filter(element => controlled.has(element.id)).map(element => { @@ -309,38 +340,47 @@ export function analyzeRun(input: { return { trace: traceOf(element.id), pathLengthMm: round(pathLengthMm, 1) }; }); - // The design-time rules, evaluated unchanged on the lifted model of every sample. + // The design-time rules, evaluated unchanged on the lifted model of every sample and + // compared with the model's own verdicts sample by sample. + type Span = { issue: ValidationIssue; firstT: number; lastT: number; samples: number }; const designIssues = new Map(evaluateSpatialRules(metamodel, model).map(issue => [issueKey(issue), issue])); - const seen = new Map(); + const runtimeOnly = new Map(); + const designAbsent = new Map(); + const note = (spans: Map, key: string, issue: ValidationIssue, t: number): void => { + const span = spans.get(key); + if (span) { + span.lastT = t; + span.samples += 1; + } else { + spans.set(key, { issue, firstT: t, lastT: t, samples: 1 }); + } + }; observations.samples.forEach(sample => { + const present = new Set(); evaluateSpatialRules(metamodel, liftModel(model, authored, sample.poses)).forEach(issue => { - const entry = seen.get(issueKey(issue)); - if (entry) { - entry.lastT = sample.t; - entry.samples += 1; - } else { - seen.set(issueKey(issue), { issue, firstT: sample.t, lastT: sample.t, samples: 1 }); - } + const key = issueKey(issue); + present.add(key); + if (!designIssues.has(key)) note(runtimeOnly, key, issue, sample.t); + }); + designIssues.forEach((issue, key) => { + if (!present.has(key)) note(designAbsent, key, issue, sample.t); }); }); - const ruleFinding = (kind: RuleFinding['kind'], issue: ValidationIssue, span?: { - firstT: number; lastT: number; samples: number; - }): RuleFinding => ({ + const ruleFinding = (kind: RuleFinding['kind'], span: Span): RuleFinding => ({ kind, - ruleId: issue.constraintId || 'unknown', - ruleName: ruleNames.get(issue.constraintId || '') || issue.constraintId || 'unknown', - severity: issue.severity, - message: issue.message, - ...(span ? { firstT: span.firstT, lastT: span.lastT } : {}), - samples: span?.samples || 0, - elements: [issue.elementId, ...(issue.relatedElementIds || [])] + ruleId: span.issue.constraintId || 'unknown', + ruleName: ruleNames.get(span.issue.constraintId || '') || span.issue.constraintId || 'unknown', + severity: span.issue.severity, + message: span.issue.message, + firstT: span.firstT, + lastT: span.lastT, + samples: span.samples, + elements: [span.issue.elementId, ...(span.issue.relatedElementIds || [])] .filter((id): id is string => Boolean(id)).map(traceOf), }); const ruleFindings: RuleFinding[] = [ - ...[...seen.entries()].filter(([key]) => !designIssues.has(key)) - .map(([, entry]) => ruleFinding('runtime-rule-violation', entry.issue, entry)), - ...[...designIssues.entries()].filter(([key]) => !seen.has(key)) - .map(([, issue]) => ruleFinding('design-violation-not-observed', issue)), + ...[...runtimeOnly.values()].map(span => ruleFinding('runtime-rule-violation', span)), + ...[...designAbsent.values()].map(span => ruleFinding('design-violation-not-observed', span)), ]; // Reconciliation proposal: adopt the final observed pose of every element that @@ -376,7 +416,7 @@ export function analyzeRun(input: { observation: { source: observations.header.source, scene: observations.header.scene, - layoutSha256: observations.header.layoutSha256, + bundleFingerprint: observations.header.bundleFingerprint, samples: observations.samples.length, durationS: round(last.t - first.t, 3), tracedElements: traced.length, @@ -432,16 +472,20 @@ export function renderReport(title: string, report: TwinReport): string { '', ]; if (report.deviations.length > 0) { - lines.push('## Deviations', '', '| Kind | Onset (s) | Delta x, y, z (mm) / yaw (deg) | Traced to |', '| --- | ---: | --- | --- |'); + const delta = (mm: { x: number; y: number; z: number }, yaw: number): string => `${mm.x}, ${mm.y}, ${mm.z} / ${yaw}`; + lines.push('## Deviations', '', + 'Realization offsets are measured from the authored pose, runtime drifts from the first sample.', '', + '| Kind | Onset (s) | At onset: x, y, z (mm) / yaw (deg) | At end | Traced to |', '| --- | ---: | --- | --- | --- |'); report.deviations.forEach(item => lines.push( - `| ${item.kind} | ${item.onsetT} | ${item.deltaMm.x}, ${item.deltaMm.y}, ${item.deltaMm.z} / ${item.deltaYawDeg} | ${traceCell(item.trace)} |` + `| ${item.kind} | ${item.onsetT} | ${delta(item.deltaMm, item.deltaYawDeg)} | ${delta(item.finalDeltaMm, item.finalDeltaYawDeg)} | ${traceCell(item.trace)} |` )); lines.push(''); } if (report.ruleFindings.length > 0) { - lines.push('## Rule verdicts that differ between model and run', '', '| Kind | Rule | First to last (s) | Message | Elements |', '| --- | --- | --- | --- | --- |'); + lines.push('## Rule verdicts that differ between model and run', '', + '| Kind | Rule | Differs from first to last (s) | Samples | Message | Elements |', '| --- | --- | --- | ---: | --- | --- |'); report.ruleFindings.forEach(item => lines.push( - `| ${item.kind} | ${item.ruleName} (${item.severity}) | ${item.firstT === undefined ? 'n/a' : `${item.firstT} to ${item.lastT}`} | ${item.message} | ${item.elements.map(trace => trace.elementName).join(', ')} |` + `| ${item.kind} | ${item.ruleName} (${item.severity}) | ${item.firstT} to ${item.lastT} | ${item.samples} | ${item.message} | ${item.elements.map(trace => trace.elementName).join(', ')} |` )); lines.push(''); } @@ -452,7 +496,7 @@ export function renderReport(title: string, report: TwinReport): string { } lines.push('## Reconciliation proposal', ''); if (report.reconciliation.patch.length === 0) { - lines.push('The run agrees with the model; nothing to reconcile.', ''); + lines.push('The final observed state agrees with the model; nothing to reconcile.', ''); } else { lines.push('Accepting the observed state would change the model as follows (`reconciled-model.json`):', '', '| Element | Attribute | From | To |', '| --- | --- | --- | --- |'); diff --git a/benchmarks/twin-loop.ts b/benchmarks/twin-loop.ts index 3789696..fe3aa72 100644 --- a/benchmarks/twin-loop.ts +++ b/benchmarks/twin-loop.ts @@ -3,17 +3,16 @@ * what happens, lift the observations back onto the model, and re-evaluate the * design-time spatial rules on what was observed. * - * npm run twin-loop # three stand-in scenarios, self-checking + * npm run twin-loop # four stand-in scenarios, self-checking * npm run twin-loop -- --observations # analyse a log recorded by the Isaac Sim bridge * * The stand-in scenarios need no simulator. A log recorded in Isaac Sim * (run_warehouse_sim.py --scene ... --observe-out ...) must come from the - * bundle this script generates in results/twin-loop/bundle; the layout hash in - * the log is checked against it. + * bundle this script generates in results/twin-loop/bundle; the bundle + * fingerprint in the log is checked against it. */ import './lib/offline-env'; import { execFileSync } from 'child_process'; -import crypto from 'crypto'; import fs from 'fs'; import path from 'path'; import { CodeGenerationTemplate, Metamodel, Model } from '../shared/types'; @@ -93,8 +92,32 @@ const scenarios: Scenario[] = [ : ['expected one realization offset of Loading Dock by 150 mm in z from t = 0 s']; }, }, + { + name: 'transient-drift', + description: 'Storage Rack #2 is pushed into the aisle at t = 20 s and back to its place at t = 40 s.', + disturbances: ['Storage Rack #2:-1.2,0@20', 'Storage Rack #2:1.2,0@40'], + check: report => { + const failures: string[] = []; + const [deviation] = report.deviations; + if (report.deviations.length !== 1 || deviation.kind !== 'runtime-drift' || deviation.onsetT !== 20 + || deviation.deltaMm.x !== -1200 || deviation.finalDeltaMm.x !== 0) { + failures.push('expected one runtime drift of Storage Rack #2 by -1200 mm at t = 20 s, back in place at the end'); + } + const aisle = report.ruleFindings.find(item => item.kind === 'runtime-rule-violation' && item.ruleId === RACK_AISLE_RULE); + if (!aisle || aisle.firstT !== 20 || aisle.lastT !== 39.5) { + failures.push('expected the rack aisle rule to fail at runtime from t = 20 s to t = 39.5 s only'); + } + if (report.reconciliation.patch.length !== 0) failures.push('expected an empty reconciliation patch'); + return failures; + }, + }, ]; +// Read a log given on the command line before the results folder is cleaned: it may live inside it. +const externalIndex = process.argv.indexOf('--observations'); +const externalFile = externalIndex >= 0 ? path.resolve(process.argv[externalIndex + 1] || '') : undefined; +const external = externalFile ? readObservationLog(externalFile) : undefined; + const runPython = (args: string[]): void => { if (!fs.existsSync(python)) throw new Error(`Missing ${python}; see benchmarks/README.md for the venv setup`); execFileSync(python, args, { cwd: simDir, stdio: ['ignore', 'ignore', 'inherit'] }); @@ -116,15 +139,22 @@ BUNDLE_FILES.forEach(name => { // Layers reference assets as @omniverse-assets/...@ relative to the bundle root. fs.symlinkSync(path.join(repoRoot, 'examples', 'omniverse-assets'), path.join(bundleDir, 'omniverse-assets'), 'dir'); const layoutPath = path.join(bundleDir, 'warehouse_layout.json'); -const layoutSha256 = crypto.createHash('sha256').update(fs.readFileSync(layoutPath)).digest('hex'); const authoredPath = path.join(bundleDir, 'authored.jsonl'); -runPython(['observe.py', 'snapshot', '--scene', path.join(bundleDir, 'warehouse_simulation.usda'), '--out', authoredPath]); +runPython([ + 'observe.py', 'snapshot', '--scene', path.join(bundleDir, 'warehouse_simulation.usda'), '--layout', layoutPath, + '--out', authoredPath, +]); const authored = readObservationLog(authoredPath); +const bundleFingerprint = authored.header.bundleFingerprint; const analyse = (name: string, title: string, observations: ObservationLog): TwinReport => { - if (observations.header.layoutSha256 !== layoutSha256) { - throw new Error(`${name}: the log was recorded against a different generated layout ` - + `(${observations.header.layoutSha256}) than this model generates (${layoutSha256})`); + if (!bundleFingerprint || observations.header.bundleFingerprint !== bundleFingerprint) { + const expected = authored.header.bundleFiles || {}; + const recorded = observations.header.bundleFiles || {}; + const differing = [...new Set([...Object.keys(expected), ...Object.keys(recorded)])] + .filter(file => expected[file] !== recorded[file]); + throw new Error(`${name}: the log was recorded against a different generation of the model; ` + + `generated files that differ: ${differing.join(', ') || 'unknown (the log carries no bundle fingerprint)'}`); } const { report, reconciledModel } = analyzeRun({ model, metamodel, modelHash: contentHash(model), metamodelHash: contentHash(metamodel), @@ -140,12 +170,9 @@ const analyse = (name: string, title: string, observations: ObservationLog): Twi return report; }; -const externalIndex = process.argv.indexOf('--observations'); -if (externalIndex >= 0) { - const file = path.resolve(process.argv[externalIndex + 1] || ''); - const observations = readObservationLog(file); - const report = analyse('external', path.basename(file), observations); - process.stdout.write(`${renderReport(path.basename(file), report)}\nReport written to ${path.join(outDir, 'external')}\n`); +if (externalFile && external) { + const report = analyse('external', path.basename(externalFile), external); + process.stdout.write(`${renderReport(path.basename(externalFile), report)}\nReport written to ${path.join(outDir, 'external')}\n`); process.exit(0); } @@ -175,7 +202,7 @@ const summary = { tracedElements: rows[0].report.provenance.observation.tracedElements, samplesPerRun: rows[0].report.provenance.observation.samples, modelHash: rows[0].report.provenance.modelHash, - layoutSha256, + bundleFingerprint, seconds: Math.round((Date.now() - started) / 100) / 10, }; fs.writeFileSync(path.join(resultsDir, 'twin-loop.json'), `${JSON.stringify(summary, null, 2)}\n`); diff --git a/examples/omniverse-assets/warehouse_sim/README.md b/examples/omniverse-assets/warehouse_sim/README.md index d2c8b84..ab90687 100644 --- a/examples/omniverse-assets/warehouse_sim/README.md +++ b/examples/omniverse-assets/warehouse_sim/README.md @@ -108,15 +108,18 @@ generated `warehouse_layout.json` (`id`). Add `--observe-out observations.jsonl` the world pose of every stamped prim while the scene runs. It needs `--scene`, because only generated prims carry the stamp. -The log is JSON Lines: a header naming the source, the hash of the layout the -run used, and the elements the simulator drives, then one record per element +The log is JSON Lines: a header naming the source, a fingerprint of the +generated files the run used (`--layout`, the `--scene` root layer, and its +sublayers), and the elements the simulator drives, then one record per element per sample (`t`, `elementId`, `prim`, `x`, `y`, `z` in metres, `yaw` in degrees). `observe.py` documents the format. `run_standin_sim.py` writes the same log without Isaac Sim, a GPU, or OPC UA: the brain drives the robots kinematically, everything else stays at its authored pose, and `--disturb "NAME:DX,DY[,DZ[,DYAW]]@T"` displaces an element -at a chosen time. `benchmarks/README.md` (Twin loop) describes how a log is +at a chosen time (for a robot, the brain sees the displaced pose too). Its +`--authored` input is `observe.py snapshot --scene ... --layout ...` taken from +the same bundle as `--layout`. `benchmarks/README.md` (Twin loop) describes how a log is lifted back onto the model and checked against its spatial rules. ## Tests diff --git a/examples/omniverse-assets/warehouse_sim/observe.py b/examples/omniverse-assets/warehouse_sim/observe.py index e3fbc94..55e051a 100644 --- a/examples/omniverse-assets/warehouse_sim/observe.py +++ b/examples/omniverse-assets/warehouse_sim/observe.py @@ -10,16 +10,20 @@ write the same records. line 1 {"schema": "spatialdsl.observations/1", "source": ..., "units": ..., - "layoutSha256": ..., "scene": ..., "controlledElementIds": [...]} + "bundleFingerprint": ..., "bundleFiles": {...}, "scene": ..., + "controlledElementIds": [...]} line 2+ {"t": seconds, "elementId": ..., "prim": ..., "layer": ..., "x": m, "y": m, "z": m, "yaw": deg} -`layoutSha256` binds the log to the generated bundle it was recorded against. +`bundleFiles` maps each generated file the run used (the layout JSON, the root +scene layer, and its sublayers) to its SHA-256; `bundleFingerprint` hashes that +map. Together they bind the log to one generation of the model. `controlledElementIds` lists the elements the simulator drives (the robots); every other element is expected to stay where the model put it. Authored poses of a generated scene, with no simulator (needs pxr / usd-core): - python observe.py snapshot --scene warehouse_simulation.usda --out authored.jsonl + python observe.py snapshot --scene warehouse_simulation.usda \ + --layout warehouse_layout.json --out authored.jsonl """ import argparse @@ -40,10 +44,38 @@ def file_sha256(path): return digest.hexdigest() +def bundle_record(files): + """Header fields that identify a generated bundle, from {file name: SHA-256}.""" + ordered = dict(sorted(files.items())) + fingerprint = hashlib.sha256(json.dumps(ordered, sort_keys=True).encode("utf-8")).hexdigest() + return {"bundleFingerprint": fingerprint, "bundleFiles": ordered} + + +def scene_bundle(scene_path, layout_path=None): + """The generated files a run depends on: the layout JSON, the root scene layer, and its sublayers.""" + from pxr import Sdf + + files = {} + if layout_path: + files[os.path.basename(layout_path)] = file_sha256(layout_path) + pending = [os.path.abspath(scene_path)] + while pending: + path = pending.pop() + name = os.path.basename(path) + if name in files: + continue + files[name] = file_sha256(path) + layer = Sdf.Layer.FindOrOpen(path) + if layer is None: + raise SystemExit("Could not open USD layer: " + path) + pending.extend(os.path.normpath(os.path.join(os.path.dirname(path), sub)) for sub in layer.subLayerPaths) + return bundle_record(files) + + class ObservationRecorder: """Append-only writer for one observation log.""" - def __init__(self, path, *, source, layout_sha256=None, scene=None, controlled=()): + def __init__(self, path, *, source, bundle=None, scene=None, controlled=()): self.path = path self.samples = 0 self._handle = open(path, "w", encoding="utf-8") @@ -52,7 +84,8 @@ def __init__(self, path, *, source, layout_sha256=None, scene=None, controlled=( "source": source, "units": {"length": "m", "angle": "deg"}, "frame": "world, right-handed, Z up", - "layoutSha256": layout_sha256, + "bundleFingerprint": (bundle or {}).get("bundleFingerprint"), + "bundleFiles": (bundle or {}).get("bundleFiles"), "scene": scene, "controlledElementIds": sorted(controlled), }) @@ -125,7 +158,7 @@ def read_log(path): return lines[0], samples -def snapshot(scene_path, out_path): +def snapshot(scene_path, out_path, layout_path=None): """Write the authored (pre-simulation) pose of every traced prim as one sample.""" from pxr import Usd @@ -138,7 +171,10 @@ def snapshot(scene_path, out_path): "No prim in %s carries %s; regenerate the bundle with the current templates." % (scene_path, ELEMENT_ID_ATTRIBUTE) ) - recorder = ObservationRecorder(out_path, source="usd-authored", scene=os.path.basename(scene_path)) + recorder = ObservationRecorder( + out_path, source="usd-authored", bundle=scene_bundle(scene_path, layout_path), + scene=os.path.basename(scene_path), + ) recorder.sample(0.0, sample_stage(traced)) recorder.close() return len(traced) @@ -149,9 +185,11 @@ def main(): commands = parser.add_subparsers(dest="command", required=True) snap = commands.add_parser("snapshot", help="write the authored poses of a generated scene") snap.add_argument("--scene", required=True) + snap.add_argument("--layout", help="the generated layout JSON, included in the bundle fingerprint") snap.add_argument("--out", required=True) args = parser.parse_args() - count = snapshot(os.path.abspath(args.scene), os.path.abspath(args.out)) + layout = os.path.abspath(args.layout) if args.layout else None + count = snapshot(os.path.abspath(args.scene), os.path.abspath(args.out), layout) print("[observe] %d traced prims written to %s" % (count, args.out)) diff --git a/examples/omniverse-assets/warehouse_sim/run_standin_sim.py b/examples/omniverse-assets/warehouse_sim/run_standin_sim.py index 0a17468..861ba2a 100644 --- a/examples/omniverse-assets/warehouse_sim/run_standin_sim.py +++ b/examples/omniverse-assets/warehouse_sim/run_standin_sim.py @@ -11,7 +11,8 @@ --authored authored.jsonl --observe-out observations.jsonl \ --disturb "Storage Rack #2:-1.2,0@20" -`--authored` is the output of `observe.py snapshot` for the generated scene. +`--authored` is the output of `observe.py snapshot --layout` for the same +generated bundle as `--layout`. `--disturb NAME:DX,DY[,DZ[,DYAW]]@T` displaces the named element by metres and degrees at simulation time T (repeatable). T = 0 means the element never sits at its authored pose. @@ -55,7 +56,13 @@ def parse_args(): def run(layout_path, authored_path, out_path, *, seconds=60.0, observe_hz=2.0, disturbances=()): layout = load_layout(layout_path) - _, authored_samples = read_log(authored_path) + authored_header, authored_samples = read_log(authored_path) + bundle_files = authored_header.get("bundleFiles") or {} + if file_sha256(layout_path) not in bundle_files.values(): + raise SystemExit( + "The layout and the authored snapshot come from different generations; take the snapshot " + "with `observe.py snapshot --layout` from the same bundle." + ) poses = {element_id: dict(record) for element_id, record in authored_samples[0][1].items()} for pose in poses.values(): pose.pop("t", None) @@ -82,7 +89,8 @@ def run(layout_path, authored_path, out_path, *, seconds=60.0, observe_hz=2.0, d } recorder = ObservationRecorder( - out_path, source="standin", layout_sha256=file_sha256(layout_path), + out_path, source="standin", + bundle={"bundleFingerprint": authored_header.get("bundleFingerprint"), "bundleFiles": bundle_files}, scene=os.path.basename(authored_path), controlled=robot_ids.values(), ) pending = sorted(disturbances, key=lambda item: item["t"]) @@ -97,6 +105,10 @@ def run(layout_path, authored_path, out_path, *, seconds=60.0, observe_hz=2.0, d pose["y"] += disturbance["dy"] pose["z"] += disturbance["dz"] pose["yaw"] += disturbance["dyaw"] + if disturbance["name"] in reported: + # A driven robot: move the pose the brain sees too, or the next step undoes the disturbance. + x, y, heading = reported[disturbance["name"]] + reported[disturbance["name"]] = (x + disturbance["dx"], y + disturbance["dy"], heading + disturbance["dyaw"]) for name, (x, y, heading) in reported.items(): poses[robot_ids[name]].update({"x": x, "y": y, "yaw": heading}) if step % sample_every == 0: diff --git a/examples/omniverse-assets/warehouse_sim/run_warehouse_sim.py b/examples/omniverse-assets/warehouse_sim/run_warehouse_sim.py index e771c08..aa5baac 100644 --- a/examples/omniverse-assets/warehouse_sim/run_warehouse_sim.py +++ b/examples/omniverse-assets/warehouse_sim/run_warehouse_sim.py @@ -358,7 +358,7 @@ def add_prop(group, record): recorder = observe.ObservationRecorder( os.path.abspath(os.path.expanduser(args.observe_out)), source="isaac-sim", - layout_sha256=observe.file_sha256(args.layout), + bundle=observe.scene_bundle(args.scene, args.layout), scene=os.path.basename(args.scene), controlled=[r["id"] for r in robots if r["id"]], ) diff --git a/examples/omniverse-assets/warehouse_sim/test_observe.py b/examples/omniverse-assets/warehouse_sim/test_observe.py index 9666434..0daf5da 100644 --- a/examples/omniverse-assets/warehouse_sim/test_observe.py +++ b/examples/omniverse-assets/warehouse_sim/test_observe.py @@ -33,7 +33,18 @@ "drones": [], } -STAMPED_SCENE = """#usda 1.0 +ROOT_SCENE = """#usda 1.0 +( + defaultPrim = "World" + metersPerUnit = 1 + upAxis = "Z" + subLayers = [ + @./stamped.usda@ + ] +) +""" + +STAMPED_LAYER = """#usda 1.0 ( defaultPrim = "World" metersPerUnit = 1 @@ -66,7 +77,8 @@ def setUp(self): with open(self.layout_path, "w", encoding="utf-8") as handle: json.dump(LAYOUT, handle) self.authored_path = self.path("authored.jsonl") - recorder = observe.ObservationRecorder(self.authored_path, source="usd-authored") + self.bundle = observe.bundle_record({"warehouse_layout.json": observe.file_sha256(self.layout_path)}) + recorder = observe.ObservationRecorder(self.authored_path, source="usd-authored", bundle=self.bundle) recorder.sample(0.0, [ {"elementId": record["id"], "prim": "/World/" + record["id"], "layer": "warehouse_layout.usda", "x": record["x"], "y": record["y"], "z": record["z"] + record.get("height", 0.0) / 2.0, "yaw": 0.0} @@ -89,10 +101,11 @@ def test_log_starts_with_a_header_and_groups_samples_by_time(self): self.assertEqual(len(samples), 1) self.assertEqual(samples[0][1]["rack"]["z"], 1.25) - def test_standin_moves_only_the_robots_and_binds_the_log_to_the_layout(self): + def test_standin_moves_only_the_robots_and_binds_the_log_to_the_bundle(self): header, samples = self.run_standin() self.assertEqual(header["controlledElementIds"], ["robot-a", "robot-b"]) - self.assertEqual(header["layoutSha256"], observe.file_sha256(self.layout_path)) + self.assertEqual(header["bundleFingerprint"], self.bundle["bundleFingerprint"]) + self.assertEqual(header["bundleFiles"], self.bundle["bundleFiles"]) self.assertEqual(len(samples), 21) # 2 Hz over 10 s, both ends included first, last = samples[0][1], samples[-1][1] self.assertEqual((first["rack"]["x"], first["rack"]["y"]), (last["rack"]["x"], last["rack"]["y"])) @@ -108,17 +121,34 @@ def test_disturbance_displaces_the_element_from_its_time_on(self): self.assertAlmostEqual(by_time[10.0]["rack"]["y"], 0.5) self.assertAlmostEqual(by_time[10.0]["rack"]["yaw"], 15.0) + def test_disturbing_a_robot_moves_the_pose_the_brain_sees(self): + disturbance = run_standin_sim.parse_disturbance("Robot A:2,3,0,15@0") + _, samples = self.run_standin(disturbances=[disturbance]) + first, second = samples[0][1]["robot-a"], samples[1][1]["robot-a"] + self.assertEqual((first["x"], first["y"], first["yaw"]), (-16.0, -5.0, 15.0)) + # The brain continues from the displaced pose instead of snapping back to (-18, -8). + self.assertLess(((second["x"] + 16.0) ** 2 + (second["y"] + 5.0) ** 2) ** 0.5, 1.0) + + def test_standin_refuses_a_layout_from_another_generation(self): + with open(self.layout_path, "w", encoding="utf-8") as handle: + json.dump({**LAYOUT, "obstacles": []}, handle) + with self.assertRaises(SystemExit): + self.run_standin() + @unittest.skipUnless(HAVE_PXR, "pxr (usd-core) is not installed") - def test_snapshot_reads_the_stamp_pose_and_authoring_layer(self): + def test_snapshot_reads_the_stamp_pose_authoring_layer_and_bundle(self): scene = self.path("scene.usda") with open(scene, "w", encoding="utf-8") as handle: - handle.write(STAMPED_SCENE) + handle.write(ROOT_SCENE) + with open(self.path("stamped.usda"), "w", encoding="utf-8") as handle: + handle.write(STAMPED_LAYER) out = self.path("snapshot.jsonl") - self.assertEqual(observe.snapshot(scene, out), 1) + self.assertEqual(observe.snapshot(scene, out, self.layout_path), 1) header, samples = observe.read_log(out) record = samples[0][1]["rack"] self.assertEqual(header["source"], "usd-authored") - self.assertEqual((record["prim"], record["layer"]), ("/World/Rack_0", "scene.usda")) + self.assertEqual(sorted(header["bundleFiles"]), ["scene.usda", "stamped.usda", "warehouse_layout.json"]) + self.assertEqual((record["prim"], record["layer"]), ("/World/Rack_0", "stamped.usda")) self.assertEqual((record["x"], record["y"], record["z"]), (10.0, 12.0, 1.25)) self.assertAlmostEqual(record["yaw"], 30.0, places=4)