From a122bd5b7aad86ba211e6b3dbcd05e389ad6b732 Mon Sep 17 00:00:00 2001 From: leroy ketelaars Date: Wed, 11 Nov 2026 13:15:42 +0000 Subject: [PATCH 1/7] Add opt-in texture production controls, model bindings and acceptance tooling --- .sdk-revision | 2 +- docs/progressive-production.md | 192 ++++++++++++++++++ docs/progressive-textures.md | 5 + .../TextureRuntimeScenario.cs | 2 +- src/Hypercube.Core/ResidentMipTailInfo.cs | 24 +++ src/Hypercube.Core/TextureWorkBudget.cs | 71 +++++++ src/Hypercube.GameCube/AramCache.cs | 2 + src/Hypercube.GameCube/ResidentMipTail.cs | 70 +++++++ .../StreamedMaterialBinding.cs | 2 +- .../StreamedModelMaterials.cs | 61 ++++++ src/Hypercube.GameCube/TextureLodStreamer.cs | 30 ++- src/Hypercube.GameCube/WorldStreamer.cs | 17 +- tests/MipTailRuntime/MipTailRuntime.csproj | 1 + tests/MipTailRuntime/Program.cs | 34 ++++ tests/StreamedModelRuntime/Program.cs | 43 ++++ .../StreamedModelRuntime.csproj | 1 + tests/TextureFamilies.Tests/Program.cs | 27 ++- tests/WorldTextureLodRuntime/Program.cs | 5 +- tests/model_sidecar_fixtures.py | 31 +++ tests/test_progressive_acceptance.py | 44 ++++ tests/test_progressive_polish.py | 73 +++++++ tests/texture_lod_host.c | 6 +- tools/build_mip_tail.py | 30 +++ tools/build_texture_comparison.py | 51 +++++ tools/build_texture_pack.py | 21 +- tools/hypercube/model_bindings.py | 158 ++++++++++++++ tools/progressive_acceptance.py | 118 +++++++++++ tools/test_progressive_runtime.py | 17 +- 28 files changed, 1124 insertions(+), 14 deletions(-) create mode 100644 docs/progressive-production.md create mode 100644 src/Hypercube.Core/ResidentMipTailInfo.cs create mode 100644 src/Hypercube.Core/TextureWorkBudget.cs create mode 100644 src/Hypercube.GameCube/ResidentMipTail.cs create mode 100644 src/Hypercube.GameCube/StreamedModelMaterials.cs create mode 100644 tests/MipTailRuntime/MipTailRuntime.csproj create mode 100644 tests/MipTailRuntime/Program.cs create mode 100644 tests/StreamedModelRuntime/Program.cs create mode 100644 tests/StreamedModelRuntime/StreamedModelRuntime.csproj create mode 100644 tests/model_sidecar_fixtures.py create mode 100644 tests/test_progressive_acceptance.py create mode 100644 tests/test_progressive_polish.py create mode 100644 tools/build_mip_tail.py create mode 100644 tools/build_texture_comparison.py create mode 100644 tools/hypercube/model_bindings.py create mode 100644 tools/progressive_acceptance.py diff --git a/.sdk-revision b/.sdk-revision index 2918621..e3bbba3 100644 --- a/.sdk-revision +++ b/.sdk-revision @@ -1 +1 @@ -dd8a187b9c35d526b479511846a92bef5873fb29 +415476f42eddba4c3a01fc81902da60b06280486 diff --git a/docs/progressive-production.md b/docs/progressive-production.md new file mode 100644 index 0000000..2f386f3 --- /dev/null +++ b/docs/progressive-production.md @@ -0,0 +1,192 @@ +# Progressive texture production controls and acceptance + +This completes the implementation lanes in the PR11 design. **Production tuning +and the page/atlas strategy decision remain conditional on real-console results.** +The checked-in phase-four measurements are Dolphin correctness evidence. No +physical GameCube measurements are claimed, and adaptive/prefetch/tail behavior +is opt-in. DNTX, fixed-size StreamingTexture and HCPK v1 retain their existing +paths. See [progressive-textures.md](progressive-textures.md) for the base mode. + +## Budget feedback, prefetch and cold copies + +`TextureBudgetController.Next(frameCpuUs, lastPumpUs, lastReadUs, audioStarved, +criticalQueued)` returns a bounded `TextureWorkBudget`. Measure CPU work **before +present**, excluding GX/VI waits; use measured read completion latency, not read +submission time. The starting 32 KiB maximum / 4 KiB minimum chunks, 1 ms pump +allowance and 8 ms CPU/storage soft limits are configurable experiment inputs. +Slow work halves issuance, sixteen stable frames grow it, and audio/gameplay or +insufficient CPU headroom pauses new issuance. `TextureLodStreamer.Pump(budget)` +still drains already-started ownership when issuance is paused. `LastPumpUs` and +the controller's pause/reduction/growth counters support profiling. A deadline +cannot interrupt a native operation already in progress. + +`Prefetch(binding, hint, budget)` admits coarse readiness only if probability, +time-to-visible, camera-direction agreement, projected size, CPU and physical +headroom are sufficient. Seen-recently lowers the probability/time threshold for +reversal. Visible demand always wins; prefetch never raises visible quality. +Supply visibility/motion evidence from the application's scene, rather than +prefetching every catalog entry. Both features are off unless called explicitly. + +`SetColdRetention(Disabled|ReversalCache|StrictRelease)` distinguishes counted +ARAM reuse from strict removal of unused levels. Initialize ARAM before enabling +it. Disable drains an in-flight protected copy before dropping it; strict mode +drops old copies when their residency is released. For a shared `WorldStreamer`, +call **WorldStreamer.SetColdRetention**, which applies the change between legacy +batches, so queued ARAM fetches cannot lose their source. Standalone streams use +their own setter. Explicit stop, final GPU fence and trim are still required for +zero-allocation shutdown. ARAM bytes are reported separately from MEM1. + +## Model sidecars and generated bindings + +With the SDK pinned by this branch: + +```sh +dotnet run --project "$DOLPHIN_DOTNET_ROOT/tools/DolphinDotNet.Assets" -- \ + model actor.glb actor.dndmodel --streamable-textures --compile-gx-materials +python3 tools/build_texture_pack.py \ + --model-sidecar actor.dndmodel.textures.json \ + --model-sidecar props.dndmodel.textures.json \ + --output out/models.hcp --bindings Generated/ModelMaterials.g.cs +``` + +The packer validates the model SHA256, DNDM section hashes, sidecar material IDs, +paths, image hashes, DNLT CRCs, samplers and combiners. Relative paths must remain +under the sidecar directory. Identical image bytes across materials/models share +one family. All coarse records precede all upgrades. Geometry, skinning and +animations stay in the model package; no resident model texture chain is hidden +in the pack. The generated `Create0`, `Create1`, etc. factories map to input order +and are recorded in the pack manifest. Generated code replaces runtime JSON +parsing and returns `StreamedModelMaterials` ownership; dispose it after its +final draw frame to release its shared multiply/add TEV programs. + +Load the model through the SDK, create an instance, then call `DemandPart` only +for visible material groups before pumping. `DrawPart` resolves and pins both +texture slots at actual submission through `DrawModelMaterialPart`. Queue the +binding plus material index, not a resolved Texture2D. Separate instances may +have independent policies while sharing family allocations. Imported authored +UV transforms are already baked into the geometry once; selected-level gutter +transforms compose at draw time for both slots. + +Offline UV density uses triangle derivatives in the model's **rest pose**, +including imported node transforms and authored UV repeats. Supply maximum +application scale times a conservative animation/deformation multiplier to +`DemandPart`; use direct projected footprints for deforming or oblique cases. +Degenerate UVs use its explicit fallback footprint. Submit opaque parts before +transparent parts and sort transparent parts in the application queue. The SDK +sorts primitives within each material group, not across separate calls. + +## Optional resident hardware mip tail + +```sh +python3 tools/build_mip_tail.py texture.dnlod --start-level 2 \ + --output tail.dndtex --maximum-native-bytes 65536 --maximum-dimension 128 +``` + +This mode is separate from the single-image family: it creates **one contiguous, +indivisible full chain** through 1×1, so GX can use trilinear filtering. Sparse, +NPOT pair, incomplete, mixed-format and padded atlas/gutter tails are rejected. +The default upper limit is 128 pixels / 64 KiB native; optional hard maxima are +256 pixels / 256 KiB. The manifest includes the whole-package CRC32. + +`ResidentMipTail.LoadAsync(path, manifestCrc, borrowedNativeBuffer, logicalBase)` +checks only the 32-byte header synchronously, then reads bounded native chunks, +validates the entire package and publishes on the application executor. Reserve +exclusive use/lifetime of the supplied staging buffer until the task completes; +it is not disposed by the loader. This optional, capped mode uses a bounded +managed package array for the existing SDK DNTX factory. The ordinary family +upload path still has no whole-image managed staging. + +Reserve the **entire** new chain, staging and any old retiring chain before +loading. `Apply(material)` enables trilinear filtering. It does not update family +residency or automatically replace an old texture. `RetireAfter(finalFence)` and +`TryCollect()` release the chain without promotion-time GPU waiting. Account its +ActiveBytes/RetiredBytes alongside the application's other native allocations; +individual tail levels cannot be reclaimed. No tail is enabled by default. + +## Matched page/atlas experiment + +`python3 tools/build_texture_comparison.py --output out/texture-comparison` +authors paired fine-page and 2×2 cell-atlas HCPK v2 packs from the same 192×128 +validation image, with identical useful RGBA8 texels at logical levels 0/1/2, +four-texel gutters and normalized geometry mappings. `--format cmpr` is a +separate equal-format lane. The manifest includes source/camera/pack hashes, +quad mappings, coarse/native headroom and a common physical ceiling. Python +checks compare **every useful texel at every authored level** in the RGBA8 pair. +The common camera trace includes movement and turns. This authors experiment +inputs; it does not simulate throughput or choose a winner. + +Render each manifest's quads with the same shader, sampler, viewport, camera +trace, audio and object/animation load. Use the same physical ceiling, charging +active images, a detached target, source buffer, retired images, idle pool and +external scene resources. Choose a ceiling large enough for the complete +coarse set plus replacement/retirement/staging headroom. The builder checks that +minimum; the runtime must enforce the ceiling including scene resources. + +Capture a common deterministic frame from both strategies and a full-detail +reference. Admit comparisons with: + +```sh +python3 tools/progressive_acceptance.py --output comparison.json compare \ + pages-run.json atlas-run.json --reference reference.png \ + --page-image pages.png --atlas-image atlas.png --maximum-rmse 2 +``` + +Admission rejects differing content, camera, shader, sampler, viewport, source +texel density, native format, SDK or physical ceiling. Both captures must be the +same camera frame and meet the reference RMSE threshold (RGB 0..255 units). +Report P95/P99 frame times, P95 coarse/detail readiness, physical/ARAM peaks, +storage/ARAM traffic, draws and texture binds together. A lower-detail or +higher-memory result cannot establish a faster strategy. Repeat runs and +compare variation before choosing defaults. Report hashes bind input runs and +captures; retain those raw artifacts with the report. + +## Version 1 measured-run contract and hardware gate + +A run JSON contains `schema:1`, a unique `runId`, `platform` (`Dolphin` or +`GameCube`), `storage` (`sd2sp2` or `disc`), `strategy` (`fine-pages` or +`cell-atlas`), and physical-console/video identification in `console` for +hardware. Include 64-character lowercase SHA256 fields `contentSha256`, +`sceneSha256`, `cameraTraceSha256`, `shaderSha256`, `samplerSha256`, `dolSha256`, +`packSha256`; a 40-character `sdkRevision`; `width`, `height`, `format` +(`RGBA8`/`CMPR`), `texelsPerWorldUnit`, `physicalCeilingBytes`, and `captureFrame`. + +Record at least 300 measured `frames`, each with `frameUs`, `audioActive:true`, +and a `memory` object. Its fields are `activeBytes`, `uploadBytes` (the entire +private target allocation, not just bytes already copied), `readBufferBytes`, +`retiredBytes`, `poolBytes`, `externalBytes`, `physicalBytes` (sum of those six), +and separate `aramBytes`. Record `finalMemory` with the same fields all zero +**after** final fence/trim/shutdown. Include at least four observations each in +`coarseReadyUs` and `detailReadyUs`, and measured totals `storageBytes`, +`aramReadBytes`, `draws`, `textureBinds`, `cancellations`, `checksumFailures`, +`pressureDeferrals`. The last three must be exercised rather than left zero. + +`stressCases` must include `camera-turn`, `multiple-objects`, `budget-pressure`, +`camera-reversal`, `corrupt-read`, `cancel-read`. Supply observed evidence in +`checks`, all true: `coarseBeforeDetail`, `noHoles`, `noStaleDraws`, `noUvJumps`, +`noCorruptPixels`, `corruptNeverPublished`, `cancelNeverPublished`, +`pairReachedHigh`, `noPromotionGpuWait`, `noSynchronousWholeTextureRead`. +These are operator/application observations: the validator cannot establish +physical hardware provenance or visible correctness from a self-reported JSON. +Retain video, pixel patches, instrumentation and source/DOL hashes for review. +Never substitute the unit-test contract data for a measured run. + +Run three independent Release repeats per strategy on **both** real GameCube +SD2SP2 and disc (12 runs), with audio active, multiple objects/animations, +camera reversal and budget pressure. Produce six accepted screenshot +comparisons, one for each matched repeat pair. Then invoke: + +```sh +python3 tools/progressive_acceptance.py --output hardware.json hardware \ + measured/*.run.json --comparison sd-pair-1.json --comparison sd-pair-2.json \ + --comparison sd-pair-3.json --comparison disc-pair-1.json \ + --comparison disc-pair-2.json --comparison disc-pair-3.json \ + --maximum-frame99-us 33334 --maximum-coarse95-us 500000 \ + --maximum-detail95-us 2000000 +``` + +These example thresholds are experiment targets, not achieved measurements. +Specify application readiness thresholds before collecting results. The gate +rejects emulator timing, changed workloads, incomplete repeats, missing matched +quality evidence, accounting leaks and failed correctness/performance checks. +Physical-console access is required to complete this gate; it remains pending +in this delivery. diff --git a/docs/progressive-textures.md b/docs/progressive-textures.md index a8a1e3d..9c6f805 100644 --- a/docs/progressive-textures.md +++ b/docs/progressive-textures.md @@ -175,3 +175,8 @@ failure/retirement counters, and zero bytes after final trim. It records p95/p99 frame times and readiness latency as emulator observations, with an explicit `hardwarePerformanceClaim: false` marker. Real GameCube SD2SP2/disc/audio pressure and latency gates still require physical-console captures. + +The opt-in budget/prefetch/ARAM controls, generated model materials, bounded +resident mip tails and matched-quality/hardware evidence contract are documented +in [progressive-production.md](progressive-production.md). Hardware acceptance +remains conditional on physical-console measurements. diff --git a/samples/ProgressiveTextures/TextureRuntimeScenario.cs b/samples/ProgressiveTextures/TextureRuntimeScenario.cs index 10aff5e..eb32f70 100644 --- a/samples/ProgressiveTextures/TextureRuntimeScenario.cs +++ b/samples/ProgressiveTextures/TextureRuntimeScenario.cs @@ -99,7 +99,7 @@ public void Step() int active=0;for(int i=0;i0,"ARAM retention did not retain cold copies."); - textures.StrictReleaseColdCopies=true;Next(7); + textures.SetColdRetention(TextureColdRetention.StrictRelease);Next(7); } else if(phase==8&&textures.Deferred>0&&frame-phaseFrame>=60) {Check(textures.ResidentIndex(1)==bindings[0].Family.CoarseIndex,"Pressure evicted coarse to start a promotion.");Next(9);} diff --git a/src/Hypercube.Core/ResidentMipTailInfo.cs b/src/Hypercube.Core/ResidentMipTailInfo.cs new file mode 100644 index 0000000..a026c3e --- /dev/null +++ b/src/Hypercube.Core/ResidentMipTailInfo.cs @@ -0,0 +1,24 @@ +using System; +using System.IO; + +namespace Hypercube; + +/// Opt-in bounded, indivisible DNTX v2 hardware mip tail. This is a +/// different residency mode from independently reclaimable DNLT family levels. +public readonly struct ResidentMipTailInfo +{ + public readonly int Width,Height,Format,Levels,NativeBytes; + private ResidentMipTailInfo(int width,int height,int format,int levels,int bytes) + {Width=width;Height=height;Format=format;Levels=levels;NativeBytes=bytes;} + public static ResidentMipTailInfo Read(byte[] header,long fileLength,int maximumNativeBytes=65536,int maximumDimension=128) + { + if(maximumNativeBytes<32||maximumNativeBytes>262144||maximumDimension<2||maximumDimension>256||(maximumDimension&(maximumDimension-1))!=0)throw new ArgumentOutOfRangeException("maximumNativeBytes"); + if(header==null||header.Length!=32||Binary.U32(header,0)!=0x58544e44||Binary.U32(header,4)!=2||Binary.U32(header,28)!=0)throw new IOException("Expected a DNTX v2 mip tail."); + int format=Binary.I32(header,8),w=Binary.I32(header,12),h=Binary.I32(header,16),declared=Binary.I32(header,20),levels=Binary.I32(header,24); + if(w<1||h<1||w>maximumDimension||h>maximumDimension||(w&(w-1))!=0||(h&(h-1))!=0||(format!=6&&format!=14)||levels<2||levels>9)throw new IOException("Mip tail dimensions, format or chain limit."); + int bytes=0,count=0,mw=w,mh=h; + while(true){bytes+=TextureFamilyReader.NativeBytes(mw,mh,format);count++;if(mw==1&&mh==1)break;mw=Math.Max(1,mw/2);mh=Math.Max(1,mh/2);} + if(count!=levels||declared!=bytes||bytes>maximumNativeBytes||fileLength!=32L+bytes)throw new IOException("Mip tail must be complete and fit its entire native byte budget."); + return new ResidentMipTailInfo(w,h,format,levels,bytes); + } +} diff --git a/src/Hypercube.Core/TextureWorkBudget.cs b/src/Hypercube.Core/TextureWorkBudget.cs new file mode 100644 index 0000000..e7ed582 --- /dev/null +++ b/src/Hypercube.Core/TextureWorkBudget.cs @@ -0,0 +1,71 @@ +using System; + +namespace Hypercube; + +public readonly struct TextureWorkBudget +{ + public readonly int UploadBytes,CpuMicroseconds; + public readonly bool AllowPrefetch; + public TextureWorkBudget(int uploadBytes,int cpuMicroseconds,bool allowPrefetch=false) + { + if(uploadBytes<0||(uploadBytes&31)!=0||cpuMicroseconds<1)throw new ArgumentOutOfRangeException("uploadBytes"); + UploadBytes=uploadBytes;CpuMicroseconds=cpuMicroseconds;AllowPrefetch=allowPrefetch&&uploadBytes>0; + } +} + +/// Opt-in feedback control. Supply measured CPU work before present, +/// the previous pump cost and read latency. Defaults are profiling starting +/// values; pending copies still drain when issuance is paused. +public sealed class TextureBudgetController +{ + private int stable; + public int MinimumChunkBytes {get;} + public int MaximumChunkBytes {get;} + public int ChunkBytes {get;private set;} + public int CpuBudgetUs {get;} + public int FrameCpuBudgetUs {get;} + public int ReadLatencySoftLimitUs {get;} + public int PausedFrames,ReducedFrames,Growths; + public TextureBudgetController(int maximumChunkBytes=32768,int minimumChunkBytes=4096,int cpuBudgetUs=1000,int frameCpuBudgetUs=8000,int readLatencySoftLimitUs=8000) + { + if(minimumChunkBytes<32||(minimumChunkBytes&31)!=0||maximumChunkBytes131072||cpuBudgetUs<1||frameCpuBudgetUs0||headroomcpu||lastReadUs>ReadLatencySoftLimitUs) + {ChunkBytes=Math.Max(MinimumChunkBytes,(ChunkBytes/2)&~31);stable=0;ReducedFrames++;} + else if(lastPumpUs=cpu) + { + if(++stable>=16){int next=Math.Min(MaximumChunkBytes,ChunkBytes*2);if(next!=ChunkBytes)Growths++;ChunkBytes=next;stable=0;} + } + else stable=0; + return new TextureWorkBudget(ChunkBytes,cpu,headroom-cpu>=cpu&&lastReadUs<=ReadLatencySoftLimitUs); + } +} + +public readonly struct TexturePrefetchHint +{ + public readonly float VisibilityProbability,SecondsToVisible,DirectionDot,ProjectedPixels; + public readonly bool SeenRecently; + public TexturePrefetchHint(float probability,float secondsToVisible,float directionDot,float projectedPixels,bool seenRecently=false) + { + if(float.IsNaN(probability)||probability<0||probability>1||float.IsNaN(secondsToVisible)||float.IsInfinity(secondsToVisible)||secondsToVisible<0|| + float.IsNaN(directionDot)||directionDot< -1||directionDot>1||float.IsNaN(projectedPixels)||float.IsInfinity(projectedPixels)||projectedPixels<0)throw new ArgumentOutOfRangeException("probability"); + VisibilityProbability=probability;SecondsToVisible=secondsToVisible;DirectionDot=directionDot;ProjectedPixels=projectedPixels;SeenRecently=seenRecently; + } +} + +public static class TexturePrefetchPolicy +{ + public static bool Likely(TexturePrefetchHint hint,TextureWorkBudget budget,int freeNativeBytes,int coarseBytes,int reserveBytes=32768) + { + if(freeNativeBytes<0||coarseBytes<1||reserveBytes<0)throw new ArgumentOutOfRangeException("freeNativeBytes"); + return budget.AllowPrefetch&&coarseBytes<=freeNativeBytes&&reserveBytes<=freeNativeBytes-coarseBytes&&hint.ProjectedPixels>=8&& + hint.VisibilityProbability>=(hint.SeenRecently?.6f:.75f)&&hint.SecondsToVisible<=(hint.SeenRecently?1:.75f)&&hint.DirectionDot>=.25f; + } +} diff --git a/src/Hypercube.GameCube/AramCache.cs b/src/Hypercube.GameCube/AramCache.cs index 5f77c2b..2b406b5 100644 --- a/src/Hypercube.GameCube/AramCache.cs +++ b/src/Hypercube.GameCube/AramCache.cs @@ -30,6 +30,8 @@ public bool Drop(int id) } public int Trim() {int before=Bytes;for(int i=1;i0&&Bytes==(buffers[protectedRecord]?.Length??0))return 0;for(int i=1;iFetchRange(id,0,target,targetOffset,bytes); public Task FetchRange(int id,int sourceOffset,NativeBuffer target,int targetOffset,int bytes) diff --git a/src/Hypercube.GameCube/ResidentMipTail.cs b/src/Hypercube.GameCube/ResidentMipTail.cs new file mode 100644 index 0000000..9664213 --- /dev/null +++ b/src/Hypercube.GameCube/ResidentMipTail.cs @@ -0,0 +1,70 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Dolphin.Graphics; +using Dolphin.Storage; +using Scheduler=Dolphin.Threading.Tasks.Scheduler; + +namespace Hypercube.GameCube; + +/// Experimental opt-in native trilinear mode. Owns one complete, +/// contiguous chain; no individual level can be released. Borrow and reserve +/// a bounded staging buffer until LoadAsync completes; no source-buffer disposal +/// or promotion-time GPU wait is hidden in the loader. +public sealed class ResidentMipTail : IDisposable +{ + private Texture2D? texture; + private GpuFence finalReader; + private bool retiring,disposed; + public ResidentMipTailInfo Info {get;} + public int LogicalBaseLevel {get;} + public int ActiveBytes=>!retiring&&texture!=null?Info.NativeBytes:0; + public int RetiredBytes=>retiring&&texture!=null?Info.NativeBytes:0; + public int PhysicalBytes=>texture==null?0:Info.NativeBytes; + public Texture2D Texture=>!disposed&&!retiring&&texture!=null?texture:throw new InvalidOperationException("Tail is retired or disposed."); + private ResidentMipTail(Texture2D texture,ResidentMipTailInfo info,int logical){this.texture=texture;Info=info;LogicalBaseLevel=logical;} + public static async Task LoadAsync(string path,uint packageCrc32,NativeBuffer staging,int logicalBaseLevel=0, + int maximumNativeBytes=65536,int maximumDimension=128,CancellationToken cancellationToken=default) + { + if(!Scheduler.IsApplicationThread)throw new InvalidOperationException("Create mip-tail ownership on the application executor."); + if(staging==null||staging.Length<32||(staging.Length&31)!=0||logicalBaseLevel<0||logicalBaseLevel>10)throw new ArgumentException("Invalid staging buffer or logical base."); + cancellationToken.ThrowIfCancellationRequested();var header=new byte[32];long length; + using(var file=new FileStream(path,FileMode.Open,FileAccess.Read,FileShare.Read)) + {length=file.Length;int done=0;while(done<32){int n=file.Read(header,done,32-done);if(n==0)throw new EndOfStreamException();done+=n;}} + var info=ResidentMipTailInfo.Read(header,length,maximumNativeBytes,maximumDimension); + // Explicitly bounded managed storage is limited to this optional tail; + // the normal progressive path never allocates an image-sized array. + var package=new byte[32+info.NativeBytes];Array.Copy(header,package,32);uint crc=Binary.Crc(header,0,32); + using(var source=new AssetStream(path)) + { + if(source.Length!=length)throw new IOException("Mip-tail source changed during bootstrap."); + for(int at=0;atGenerated per-instance material wiring. Geometry and animations remain +/// in the SDK model; all instances share family allocations in the stream. +/// Owns at most two immutable TEV effects, never the instance or textures. +public sealed class StreamedModelMaterials : IDisposable +{ + private readonly TextureLodStreamer stream; + private readonly StreamedMaterialBinding?[] parts; + private readonly float[] worldU,worldV; + private GxEffect? multiply,add; + private bool disposed; + public int Count=>parts.Length; + public StreamedModelMaterials(TextureLodStreamer stream,int materialCount) + { + if(stream==null||materialCount<1||materialCount>4096)throw new ArgumentException("Invalid model material table."); + this.stream=stream;parts=new StreamedMaterialBinding?[materialCount];worldU=new float[materialCount];worldV=new float[materialCount]; + } + public void SetPart(int index,Material template,StreamedMaterialTexture? primary,StreamedMaterialTexture? secondary,int combiner,float worldPerU,float worldPerV) + { + Check(index);if(template==null||parts[index]!=null||combiner<0||combiner>2|| + (secondary!=null)!=(combiner!=0)||combiner!=0&&primary==null|| + float.IsNaN(worldPerU)||float.IsInfinity(worldPerU)||worldPerU<0||float.IsNaN(worldPerV)||float.IsInfinity(worldPerV)||worldPerV<0)throw new ArgumentException("Invalid streamed model material."); + if(combiner!=0) + { + var effect=combiner==1?multiply:add; + if(effect==null) + { + var program=new TevProgramBuilder();program.AddStage(TevInput.Zero,TevInput.Raster,TevInput.Texture,TevInput.Zero); + if(combiner==1)program.AddStage(TevInput.Zero,TevInput.Previous,TevInput.SecondaryTexture,TevInput.Zero); + else program.AddStage(TevInput.Zero,TevInput.SecondaryTexture,TevInput.One,TevInput.Previous); + effect=program.CreateEffect();if(combiner==1)multiply=effect;else add=effect; + } + template.Effect=effect; + } + parts[index]=new StreamedMaterialBinding(stream,template,primary,secondary);worldU[index]=worldPerU;worldV[index]=worldPerV; + } + private void Check(int index) + {if(disposed)throw new ObjectDisposedException("StreamedModelMaterials");if(index<0||index>=parts.Length)throw new ArgumentOutOfRangeException("index");} + public StreamedMaterialBinding GetPart(int index) + {Check(index);return parts[index]??throw new InvalidOperationException("Missing generated material.");} + /// Supply maximum application world scale times a conservative + /// animation deformation multiplier. Rest-pose UV density already includes + /// imported node transforms and authored UV repeats. Degenerate UVs use + /// the explicit fallback footprint. Call only for visible material groups. + public void DemandPart(int index,float worldScale,float centerDistance,float radius,float verticalFovDegrees,int viewportHeight,TextureFootprint fallback) + { + var part=GetPart(index);if(!(worldScale>0)||float.IsInfinity(worldScale))throw new ArgumentOutOfRangeException("worldScale"); + var footprint=worldU[index]>0&&worldV[index]>0?TextureFootprint.Project(worldU[index]*worldScale,worldV[index]*worldScale,centerDistance,radius,verticalFovDegrees,viewportHeight):fallback; + if(part.Primary!=null)stream.Demand(part.Primary,footprint);if(part.Secondary!=null)stream.Demand(part.Secondary,footprint); + } + /// Submit opaque parts first. Sort transparent material parts in + /// the application queue; the SDK sorts primitives within each part. + public void DrawPart(Renderer renderer,ModelInstance instance,Transform transform,int index,int frame) + =>renderer.DrawModelMaterialPart(instance,index,transform,GetPart(index).PrepareForDraw(frame)); + public void Dispose() + {if(disposed)return;multiply?.Dispose();add?.Dispose();disposed=true;} +} diff --git a/src/Hypercube.GameCube/TextureLodStreamer.cs b/src/Hypercube.GameCube/TextureLodStreamer.cs index c072405..4221286 100644 --- a/src/Hypercube.GameCube/TextureLodStreamer.cs +++ b/src/Hypercube.GameCube/TextureLodStreamer.cs @@ -5,9 +5,12 @@ using Dolphin.Graphics; using Dolphin.Storage; using Dolphin.Timing; +using Dolphin.Memory; namespace Hypercube.GameCube; +public enum TextureColdRetention { Disabled, ReversalCache, StrictRelease } + public readonly struct ResolvedTextureLod { public readonly Texture2D Texture; @@ -46,6 +49,16 @@ public sealed class TextureLodStreamer : IDisposable public readonly TextureFamilyDemand Demands; public readonly AramCache Cold; public bool UseAram,StrictReleaseColdCopies; + private bool trimDisabledCold; + public TextureColdRetention ColdRetention=>!UseAram?TextureColdRetention.Disabled:StrictReleaseColdCopies?TextureColdRetention.StrictRelease:TextureColdRetention.ReversalCache; + public void SetColdRetention(TextureColdRetention retention) + { + if(retentionTextureColdRetention.StrictRelease)throw new ArgumentOutOfRangeException("retention"); + if(retention!=TextureColdRetention.Disabled&&!Aram.IsAvailable)throw new InvalidOperationException("Initialize ARAM before enabling a cold tier."); + UseAram=retention!=TextureColdRetention.Disabled;StrictReleaseColdCopies=retention!=TextureColdRetention.ReversalCache; + trimDisabledCold=!UseAram; + if(trimDisabledCold)Cold.TrimExcept(slot<0?0:Policy.Id(slot)); + } public bool CorruptNextChunk,ShortNextRead; public int Frame {get;private set;} public int CompletedFrame {get;private set;} @@ -59,6 +72,7 @@ public sealed class TextureLodStreamer : IDisposable public int AramBytes=>Cold.Bytes; public int PeakBytes,Publications,Cancellations,StaleUploads,Failures,ChecksumFailures,Deferred,Retirements,StorageBytes,ColdReadBytes,UploadedBytes,ReadOperations,Preemptions; public int CoarseReadyCount,DetailReadyCount,MaxPumpUs; + public int LastPumpUs {get;private set;} public long CoarseLatencyUs,DetailLatencyUs; public bool OwnsStaging=>read!=null||copy!=null||store!=null; public bool HasTransfer=>upload!=null||OwnsStaging; @@ -86,6 +100,13 @@ public void BeginFrame(int frame,int completedFrame) } public void Demand(StreamedMaterialTexture binding,TextureFootprint footprint,ResourcePriority priority=ResourcePriority.Visible)=>Demands.Demand(binding,footprint,priority); public void DemandKnownLod(StreamedMaterialTexture binding,int logicalLevel,ResourcePriority priority=ResourcePriority.Visible)=>Demands.DemandKnownLod(binding,logicalLevel,priority); + public bool Prefetch(StreamedMaterialTexture binding,TexturePrefetchHint hint,TextureWorkBudget budget,int reserveBytes=32768) + { + int free=Math.Max(0,Policy.BudgetBytes-PhysicalBytes-(externalBytes?.Invoke()??0)); + var family=binding.Family;int bytes=catalog.Get(family.GetLevel(family.CoarseIndex).RecordId).Length; + if(!TexturePrefetchPolicy.Likely(hint,budget,free,bytes,reserveBytes))return false; + Demands.DemandKnownLod(binding,family.GetLevel(family.CoarseIndex).LogicalLevel,ResourcePriority.Prefetch);return true; + } public bool CoarseReady(int family)=>active[family]!=null; public bool DetailReady(int family)=>active[family]!=null&&resident[family]==Demands.TargetIndex(family); public int ResidentIndex(int family)=>resident[family]; @@ -221,8 +242,13 @@ public void Pump(int uploadBudgetBytes=32768,int cpuBudgetUs=1000,bool allowStar if(uploadBudgetBytes<0||cpuBudgetUs<0)throw new ArgumentOutOfRangeException("uploadBudgetBytes"); long start=Clock.Timestamp; try{PumpWork(uploadBudgetBytes,cpuBudgetUs,allowStart,start);} - finally{int cost=(int)(Clock.Timestamp-start);if(cost>MaxPumpUs)MaxPumpUs=cost;PeakBytes=Math.Max(PeakBytes,PhysicalBytes+(externalBytes?.Invoke()??0));} + finally{ + if(trimDisabledCold&&Cold.Bytes>0)Cold.TrimExcept(slot<0?0:Policy.Id(slot)); + int cost=(int)(Clock.Timestamp-start);LastPumpUs=cost;if(cost>MaxPumpUs)MaxPumpUs=cost;PeakBytes=Math.Max(PeakBytes,PhysicalBytes+(externalBytes?.Invoke()??0)); + } } + public void Pump(TextureWorkBudget budget,bool allowStart=true) + {if(budget.CpuMicroseconds<1)throw new ArgumentException("Use a constructed work budget.");Pump(budget.UploadBytes,budget.CpuMicroseconds,allowStart);} private void PumpWork(int budget,int cpuBudget,bool allowStart,long start) { Collect();if(ownPolicy){PrepareDemand();Policy.CancelUnneeded();} @@ -263,7 +289,7 @@ private void PumpWork(int budget,int cpuBudget,bool allowStart,long start) } } if(!Policy.Complete(slot,tag,true,Frame)){Discard(false);return;} - if(!Policy.CanPublish(slot,record.Length)){Policy.Defer(slot,Frame);upload!.Dispose();upload=null;cancellation!.Dispose();cancellation=null;slot=-1;Deferred++;return;} + if(!Policy.CanPublish(slot,record.Length)){Policy.Defer(slot,Frame);upload!.Dispose();upload=null;cancellation!.Dispose();cancellation=null;slot=-1;if(StrictReleaseColdCopies)Cold.Drop(record.Id);Deferred++;return;} if(!upload!.TryPublish(out var texture))throw new InvalidOperationException("Validated texture did not publish."); if(active[familyId]!=null)Retire(familyId); active[familyId]=texture;resident[familyId]=levelIndex;activeSlots[familyId]=slot;generations[familyId]++;lastUse[familyId]=default; diff --git a/src/Hypercube.GameCube/WorldStreamer.cs b/src/Hypercube.GameCube/WorldStreamer.cs index 6331261..cc7ec9d 100644 --- a/src/Hypercube.GameCube/WorldStreamer.cs +++ b/src/Hypercube.GameCube/WorldStreamer.cs @@ -35,6 +35,14 @@ public sealed class WorldStreamer : IDisposable private bool stopping; public int Frame,CompletedFrame,Group=1,Format=14,ReadBatchBytes=32768,MaxBatchRecords=4; public bool UseAram,InjectFaults,AdaptiveFallback,CoarseFirst,TexturesOnly,ProfileReads; + private int pendingColdRetention=-1; + private bool strictCold; + public void SetColdRetention(TextureColdRetention retention) + { + if(retentionTextureColdRetention.StrictRelease)throw new ArgumentOutOfRangeException("retention"); + if(retention!=TextureColdRetention.Disabled&&!Dolphin.Memory.Aram.IsAvailable)throw new InvalidOperationException("Initialize ARAM before enabling a cold tier."); + pendingColdRetention=(int)retention; + } private long readAt; private DemandPlan? demandPlan; private bool recordDemandPlan; @@ -144,7 +152,7 @@ private bool Release(int slot,ResourcePriority incoming) if(meshes[slot]!=null){meshes[slot]!.Dispose();meshes[slot]=null;otherBytes-=owned[slot];} if(collisionBuffers[slot]!=null) {collisionBuffers[slot]!.Dispose();collisionBuffers[slot]=null;otherBytes-=owned[slot];Cells.CollisionMissing(r.Cell);} - owned[slot]=0;Policy.Evict(slot,CompletedFrame,incoming);return true; + owned[slot]=0;Policy.Evict(slot,CompletedFrame,incoming);if(strictCold)Cold.Drop(r.Id);return true; } private bool ReclaimForTextureLod(ResourcePriority priority) { @@ -259,6 +267,13 @@ public void Pump() } private void PumpWork(long pumpAt) { + // Do not trim a legacy coalesced ARAM batch while it still has future + // per-record fetches to issue. Family transfers protect their own ID. + if(count==0&&pendingColdRetention>=0) + { + var retention=(TextureColdRetention)pendingColdRetention;UseAram=retention!=TextureColdRetention.Disabled;strictCold=retention==TextureColdRetention.StrictRelease; + if(TextureLods!=null)TextureLods.SetColdRetention(retention);else if(!UseAram)Cold.Trim();pendingColdRetention=-1; + } int publishedBefore=Publications; TextureLods?.PrepareDemand(); Policy.CancelUnneeded(); diff --git a/tests/MipTailRuntime/MipTailRuntime.csproj b/tests/MipTailRuntime/MipTailRuntime.csproj new file mode 100644 index 0000000..86817ed --- /dev/null +++ b/tests/MipTailRuntime/MipTailRuntime.csproj @@ -0,0 +1 @@ +Exe diff --git a/tests/MipTailRuntime/Program.cs b/tests/MipTailRuntime/Program.cs new file mode 100644 index 0000000..7c8bc76 --- /dev/null +++ b/tests/MipTailRuntime/Program.cs @@ -0,0 +1,34 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Dolphin.Graphics; +using Dolphin.Storage; +using Hypercube; +using Hypercube.GameCube; +using Scheduler=Dolphin.Threading.Tasks.Scheduler; + +internal static class Program +{ + private static void Check(bool v){if(!v)throw new Exception("Resident mip tail assertion.");} + private static void Drain(Task task){for(int i=0;i<10000&&!task.IsCompleted;i++)Scheduler.Pump(1);Check(task.IsCompleted);task.GetAwaiter().GetResult();} + static int Main() + { + Check(Storage.Mount(StorageDevice.SdSerialPort2)); + byte[] checksum=File.ReadAllBytes("sd:/tail.crc");uint crc=Binary.U32(checksum,0); + using var staging=NativeBuffer.Allocate(4096);var device=GraphicsDevice.Default;var renderer=new Renderer();var camera=new Camera();var transform=new Transform(); + VertexPositionNormalColorTexture V(float x,float y,float u,float v)=>new VertexPositionNormalColorTexture(new Vector3(x,y,0),new Vector3(0,0,1),Color32.White,new Vector2(u,v)); + using var mesh=Mesh.Create(new[]{V(-1,1,0,0),V(1,1,1,0),V(1,-1,1,1),V(-1,1,0,0),V(1,-1,1,1),V(-1,-1,0,1)}); + int before=device.NativeBytes;var loading=ResidentMipTail.LoadAsync("sd:/tail.dndtex",crc,staging);Drain(loading);using var tail=loading.Result; + Check(tail.Info.Levels==6&&tail.Texture.MipLevels==6&&tail.ActiveBytes==tail.Texture.NativeBytes&&device.NativeBytes-before==tail.PhysicalBytes); + var material=new Material();tail.Apply(material);Check(material.Sampler.MinFilter==TextureMinFilter.LinearMipmapLinear&&material.Sampler.MaxLod==5); + var bad=ResidentMipTail.LoadAsync("sd:/tail.dndtex",crc^1,staging); + try{Drain(bad);return 2;}catch(InvalidDataException){} + Check(tail.ActiveBytes>0&&device.NativeBytes-before==tail.PhysicalBytes); + using(var cancel=new CancellationTokenSource()) + {var task=ResidentMipTail.LoadAsync("sd:/tail.dndtex",crc,staging,cancellationToken:cancel.Token);cancel.Cancel();try{Drain(task);return 3;}catch(OperationCanceledException){}} + device.BeginFrame(Color32.Black);renderer.Begin3D(camera);renderer.DrawMesh(mesh,material,transform);renderer.End3D(); + tail.RetireAfter(device.InsertFence());Check(tail.ActiveBytes==0&&tail.RetiredBytes>0&&!tail.TryCollect());device.EndFrame(); + Check(tail.TryCollect()&&tail.PhysicalBytes==0&&device.NativeBytes==before);return 0; + } +} diff --git a/tests/StreamedModelRuntime/Program.cs b/tests/StreamedModelRuntime/Program.cs new file mode 100644 index 0000000..6823ec6 --- /dev/null +++ b/tests/StreamedModelRuntime/Program.cs @@ -0,0 +1,43 @@ +using System; +using Dolphin.Graphics; +using Dolphin.Storage; +using Hypercube; +using Hypercube.GameCube; +using Scheduler=Dolphin.Threading.Tasks.Scheduler; + +internal static class Program +{ + private static void Check(bool v){if(!v)throw new Exception("Streamed model binding assertion.");} + static int Main() + { + Check(Storage.Mount(StorageDevice.SdSerialPort2)); + using var source=new PackedAssetSource("sd:/models.hcp");using var stream=new TextureLodStreamer(source.Catalog,source.Assets); + using var materials=ModelMaterials.Create0(stream,TextureQualityPolicy.FullQuality); + Check(materials.Count==3&&source.Catalog.FamilyCount==1); + Check(materials.GetPart(1).Primary!.Family==materials.GetPart(2).Secondary!.Family); + Check(materials.GetPart(1).Template.Effect!=materials.GetPart(2).Template.Effect); + var loading=Model.LoadAsync("sd:/parts.dndmodel");for(int i=0;i<10000&&!loading.IsCompleted;i++)Scheduler.Pump(1); + using var model=loading.GetAwaiter().GetResult();using var instance=model.CreateInstance();model.Dispose(); + var renderer=new Renderer();var camera=new Camera();camera.LookAt(new Vector3(0,0,5),Vector3.Zero,new Vector3(0,1,0));var transform=new Transform();var device=GraphicsDevice.Default; + bool sawCoarse=false,detail=false; + var controller=new TextureBudgetController(); + for(int frame=0;frame<3000;frame++) + { + Scheduler.Pump(16);stream.BeginFrame(frame,Math.Max(0,frame-1)); + for(int part=1;part<3;part++)materials.DemandPart(part,1,5,1,60,480,new TextureFootprint(256,256)); + var budget=controller.Next(1000,stream.LastPumpUs,0); + stream.Prefetch(materials.GetPart(1).Primary!,new TexturePrefetchHint(.9f,.3f,.8f,100),budget); + stream.Pump(budget);var resolved=stream.Resolve(materials.GetPart(1).Primary!); + if(resolved.LogicalLevel==4)sawCoarse=true; + detail=stream.DetailReady(1); + device.BeginFrame(Color32.Black);renderer.Begin3D(camera); + materials.DrawPart(renderer,instance,transform,1,frame);materials.DrawPart(renderer,instance,transform,2,frame); + renderer.End3D();stream.EndFrame(device.InsertFence());device.EndFrame(); + if(detail)break; + } + Check(sawCoarse&&detail&&stream.Publications==2); + stream.SetColdRetention(TextureColdRetention.Disabled); + for(int n=0;n<10000&&!stream.Stop();n++){Scheduler.Pump(16);device.BeginFrame(Color32.Black);device.EndFrame();} + Check(!stream.HasTransfer);stream.Dispose();Check(stream.PhysicalBytes==0);return 0; + } +} diff --git a/tests/StreamedModelRuntime/StreamedModelRuntime.csproj b/tests/StreamedModelRuntime/StreamedModelRuntime.csproj new file mode 100644 index 0000000..7259f37 --- /dev/null +++ b/tests/StreamedModelRuntime/StreamedModelRuntime.csproj @@ -0,0 +1 @@ +Exe diff --git a/tests/TextureFamilies.Tests/Program.cs b/tests/TextureFamilies.Tests/Program.cs index 5b15c7a..3f41e76 100644 --- a/tests/TextureFamilies.Tests/Program.cs +++ b/tests/TextureFamilies.Tests/Program.cs @@ -78,4 +78,29 @@ void RejectWord(int at,uint value) Check(!scheduler.Complete(upgrade,generation,true,i*30),"Stale generation never publishes"); Check(scheduler.State(slot)==ResourceState.Resident&&scheduler.ResidentBytes==catalog.Get(coarse).Length,"Churn preserves fallback ownership"); } -Console.WriteLine("Texture families: v2/CRC/sparse/pair/gutters/shared demand/density/hysteresis/dwell/caps/fuzz PASS"); +var controller=new TextureBudgetController(); +Check(controller.Next(1000,0,0).UploadBytes==32768,"Budget starts bounded"); +Check(controller.Next(1000,1200,0).UploadBytes==16384,"Slow pump reduces chunk work"); +Check(!controller.Next(1000,100,12000).AllowPrefetch&&controller.ChunkBytes==8192,"Slow storage reduces speculative work"); +Check(controller.Next(1000,100,100,audioStarved:true).UploadBytes==0,"Audio pressure pauses issuance"); +Check(controller.Next(1000,100,100,criticalQueued:1).UploadBytes==0,"Gameplay pressure pauses issuance"); +Check(controller.Next(7999,100,100).UploadBytes==0,"Insufficient CPU headroom pauses issuance"); +TextureWorkBudget budget=default; +for(int i=0;i<32;i++)budget=controller.Next(1000,100,100); +Check(budget.UploadBytes==32768&&budget.AllowPrefetch&&controller.Growths==2,"Stable headroom recovers gradually within cap"); +var hint=new TexturePrefetchHint(.9f,.5f,.8f,128); +Check(TexturePrefetchPolicy.Likely(hint,budget,65536,1024),"Likely near-future visible content may prefetch"); +Check(!TexturePrefetchPolicy.Likely(new TexturePrefetchHint(.1f,.5f,.8f,128),budget,65536,1024),"Unlikely content does not prefetch"); +Check(!TexturePrefetchPolicy.Likely(hint,budget,1024,1024),"Prefetch preserves memory headroom"); +Check(!TexturePrefetchPolicy.Likely(hint,new TextureWorkBudget(0,1),65536,1024),"Paused work cannot speculate"); +var tailHeader=new byte[32]; +void TailWord(int at,int value)=>BitConverter.GetBytes(value).CopyTo(tailHeader,at); +TailWord(0,0x58544e44);TailWord(4,2);TailWord(8,6);TailWord(12,16);TailWord(16,8);TailWord(20,832);TailWord(24,5); +var tailInfo=ResidentMipTailInfo.Read(tailHeader,864);Check(tailInfo.NativeBytes==832&&tailInfo.Levels==5,"Bounded rectangular contiguous tail"); +foreach(int at in new[]{0,4,8,12,16,20,24,28}) +{ + var bad=(byte[])tailHeader.Clone();bad[at]^=128; + try{ResidentMipTailInfo.Read(bad,864);throw new Exception("Invalid mip tail accepted");}catch(IOException){} +} +try{ResidentMipTailInfo.Read(tailHeader,864,maximumNativeBytes:512);throw new Exception("Tail byte ceiling ignored");}catch(IOException){} +Console.WriteLine("Texture families: v2/CRC/sparse/pair/gutters/shared demand/density/hysteresis/dwell/caps/fuzz/budgets/prefetch PASS"); diff --git a/tests/WorldTextureLodRuntime/Program.cs b/tests/WorldTextureLodRuntime/Program.cs index 29e26a0..8aab038 100644 --- a/tests/WorldTextureLodRuntime/Program.cs +++ b/tests/WorldTextureLodRuntime/Program.cs @@ -28,7 +28,8 @@ static int Main() if(lods.UploadBytes>131072){stream.DemandCollision(1);lateRequested=true;} if(lateRequested)lods.Demand(late,new TextureFootprint(128,128)); stream.Pump(); - if(frame==0)Check(stream.Policy.State(stream.Policy.Find(world.Catalog.Collision(0)))==ResourceState.Reading&&!lods.HasTransfer,"Gameplay did not precede missing coarse texture."); + if(frame==0) + {stream.SetColdRetention(TextureColdRetention.StrictRelease);Check(stream.Policy.State(stream.Policy.Find(world.Catalog.Collision(0)))==ResourceState.Reading&&!lods.HasTransfer,"Gameplay did not precede missing coarse texture.");Check(lods.ColdRetention==TextureColdRetention.ReversalCache,"Cold policy changed before the outstanding batch drained.");} if(lods.UploadBytes>131072&&stream.Ready(world.Catalog.Collision(1)))preempted=true; if(lods.CoarseReady(late.Family.Id)&&!lods.DetailReady(binding.Family.Id))lateReady=true; if(lods.DetailReady(binding.Family.Id)) @@ -36,7 +37,7 @@ static int Main() Check(preempted,"Chunked promotion blocked gameplay/collision preemption."); Check(lateReady,"A late visible coarse family was blocked behind a detached upgrade."); Check(stream.PhysicalBytes<=stream.Policy.BudgetBytes&&world.NativeBytes>0,"World/staging/permanent residency not counted."); - Check(stream.NormalizedTextureUvs,"Normalized UV opt-in lost.");stopping=true; + Check(lods.ColdRetention==TextureColdRetention.StrictRelease,"Deferred shared cold policy was never applied.");Check(stream.NormalizedTextureUvs,"Normalized UV opt-in lost.");stopping=true; } } else done=stream.Stop(); diff --git a/tests/model_sidecar_fixtures.py b/tests/model_sidecar_fixtures.py new file mode 100644 index 0000000..b57a859 --- /dev/null +++ b/tests/model_sidecar_fixtures.py @@ -0,0 +1,31 @@ +"""Independent DNDM fixture for the sidecar/pack/generated-binding contract.""" +import hashlib +import json +from pathlib import Path +import struct +from progressive_fixtures import dnlt + +def fnv(data): + value=2166136261 + for b in data:value=((value^b)*16777619)&0xffffffff + return value + +def build(root): + root=Path(root);root.mkdir(parents=True,exist_ok=True) + texture=root/'shared.dnlod';dnlt(texture,16,8,list(range(5)),1);digest=hashlib.sha256(texture.read_bytes()).hexdigest() + node=struct.pack('>i64s10f',-1,b'Root',0,0,0,0,0,0,1,2,3,1) + vertices=b'' + for x,y,u,v in ((0,0,0,0),(1,0,1,0),(0,1,0,1)): + vertices+=struct.pack('>12f4i4f',x,y,0,0,0,1,u,v,1,1,1,1,0,0,0,0,1,0,0,0) + mesh=struct.pack('>5i6f',0,-1,1,3,3,0,0,0,1,1,0)+vertices+struct.pack('>3I',0,1,2) + material=struct.pack('>4f2if3i',1,1,1,1,-1,2,.5,0,-1,0) + sections=[struct.pack('>I',1)+node,struct.pack('>I',2)+mesh+mesh[:8]+struct.pack('>i',2)+mesh[12:],struct.pack('>I',3)+material*3,bytes(4),bytes(4),bytes(4)] + data=bytearray(160);offset=160 + for i,s in enumerate(sections): + struct.pack_into('>4I',data,16+i*16,i+1,offset,len(s),fnv(s));data.extend(s);data.extend(bytes((-len(data))%32));offset=len(data) + struct.pack_into('>4I',data,0,0x444e444d,2,len(data),6) + model=root/'parts.dndmodel';model.write_bytes(data) + textures=[dict(textureIndex=i,family='textures/'+digest,file=texture.name,sha256=digest,width=16,height=8,wrapS=i,wrapT=2-i,linear=i==0) for i in range(2)] + materials=[dict(materialIndex=i,primaryTexture=-1 if i==0 else 0,secondaryTexture=-1 if i==0 else 1,combiner=i,color=[1,1,1,1],mode=2,cutoff=.5,doubleSided=True) for i in range(3)] + doc=dict(version=1,model=model.name,modelSha256=hashlib.sha256(data).hexdigest(),embeddedTextureCount=0,textures=textures,materials=materials) + path=root/'parts.dndmodel.textures.json';path.write_text(json.dumps(doc));return path diff --git a/tests/test_progressive_acceptance.py b/tests/test_progressive_acceptance.py new file mode 100644 index 0000000..99c4a8b --- /dev/null +++ b/tests/test_progressive_acceptance.py @@ -0,0 +1,44 @@ +import copy +from pathlib import Path +import sys +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tools')) +import unittest +from progressive_acceptance import CHECKS,STRESS,TIERS,validate_run,hardware_gate,compare +from unittest.mock import patch + +def measured(platform='Dolphin',storage='sd2sp2',strategy='fine-pages',run_id='one'): + # Contract data only: deliberately not saved as measured performance evidence. + memory={k:32 for k in TIERS};memory.update(aramBytes=64,physicalBytes=192) + return dict(schema=1,platform=platform,storage=storage,strategy=strategy,runId=run_id,console='test fixture only', + contentSha256='a'*64,sceneSha256='b'*64,cameraTraceSha256='c'*64,shaderSha256='d'*64,samplerSha256='e'*64,dolSha256='f'*64,packSha256='1'*64,sdkRevision='2'*40, + width=2,height=2,format='RGBA8',texelsPerWorldUnit=128,physicalCeilingBytes=256,captureFrame=120, + checks={k:True for k in CHECKS},stressCases=list(STRESS),frames=[dict(frameUs=16000,audioActive=True,memory=memory)]*300, + finalMemory={k:0 for k in (*TIERS,'aramBytes','physicalBytes')},coarseReadyUs=[100]*4,detailReadyUs=[200]*4,storageBytes=1024,aramReadBytes=256,draws=1000,textureBinds=1000,cancellations=1,checksumFailures=1,pressureDeferrals=1) + +class ProgressiveAcceptance(unittest.TestCase): + def test_accounting_correctness_and_hardware_are_required(self): + self.assertEqual(validate_run(measured())['frame99Us'],16000) + with self.assertRaises(ValueError):validate_run(measured(),hardware=True) + for mutate in (lambda r:r['finalMemory'].update(uploadBytes=32),lambda r:r['checks'].update(cancelNeverPublished=False),lambda r:r.update(physicalCeilingBytes=100),lambda r:r['frames'][0].update(audioActive=False),lambda r:r.update(frames=[]),lambda r:r.update(checksumFailures=0)): + run=copy.deepcopy(measured());mutate(run) + with self.assertRaises(ValueError):validate_run(run) + def test_repeated_sd_and_disc_both_strategies_and_thresholds(self): + runs=[measured('GameCube',storage,strategy,storage+strategy+str(i)) for storage in ('sd2sp2','disc') for strategy in ('fine-pages','cell-atlas') for i in range(3)] + self.assertTrue(hardware_gate(runs,33334,1000,1000)['passed']) + with self.assertRaises(ValueError):hardware_gate(runs[:-1],33334,1000,1000) + with self.assertRaises(ValueError):hardware_gate(runs,15000,1000,1000) + runs[0]['platform']='Dolphin' + with self.assertRaises(ValueError):hardware_gate(runs,33334,1000,1000) + def test_matched_quality_identity_admission(self): + pages=measured();atlas=measured(strategy='cell-atlas') + # RGB decoding is separate from contract validation; no fake report is emitted. + with patch('progressive_acceptance.rgb',return_value=bytes(12)): + pages['physicalCeilingBytes']=512 + with self.assertRaises(ValueError):compare(pages,atlas,'ref','pages','atlas') + pages['physicalCeilingBytes']=256;atlas['captureFrame']=121 + with self.assertRaises(ValueError):compare(pages,atlas,'ref','pages','atlas') + with patch('progressive_acceptance.rgb',side_effect=[bytes(12),bytes([255])*12,bytes(12)]): + atlas['captureFrame']=120 + with self.assertRaises(ValueError):compare(pages,atlas,'ref','pages','atlas') + +if __name__=='__main__':unittest.main() diff --git a/tests/test_progressive_polish.py b/tests/test_progressive_polish.py new file mode 100644 index 0000000..a42adeb --- /dev/null +++ b/tests/test_progressive_polish.py @@ -0,0 +1,73 @@ +import copy +import json +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import unittest +from progressive_fixtures import dnlt +from model_sidecar_fixtures import build +from hypercube.model_bindings import read_sidecar,generate +from hypercube import pack +from build_mip_tail import build as build_tail + +class ProgressivePolish(unittest.TestCase): + def test_model_density_and_shared_content_generated_materials(self): + with tempfile.TemporaryDirectory() as tmp: + path=build(tmp);doc=read_sidecar(path) + self.assertEqual([(m['worldPerU'],m['worldPerV']) for m in doc['materials']],[(0,0),(2,3),(2,3)]) + code=generate([doc]);self.assertIn('TextureAddressMode.Mirror',code);self.assertIn('TextureFilter.Nearest',code) + self.assertIn('stream.Get(',code);self.assertIn(',1,2f,3f)',code);self.assertIn(',2,2f,3f)',code) + root=Path(tmp);script=Path(__file__).resolve().parents[1]/'tools/build_texture_pack.py' + subprocess.run([sys.executable,str(script),'--model-sidecar',str(path),'--model-sidecar',str(path),'--bindings',str(root/'a.cs'),'--output',str(root/'a.hcp')],check=True) + self.assertEqual(pack.inspect(root/'a.hcp'),5) + self.assertEqual(json.loads((root/'a.manifest.json').read_text())['families'][0]['name'],doc['textures'][0]['family']) + def test_model_rejects_wrong_hash_paths_mappings_and_nonfinite(self): + with tempfile.TemporaryDirectory() as tmp: + path=build(tmp);original=json.loads(path.read_text()) + for mutate in (lambda d:d.update(model='../escape'),lambda d:d.update(modelSha256='0'*64),lambda d:d['textures'][0].update(file='/tmp/no'),lambda d:d['textures'][0].update(sha256='0'*64),lambda d:d['materials'][1].update(primaryTexture=99),lambda d:d['materials'][1].update(color=[float('nan')]*4),lambda d:d['materials'][1].update(combiner=0)): + doc=copy.deepcopy(original);mutate(doc);path.write_text(json.dumps(doc)) + with self.assertRaises(ValueError):read_sidecar(path) + for name in ('a.b','a;','class','a\nB'): + with self.assertRaises(ValueError):generate([],name) + def test_tail_complete_rectangular_crc_and_byte_ceiling(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);source=root/'a.dnlod';output=root/'tail.dndtex';dnlt(source,32,16,list(range(6)),1) + report=build_tail(source,output,1) + self.assertEqual((report['width'],report['height'],report['levels']),(16,8,5)) + self.assertEqual(len(output.read_bytes()),32+report['nativeBytes']) + self.assertEqual(struct.unpack_from('>i + for py in range(useful): + for px in range(useful): + self.assertEqual(pixel(page['levels'][i],4+px,4+py),pixel(atlas['levels'][i],4+(x%2)*useful+px,4+(y%2)*useful+py)) + self.assertEqual(report['strategies'][0]['coarseBytes']>0,True) + with self.assertRaises(ValueError):comparison(root,physical_ceiling=32) + +if __name__=='__main__':unittest.main() diff --git a/tests/texture_lod_host.c b/tests/texture_lod_host.c index 6f4b7bb..afd5718 100644 --- a/tests/texture_lod_host.c +++ b/tests/texture_lod_host.c @@ -18,7 +18,11 @@ int main(void) DndException *e=(DndException*)dnd_exception_object();if(e&&e->message){for(uint32_t i=0;imessage->length;i++)fputc(e->message->chars[i],stderr);fputc('\n',stderr);}return 1; } assert(dnd_draw_total_bytes()==0&&dnd_draw_live()==0&&dnd_fs_open_handles()==0&&dnd_aram_stat(1)==0); -#ifdef HYPERCUBE_WORLD_RUNTIME +#ifdef HYPERCUBE_MODEL_RUNTIME + assert(texture_host_draws>0&&texture_host_publishes==2); +#elif defined(HYPERCUBE_TAIL_RUNTIME) + assert(texture_host_draws>0&&texture_host_waits==3); +#elif defined(HYPERCUBE_WORLD_RUNTIME) assert(texture_host_draws>5&&texture_host_publishes>=2); #else assert(texture_host_draws>100&&texture_host_publishes>=8); diff --git a/tools/build_mip_tail.py b/tools/build_mip_tail.py new file mode 100644 index 0000000..0517605 --- /dev/null +++ b/tools/build_mip_tail.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Author a bounded indivisible DNTX tail from validated independent DNLT levels.""" +import argparse +import hashlib +import json +from pathlib import Path +import struct +from hypercube.progressive import read_dnlt,crc + +def build(source,output,start_level,maximum_native_bytes=65536,maximum_dimension=128): + if not 32<=maximum_native_bytes<=262144 or maximum_dimension not in (2,4,8,16,32,64,128,256):raise ValueError('Tail budget/dimension limit') + family=read_dnlt(source) + if family['flags'] not in (1,3):raise ValueError('A hardware tail requires authored mip levels') + levels=[x for x in family['levels'] if x['logical']>=start_level] + if len(levels)<2 or levels[0]['logical']!=start_level or max(levels[0]['width'],levels[0]['height'])>maximum_dimension:raise ValueError('Tail start or dimension limit') + first=levels[0];w,h=first['width'],first['height'] + if w&(w-1) or h&(h-1):raise ValueError('Hardware tail dimensions must be powers of two') + for i,level in enumerate(levels): + if level['logical']!=start_level+i or level['width']!=max(1,w>>i) or level['height']!=max(1,h>>i) or tuple(level['uv'])!=(1.,1.,0.,0.) or level['fmt']!=first['fmt']:raise ValueError('A hardware tail cannot contain sparse, padded-atlas or mixed-format levels') + if levels[-1]['width']!=1 or levels[-1]['height']!=1:raise ValueError('Tail must end at 1x1') + payload=b''.join(x['payload'] for x in levels) + if len(payload)>maximum_native_bytes:raise ValueError('Entire mip tail exceeds native memory ceiling') + package=struct.pack('<4s7I',b'DNTX',2,first['fmt'],w,h,len(payload),len(levels),0)+payload + output=Path(output);output.parent.mkdir(parents=True,exist_ok=True);output.write_bytes(package) + report=dict(mode='resident-mip-tail',logicalBaseLevel=start_level,width=w,height=h,format=first['fmt'],levels=len(levels),nativeBytes=len(payload),packageCrc32=crc(package),sha256=hashlib.sha256(package).hexdigest(),sourceSha256=hashlib.sha256(Path(source).read_bytes()).hexdigest()) + output.with_suffix(output.suffix+'.manifest.json').write_text(json.dumps(report,indent=2,sort_keys=True)+'\n') + return report +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('source',type=Path);p.add_argument('--output',type=Path,required=True);p.add_argument('--start-level',type=int,required=True);p.add_argument('--maximum-native-bytes',type=int,default=65536);p.add_argument('--maximum-dimension',type=int,default=128);a=p.parse_args() + print(json.dumps(build(a.source,a.output,a.start_level,a.maximum_native_bytes,a.maximum_dimension))) diff --git a/tools/build_texture_comparison.py b/tools/build_texture_comparison.py new file mode 100644 index 0000000..afb47e9 --- /dev/null +++ b/tools/build_texture_comparison.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Matched-content page/atlas assets for the physical-console comparison lane. +Uses the existing validation source image and native encoders. No timing is +simulated. Both packs contain independent levels and identical useful texels. +""" +import argparse +import hashlib +import json +import math +from pathlib import Path +import struct +from hypercube import pack,progressive + +def family(path,group,x,y,fmt): + entries=[] + for lod in (0,1,2): + width,useful,pixels=pack.tile(group,lod,x,y) + payload=pack.rgba8(pixels,width) if fmt==6 else pack.cmpr(pixels,width) + entries.append(dict(lod=lod,width=width,useful=useful,payload=payload)) + count=len(entries);start=progressive.align(64+count*48);table=bytearray(32*count);uv=bytearray();payloads=bytearray() + for i,level in enumerate(entries): + width,useful=level['width'],level['useful'];uv.extend(struct.pack('<4f',useful/width,useful/width,4/width,4/width)) + for i in reversed(range(count)): + e=entries[i];p=e['payload'];struct.pack_into('physical_ceiling:raise ValueError('Ceiling cannot hold coarse plus replacement/retirement/staging headroom') + reports.append(dict(strategy=strategy,pack=file.name,packSha256=hashlib.sha256(file.read_bytes()).hexdigest(),quads=quads,coarseBytes=coarse,minimumPhysicalBytes=minimum)) + report=dict(schema=1,contentSha256=hashlib.sha256(content).hexdigest(),cameraTraceSha256=hashlib.sha256((root/'camera.json').read_bytes()).hexdigest(),format='RGBA8' if fmt==6 else 'CMPR',texelsPerWorldUnit=8,physicalCeilingBytes=physical_ceiling,sourceDimensions=[192,128],levels=[0,1,2],gutter=4,strategies=reports,hardwarePerformanceClaim=False) + (root/'comparison.json').write_text(json.dumps(report,indent=2,sort_keys=True)+'\n');return report +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--output',type=Path,default=Path('out/texture-comparison'));p.add_argument('--physical-ceiling',type=int,default=524288);p.add_argument('--format',choices=['rgba8','cmpr'],default='rgba8');a=p.parse_args();build(a.output,a.physical_ceiling,6 if a.format=='rgba8' else 14) diff --git a/tools/build_texture_pack.py b/tools/build_texture_pack.py index 80e786c..8aa20ec 100644 --- a/tools/build_texture_pack.py +++ b/tools/build_texture_pack.py @@ -5,19 +5,38 @@ import json from pathlib import Path from hypercube import pack +from hypercube.model_bindings import read_sidecar,generate def main(): parser=argparse.ArgumentParser(description=__doc__) - parser.add_argument('--texture',action='append',required=True,metavar='NAME=FILE.dnlod') + parser.add_argument('--texture',action='append',default=[],metavar='NAME=FILE.dnlod') parser.add_argument('--output',type=Path,required=True) parser.add_argument('--manifest',type=Path) + parser.add_argument('--model-sidecar',action='append',type=Path,default=[]) + parser.add_argument('--bindings',type=Path) + parser.add_argument('--class-name',default='ModelMaterials') args=parser.parse_args();builder=pack.Builder();sources=[];inputs=[] + if not args.texture and not args.model_sidecar:parser.error('Supply textures or model sidecars') + if args.model_sidecar and not args.bindings:parser.error('Model sidecars require --bindings') + models=[read_sidecar(path) for path in args.model_sidecar] for value in args.texture: name,separator,path=value.partition('=') if not separator:parser.error('Use --texture NAME=FILE.dnlod') inputs.append((name,path));sources.append(dict(name=name,path=path,sha256=hashlib.sha256(Path(path).read_bytes()).hexdigest())) + seen={name:hashlib.sha256(Path(path).read_bytes()).hexdigest() for name,path in inputs} + for doc in models: + for texture in doc['textures']: + name=texture['family'];digest=texture['sha256'] + if name in seen: + if seen[name]!=digest:raise ValueError('Conflicting shared model family') + continue + seen[name]=digest;inputs.append((name,texture['path']));sources.append(dict(name=name,path=texture['path'],sha256=digest)) + bindings=generate(models,args.class_name) if models else None builder.import_textures(inputs) report=builder.write(args.output);pack.inspect(args.output) + if bindings is not None: + args.bindings.parent.mkdir(parents=True,exist_ok=True);args.bindings.write_text(bindings) + report['models']=[dict(model=d['model'],sha256=d['modelSha256'],factory='Create'+str(i)) for i,d in enumerate(models)] report['sources']=sources;report['families']=[] for family in builder.families: report['families'].append({**{k:v for k,v in family.items() if k!='levels'},'levels':[{k:v for k,v in level.items() if k!='payload'} for level in family['levels']]}) diff --git a/tools/hypercube/model_bindings.py b/tools/hypercube/model_bindings.py new file mode 100644 index 0000000..3328365 --- /dev/null +++ b/tools/hypercube/model_bindings.py @@ -0,0 +1,158 @@ +"""Validate SDK model sidecars and generate allocation-free material wiring. +Density is measured from rest-pose geometry, after authored UV transforms. +Animated deformation requires an application-provided conservative multiplier. +""" +import hashlib +import json +import math +from pathlib import Path +import re +import struct + +from .progressive import read_dnlt + +def _inside(root, name): + if not isinstance(name,str) or not name or Path(name).is_absolute():raise ValueError('Relative model asset path required') + path=(root/name).resolve() + if not path.is_relative_to(root):raise ValueError('Model asset escapes sidecar directory') + return path + +def _digest(path, expected): + if not isinstance(expected,str) or not re.fullmatch('[0-9a-f]{64}',expected) or hashlib.sha256(path.read_bytes()).hexdigest()!=expected:raise ValueError('Model asset SHA256 mismatch') + +def _integer(v,low,high): + if type(v)!=int or not low<=v<=high:raise ValueError('Model integer range') + return v + +def _number(v,low,high): + if type(v) not in (int,float) or not math.isfinite(v) or not low<=v<=high:raise ValueError('Model numeric range') + return v + +def model_density(path): + data=path.read_bytes() + if len(data)<112 or data[:4]!=b'DNDM' or struct.unpack_from('>I',data,4)[0] not in (1,2):raise ValueError('DNDM version') + + def fnv(raw): + value=2166136261 + for b in raw:value=((value^b)*16777619)&0xffffffff + return value + if len(data)>16*1024*1024 or struct.unpack_from('>2I',data,8)!=(len(data),6):raise ValueError('DNDM size/sections') + sections=[];end=160 + for i in range(6): + kind,offset,size,checksum=struct.unpack_from('>4I',data,16+i*16) + if kind!=i+1 or offsetlen(data) or fnv(data[offset:offset+size])!=checksum:raise ValueError('DNDM section') + sections.append(memoryview(data)[offset:offset+size]);end=offset+size + if struct.unpack_from('>I',sections[3])[0]!=0:raise ValueError('Streamed model contains embedded textures') + material_count=struct.unpack_from('>I',sections[2])[0] + _integer(material_count,1,4096) + nodes=sections[0];count=struct.unpack_from('>I',nodes)[0] + if not 1<=count<=1024 or len(nodes)!=4+108*count:raise ValueError('DNDM node table') + transforms=[None]*count + def global_matrix(index,parents=()): + if index in parents or len(parents)>=128 or not 0<=indexi',nodes,at)[0] + tx,ty,tz,x,y,z,w,sx,sy,sz=struct.unpack_from('>10f',nodes,at+68) + if not all(math.isfinite(v) for v in (tx,ty,tz,x,y,z,w,sx,sy,sz)):raise ValueError('DNDM transform') + m=((1-2*(y*y+z*z))*sx,2*(x*y-z*w)*sy,2*(x*z+y*w)*sz,tx, + 2*(x*y+z*w)*sx,(1-2*(x*x+z*z))*sy,2*(y*z-x*w)*sz,ty, + 2*(x*z-y*w)*sx,2*(y*z+x*w)*sy,(1-2*(x*x+y*y))*sz,tz, + 0,0,0,1) + if parent!=-1: + a=global_matrix(parent,parents+(index,));m=tuple(sum(a[r*4+k]*m[k*4+c] for k in range(4)) for r in range(4) for c in range(4)) + transforms[index]=m;return m + for i in range(count):global_matrix(i) + skins=[];raw_skins=sections[4];skin_at=4 + skin_count=struct.unpack_from('>I',raw_skins)[0];_integer(skin_count,0,4096) + for _ in range(skin_count): + joints=struct.unpack_from('>I',raw_skins,skin_at)[0];skin_at+=4;_integer(joints,1,256);palette=[] + for _ in range(joints): + joint=struct.unpack_from('>I',raw_skins,skin_at)[0];inverse=struct.unpack_from('>16f',raw_skins,skin_at+4);skin_at+=68 + if joint>=count or not all(math.isfinite(v) for v in inverse):raise ValueError('DNDM skin palette') + # DNDM inverse bind matrices are column-major; use row-major math here. + inverse=tuple(inverse[c*4+r] for r in range(4) for c in range(4));a=transforms[joint] + palette.append(tuple(sum(a[r*4+k]*inverse[k*4+c] for k in range(4)) for r in range(4) for c in range(4))) + skins.append(palette) + if skin_at!=len(raw_skins):raise ValueError('DNDM skin bytes') + result=[[0.,0.] for _ in range(material_count)];meshes=sections[1];at=4 + mesh_count=struct.unpack_from('>I',meshes)[0];_integer(mesh_count,1,4096) + for _ in range(mesh_count): + node,skin,material,vertices,indices=struct.unpack_from('>5i',meshes,at) + if not 0<=nodelen(meshes):raise ValueError('DNDM vertex bounds') + if not -1<=skin3f',meshes,base);u,v=struct.unpack_from('>2f',meshes,base+24) + values=(x,y,z,u,v) + if not all(math.isfinite(t) for t in values):raise ValueError('DNDM vertex') + def transformed(matrix):return tuple(sum(matrix[r*4+k]*values[k] for k in range(3))+matrix[r*4+3] for r in range(3)) + if skin>=0: + joints=struct.unpack_from('>4i',meshes,base+48);weights=struct.unpack_from('>4f',meshes,base+64) + if any(not math.isfinite(weight) or weight<0 for weight in weights) or abs(sum(weights)-1)>.001:raise ValueError('DNDM skin weights') + positions=[] + for joint,weight in zip(joints,weights): + if not 0<=jointI',meshes,start+vertices*80+4*(j+k))[0]) for k in range(3)] + e1=[va[1][0][k]-va[0][0][k] for k in range(3)];e2=[va[2][0][k]-va[0][0][k] for k in range(3)] + du1,dv1=(va[1][1][k]-va[0][1][k] for k in range(2));du2,dv2=(va[2][1][k]-va[0][1][k] for k in range(2));det=du1*dv2-du2*dv1 + if abs(det)>1e-12: + u=math.sqrt(sum(((e1[k]*dv2-e2[k]*dv1)/det)**2 for k in range(3))) + v=math.sqrt(sum(((e2[k]*du1-e1[k]*du2)/det)**2 for k in range(3))) + result[material]=[max(result[material][0],u),max(result[material][1],v)] + at=end + if at!=len(meshes):raise ValueError('DNDM trailing mesh bytes') + return result + +def read_sidecar(path): + path=Path(path).resolve();root=path.parent;doc=json.loads(path.read_text()) + if doc.get('version')!=1 or doc.get('embeddedTextureCount')!=0:raise ValueError('Model sidecar version') + model=_inside(root,doc['model']);_digest(model,doc['modelSha256']);density=model_density(model) + textures=doc['textures'];materials=doc['materials'] + if not isinstance(textures,list) or len(textures)>4096 or not isinstance(materials,list) or len(materials)!=len(density):raise ValueError('Model sidecar counts') + for i,t in enumerate(textures): + if t['textureIndex']!=i:raise ValueError('Model texture index') + source=_inside(root,t['file']);_digest(source,t['sha256']) + if t['family']!='textures/'+t['sha256']:raise ValueError('Content-addressed family') + d=read_dnlt(source) + if (t['width'],t['height'])!=(d['width'],d['height']):raise ValueError('Model texture dimensions') + _integer(t['wrapS'],0,2);_integer(t['wrapT'],0,2) + if type(t['linear'])!=bool:raise ValueError('Model filter') + t['path']=str(source) + for i,m in enumerate(materials): + if m['materialIndex']!=i:raise ValueError('Model material index') + for key in ('primaryTexture','secondaryTexture'):_integer(m[key],-1,len(textures)-1) + _integer(m['combiner'],0,2);_integer(m['mode'],0,7);_number(m['cutoff'],0,1) + if m['mode']&5==5 or len(m['color'])!=4 or type(m['doubleSided'])!=bool:raise ValueError('Model material flags') + for c in m['color']:_number(c,0,1) + if (m['secondaryTexture']>=0)!=(m['combiner']!=0) or m['combiner'] and m['primaryTexture']<0:raise ValueError('Model combiner textures') + m['worldPerU'],m['worldPerV']=density[i] + return doc + +def generate(models, class_name='ModelMaterials'): + if not re.fullmatch('[A-Za-z_][A-Za-z0-9_]*',class_name) or class_name in ('abstract as base bool break byte case catch char checked class const continue decimal default delegate do double else enum event explicit extern false finally fixed float for foreach goto if implicit in int interface internal is lock long namespace new null object operator out override params private protected public readonly ref return sbyte sealed short sizeof stackalloc static string struct switch this throw true try typeof uint ulong unchecked unsafe ushort using virtual void volatile while record required file'.split()):raise ValueError('C# class identifier') + lines=['// Generated from verified SDK model sidecars. Do not edit.', 'using Dolphin.Graphics;', 'using Hypercube;', 'using Hypercube.GameCube;', 'public static class '+class_name, '{'] + def num(v):return format(v,'.9g')+'f' + for index,doc in enumerate(models): + lines += [' // '+json.dumps(doc['model']),f' public static StreamedModelMaterials Create{index}(TextureLodStreamer stream, TextureQualityPolicy policy=TextureQualityPolicy.DistanceMip)', ' {',f' var result=new StreamedModelMaterials(stream,{len(doc["materials"])});', ' try', ' {'] + for m in doc['materials']: + refs=[] + for key in ('primaryTexture','secondaryTexture'): + t=doc['textures'][m[key]] if m[key]>=0 else None + refs.append('stream.Get('+json.dumps(t['family'])+',policy)' if t else 'null') + color=','.join(str(int(c*255+.5)) for c in m['color']) + lines += [f' var m{m["materialIndex"]}=new Material{{Color=new Color32({color}),Lighting=LightingMode.{"Unlit" if m["mode"]&2 else "Diffuse"},Blend=BlendMode.{"Alpha" if m["mode"]&4 else "Opaque"},DepthWrite={str(not bool(m["mode"]&4)).lower()},Cull=CullMode.{"None" if m["doubleSided"] else "Back"},AlphaCutoff={num(m["cutoff"] if m["mode"]&1 else 0)}}};'] + for key,slot in (('primaryTexture','Sampler'),('secondaryTexture','SecondarySampler')): + if m[key]>=0: + t=doc['textures'][m[key]];wrap=['Clamp','Repeat','Mirror'];filt='Linear' if t['linear'] else 'Nearest' + lines += [f' m{m["materialIndex"]}.{slot}=new SamplerState(TextureAddressMode.{wrap[t["wrapS"]]},TextureAddressMode.{wrap[t["wrapT"]]},TextureFilter.{filt});'] + lines += [f' result.SetPart({m["materialIndex"]},m{m["materialIndex"]},{refs[0]},{refs[1]},{m["combiner"]},{num(m["worldPerU"])},{num(m["worldPerV"])});'] + lines += [' return result;', ' }', ' catch {result.Dispose();throw;}', ' }'] + return '\n'.join(lines+['}','']) diff --git a/tools/progressive_acceptance.py b/tools/progressive_acceptance.py new file mode 100644 index 0000000..4d8ca23 --- /dev/null +++ b/tools/progressive_acceptance.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Evidence admission for PR11: matched-quality strategy comparisons and hardware gates. +Input records are measured by the application/capture operator, never synthesized +by this tool. See docs/progressive-production.md for the versioned run contract. +""" +import argparse +import hashlib +import json +import math +from pathlib import Path +import re +import subprocess + +TIERS=('activeBytes','uploadBytes','readBufferBytes','retiredBytes','poolBytes','externalBytes') +MATCHED=('contentSha256','sceneSha256','cameraTraceSha256','shaderSha256','samplerSha256','width','height','format','texelsPerWorldUnit','physicalCeilingBytes','sdkRevision','platform','storage') +CHECKS=('coarseBeforeDetail','noHoles','noStaleDraws','noUvJumps','noCorruptPixels','corruptNeverPublished','cancelNeverPublished','pairReachedHigh','noPromotionGpuWait','noSynchronousWholeTextureRead') +STRESS=('camera-turn','multiple-objects','budget-pressure','camera-reversal','corrupt-read','cancel-read') + +def require(value,message): + if not value:raise ValueError(message) + +def numeric(value,minimum=0): + require(type(value) in (int,float) and math.isfinite(value) and value>=minimum,'Invalid measured number') + return value + +def digest(value):require(isinstance(value,str) and re.fullmatch('[0-9a-f]{64}',value),'Missing SHA256 identity') + +def percentile(values,p): + values=sorted(values);return values[max(0,math.ceil(len(values)*p/100)-1)] + +def validate_run(run,hardware=False): + require(run['schema']==1 and run['platform'] in ('Dolphin','GameCube') and run['storage'] in ('sd2sp2','disc'),'Run source/version') + require(run['strategy'] in ('fine-pages','cell-atlas'),'Strategy') + for field in MATCHED: + if field.endswith('Sha256'):digest(run[field]) + for field in ('dolSha256','packSha256'):digest(run[field]) + require(re.fullmatch('[0-9a-f]{40}',run['sdkRevision']) is not None,'SDK revision') + for field in ('width','height','texelsPerWorldUnit','physicalCeilingBytes'):numeric(run[field],1) + require(run['format'] in ('RGBA8','CMPR'),'Native format') + if hardware: + require(run['platform']=='GameCube','Emulator timing cannot satisfy hardware acceptance') + require(isinstance(run.get('console'),str) and bool(run['console'].strip()),'Identify physical console and video mode') + require(all(run['checks'].get(k) is True for k in CHECKS),'Missing/failed correctness check') + require(set(STRESS)<=set(run['stressCases']),'Missing stress workload') + frames=run['frames'];require(len(frames)>=300,'At least 300 measured frames required') + times=[];peak=0;aram_peak=0 + for frame in frames: + times.append(numeric(frame['frameUs'],1));require(frame['audioActive'] is True,'Audio stopped during pressure') + memory=frame['memory'];total=sum(numeric(memory[k]) for k in TIERS);aram_peak=max(aram_peak,numeric(memory['aramBytes'])) + require(total==numeric(memory['physicalBytes']) and total<=run['physicalCeilingBytes'],'Physical accounting/ceiling mismatch') + peak=max(peak,total) + require(all(run['finalMemory'].get(k)==0 for k in (*TIERS,'aramBytes','physicalBytes')),'Native/ARAM allocation survived final fence/trim') + for key in ('coarseReadyUs','detailReadyUs'): + require(len(run[key])>=4,'Missing readiness observations') + for value in run[key]:numeric(value,1) + for key in ('storageBytes','aramReadBytes','draws','textureBinds','cancellations','checksumFailures','pressureDeferrals'):numeric(run[key]) + require(run['cancellations']>0 and run['checksumFailures']>0 and run['pressureDeferrals']>0,'Failures/pressure were not exercised') + return dict(frame95Us=percentile(times,95),frame99Us=percentile(times,99),coarseReady95Us=percentile(run['coarseReadyUs'],95),detailReady95Us=percentile(run['detailReadyUs'],95),physicalPeakBytes=peak,aramPeakBytes=aram_peak,frames=len(frames)) + +def rgb(path,width,height): + result=subprocess.run(['ffmpeg','-v','error','-i',str(path),'-frames:v','1','-f','rawvideo','-pix_fmt','rgb24','-'],capture_output=True,check=True) + require(len(result.stdout)==width*height*3,'Capture dimensions disagree');return result.stdout + +def compare(a,b,reference,page_image,atlas_image,max_rmse=2): + aa=validate_run(a,hardware=a.get('platform')=='GameCube');bb=validate_run(b,hardware=b.get('platform')=='GameCube') + require(a['strategy']=='fine-pages' and b['strategy']=='cell-atlas','Order strategies: pages, atlas') + require(all(a[k]==b[k] for k in MATCHED),'Quality, workload, SDK or physical ceiling is not matched') + # All three captures must represent the same deterministic camera sample. + require(a['captureFrame']==b['captureFrame'],'Capture camera frame differs') + ref=rgb(reference,a['width'],a['height']);images=[rgb(path,a['width'],a['height']) for path in (page_image,atlas_image)] + errors=[math.sqrt(sum((x-y)**2 for x,y in zip(ref,pixels))/len(ref)) for pixels in images] + require(all(error<=max_rmse for error in errors),'Screenshot quality differs from the shared full-detail reference') + return dict(schema=1,hardwarePerformanceClaim=a['platform']=='GameCube',matched=True,storage=a['storage'],runIds=[a['runId'],b['runId']],maximumRmse=max_rmse,rmse=dict(pages=errors[0],atlas=errors[1]),pages=aa,atlas=bb,comparison={k:dict(pages=a[k],atlas=b[k]) for k in ('storageBytes','aramReadBytes','draws','textureBinds')},evidenceSha256={str(p):hashlib.sha256(Path(p).read_bytes()).hexdigest() for p in (reference,page_image,atlas_image)}) + +def hardware_gate(runs,maximum_frame99_us,maximum_coarse95_us,maximum_detail95_us): + require(len(runs)>=12,'Require three repeats of both strategies on SD2SP2 and disc') + results=[] + require(len({r['runId'] for r in runs})==len(runs),'Every hardware run needs a unique identity') + for storage in ('sd2sp2','disc'): + for strategy in ('fine-pages','cell-atlas'): + group=[r for r in runs if r['storage']==storage and r['strategy']==strategy] + require(len(group)>=3,'Missing strategy/storage repeats') + require(len({r['runId'] for r in group})==len(group),'Repeated run identity') + first=group[0] + require(all(all(r[k]==first[k] for k in MATCHED) for r in group),'Repeat workload changed') + for storage in ('sd2sp2','disc'): + group=[r for r in runs if r['storage']==storage];first=group[0] + require(all(all(r[k]==first[k] for k in MATCHED) for r in group),'Hardware strategies do not match quality/workload/ceiling') + for run in runs: + result=validate_run(run,hardware=True) + require(result['frame99Us']<=maximum_frame99_us and result['coarseReady95Us']<=maximum_coarse95_us and result['detailReady95Us']<=maximum_detail95_us,'Hardware performance threshold failed') + results.append(dict(runId=run['runId'],storage=run['storage'],strategy=run['strategy'],**result)) + return dict(schema=1,hardwarePerformanceClaim=True,passed=True,thresholds=dict(frame99Us=maximum_frame99_us,coarse95Us=maximum_coarse95_us,detail95Us=maximum_detail95_us),runs=results) + +def main(): + p=argparse.ArgumentParser(description=__doc__);sub=p.add_subparsers(dest='command',required=True) + c=sub.add_parser('compare');c.add_argument('pages',type=Path);c.add_argument('atlas',type=Path) + for flag in ('reference','page-image','atlas-image'):c.add_argument('--'+flag,type=Path,required=True) + c.add_argument('--maximum-rmse',type=float,default=2) + h=sub.add_parser('hardware');h.add_argument('runs',type=Path,nargs='+');h.add_argument('--comparison',type=Path,action='append',required=True,help='Six accepted page/atlas screenshot comparison reports') + for flag in ('frame99-us','coarse95-us','detail95-us'):h.add_argument('--maximum-'+flag,type=int,required=True) + p.add_argument('--output',type=Path,required=True);args=p.parse_args() + if args.command=='compare': + numeric(args.maximum_rmse);result=compare(json.loads(args.pages.read_text()),json.loads(args.atlas.read_text()),args.reference,args.page_image,args.atlas_image,args.maximum_rmse) + else: + for value in (args.maximum_frame99_us,args.maximum_coarse95_us,args.maximum_detail95_us):numeric(value,1) + result=hardware_gate([json.loads(path.read_text()) for path in args.runs],args.maximum_frame99_us,args.maximum_coarse95_us,args.maximum_detail95_us) + require(len(args.comparison)>=6,'Require matched screenshot evidence for each hardware pair') + covered=set();comparisons=[] + for path in args.comparison: + report=json.loads(path.read_text());require(report.get('schema')==1 and report.get('matched') is True and report.get('hardwarePerformanceClaim') is True,'Hardware matched-quality evidence required') + hashes=report.get('runEvidenceSha256',{});require(len(hashes)==2,'Missing comparison input hashes');covered.update(hashes.values());comparisons.append(dict(path=str(path),sha256=hashlib.sha256(path.read_bytes()).hexdigest())) + require(all(hashlib.sha256(path.read_bytes()).hexdigest() in covered for path in args.runs),'A hardware run lacks matched screenshot evidence') + result['comparisons']=comparisons + inputs=[args.pages,args.atlas] if args.command=='compare' else args.runs + result['runEvidenceSha256']={str(path):hashlib.sha256(path.read_bytes()).hexdigest() for path in inputs} + args.output.parent.mkdir(parents=True,exist_ok=True);args.output.write_text(json.dumps(result,indent=2,sort_keys=True)+'\n') +if __name__=='__main__':main() diff --git a/tools/test_progressive_runtime.py b/tools/test_progressive_runtime.py index abd7dac..e356bee 100644 --- a/tools/test_progressive_runtime.py +++ b/tools/test_progressive_runtime.py @@ -20,6 +20,14 @@ def main(projects,configurations): kept=os.environ.get('HYPERCUBE_TEST_WORK') with nullcontext(kept) if kept else tempfile.TemporaryDirectory(prefix='hypercube-lods-') as folder: work=Path(folder);(work/'sd').mkdir(parents=True,exist_ok=True);shutil.copyfile(ROOT/'out/progressive/textures.hcp',work/'sd/world.hcp');shutil.copyfile(ROOT/'out/progressive/world-mixed.hcp',work/'sd/world-mixed.hcp') + sys.path.insert(0,str(ROOT/'tests'));from progressive_fixtures import dnlt + from build_mip_tail import build as build_tail + import struct + dnlt(work/'tail.dnlod',32,16,list(range(6)),1) + tail=build_tail(work/'tail.dnlod',work/'sd/tail.dndtex',0);(work/'sd/tail.crc').write_bytes(struct.pack('data&&t->width==4&&t->height==4);const unsigned char *p=t->data;assert(p[0]==0&&p[1]==0);','assert(t->data&&t->width>0&&t->height>0);') backend=backend.replace('graphics_draws++;','texture_host_draws++;graphics_draws++;').replace('void dnd_draw_backend_wait(void){}','void dnd_draw_backend_wait(void){texture_host_waits++;}').replace('void dnd_draw_backend_publish_texture(void){}','void dnd_draw_backend_publish_texture(void){texture_host_publishes++;}') @@ -33,8 +41,11 @@ def main(projects,configurations): run('dotnet',SDK/'tools/DolphinDotNet.Compiler/bin/Release/net8.0/DolphinDotNet.Compiler.dll','--aot',f'tests/{project}/bin/{config}/net8.0/{project}.dll',generated) dead='-Wl,-dead_strip' if sys.platform=='darwin' else '-Wl,--gc-sections' defines=['-DHYPERCUBE_WORLD_RUNTIME'] if project=='WorldTextureLodRuntime' else [] - run('cc','-std=c11','-O1','-D_DEFAULT_SOURCE','-D_DARWIN_C_SOURCE','-DDND_GRAPHICS_TEST',*defines,'-ffunction-sections','-fdata-sections','-pthread',f'-I{SDK}/include',generated,'tests/texture_lod_host.c',work/'backend.c',*[SDK/f'source/{s}.c' for s in sources],dead,'-lm','-o',exe) + if project=='MipTailRuntime':defines=['-DHYPERCUBE_TAIL_RUNTIME'] + if project=='StreamedModelRuntime':defines=['-DHYPERCUBE_MODEL_RUNTIME'] + project_sources=sources+(['model_source','model_ufbx','model_obj'] if project=='StreamedModelRuntime' else []) + run('cc','-std=c11','-O1','-D_DEFAULT_SOURCE','-D_DARWIN_C_SOURCE','-DDND_GRAPHICS_TEST',*defines,'-ffunction-sections','-fdata-sections','-pthread',f'-I{SDK}/include',generated,'tests/texture_lod_host.c',work/'backend.c',*[SDK/f'source/{s}.c' for s in project_sources],dead,'-lm','-o',exe) run(exe,env=env) if __name__=='__main__': - p=argparse.ArgumentParser(description=__doc__);p.add_argument('--project',choices=['TextureLodRuntime','WorldTextureLodRuntime']);p.add_argument('--configuration',choices=['Debug','Release']);a=p.parse_args() - main([a.project] if a.project else ['TextureLodRuntime','WorldTextureLodRuntime'],[a.configuration] if a.configuration else ['Debug','Release']) + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--project',choices=['TextureLodRuntime','WorldTextureLodRuntime','MipTailRuntime','StreamedModelRuntime']);p.add_argument('--configuration',choices=['Debug','Release']);a=p.parse_args() + main([a.project] if a.project else ['TextureLodRuntime','WorldTextureLodRuntime','MipTailRuntime','StreamedModelRuntime'],[a.configuration] if a.configuration else ['Debug','Release']) From 2d1ebe16dd6c5851af2cc0cf95267c992f8a45ef Mon Sep 17 00:00:00 2001 From: leroy ketelaars Date: Wed, 11 Nov 2026 13:18:52 +0000 Subject: [PATCH 2/7] Require matched camera windows and verify capture admission --- docs/progressive-production.md | 7 ++++--- tests/test_progressive_acceptance.py | 11 +++++++++-- tools/hypercube/model_bindings.py | 2 +- tools/progressive_acceptance.py | 16 ++++++++++------ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/docs/progressive-production.md b/docs/progressive-production.md index 2f386f3..38fbe39 100644 --- a/docs/progressive-production.md +++ b/docs/progressive-production.md @@ -133,7 +133,7 @@ python3 tools/progressive_acceptance.py --output comparison.json compare \ Admission rejects differing content, camera, shader, sampler, viewport, source texel density, native format, SDK or physical ceiling. Both captures must be the -same camera frame and meet the reference RMSE threshold (RGB 0..255 units). +same camera frame over the same measured frame window and meet the reference RMSE threshold (RGB 0..255 units). Report P95/P99 frame times, P95 coarse/detail readiness, physical/ARAM peaks, storage/ARAM traffic, draws and texture binds together. A lower-detail or higher-memory result cannot establish a faster strategy. Repeat runs and @@ -148,9 +148,10 @@ A run JSON contains `schema:1`, a unique `runId`, `platform` (`Dolphin` or hardware. Include 64-character lowercase SHA256 fields `contentSha256`, `sceneSha256`, `cameraTraceSha256`, `shaderSha256`, `samplerSha256`, `dolSha256`, `packSha256`; a 40-character `sdkRevision`; `width`, `height`, `format` -(`RGBA8`/`CMPR`), `texelsPerWorldUnit`, `physicalCeilingBytes`, and `captureFrame`. +(`RGBA8`/`CMPR`), `texelsPerWorldUnit`, `physicalCeilingBytes`, `measuredStartFrame`, and `captureFrame`. -Record at least 300 measured `frames`, each with `frameUs`, `audioActive:true`, +Record at least 300 measured `frames`, each with consecutive camera `frame` indices starting at `measuredStartFrame`, +`frameUs`, `audioActive:true`, and a `memory` object. Its fields are `activeBytes`, `uploadBytes` (the entire private target allocation, not just bytes already copied), `readBufferBytes`, `retiredBytes`, `poolBytes`, `externalBytes`, `physicalBytes` (sum of those six), diff --git a/tests/test_progressive_acceptance.py b/tests/test_progressive_acceptance.py index 99c4a8b..7a673d0 100644 --- a/tests/test_progressive_acceptance.py +++ b/tests/test_progressive_acceptance.py @@ -3,6 +3,7 @@ import sys sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tools')) import unittest +import tempfile from progressive_acceptance import CHECKS,STRESS,TIERS,validate_run,hardware_gate,compare from unittest.mock import patch @@ -11,8 +12,8 @@ def measured(platform='Dolphin',storage='sd2sp2',strategy='fine-pages',run_id='o memory={k:32 for k in TIERS};memory.update(aramBytes=64,physicalBytes=192) return dict(schema=1,platform=platform,storage=storage,strategy=strategy,runId=run_id,console='test fixture only', contentSha256='a'*64,sceneSha256='b'*64,cameraTraceSha256='c'*64,shaderSha256='d'*64,samplerSha256='e'*64,dolSha256='f'*64,packSha256='1'*64,sdkRevision='2'*40, - width=2,height=2,format='RGBA8',texelsPerWorldUnit=128,physicalCeilingBytes=256,captureFrame=120, - checks={k:True for k in CHECKS},stressCases=list(STRESS),frames=[dict(frameUs=16000,audioActive=True,memory=memory)]*300, + width=2,height=2,format='RGBA8',texelsPerWorldUnit=128,physicalCeilingBytes=256,captureFrame=120,measuredStartFrame=0, + checks={k:True for k in CHECKS},stressCases=list(STRESS),frames=[dict(frame=i,frameUs=16000,audioActive=True,memory=memory) for i in range(300)], finalMemory={k:0 for k in (*TIERS,'aramBytes','physicalBytes')},coarseReadyUs=[100]*4,detailReadyUs=[200]*4,storageBytes=1024,aramReadBytes=256,draws=1000,textureBinds=1000,cancellations=1,checksumFailures=1,pressureDeferrals=1) class ProgressiveAcceptance(unittest.TestCase): @@ -29,6 +30,12 @@ def test_repeated_sd_and_disc_both_strategies_and_thresholds(self): with self.assertRaises(ValueError):hardware_gate(runs,15000,1000,1000) runs[0]['platform']='Dolphin' with self.assertRaises(ValueError):hardware_gate(runs,33334,1000,1000) + def test_actual_capture_decode_and_quality_report(self): + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'capture.ppm';path.write_bytes(b'P6\n2 2\n255\n'+bytes([30,60,90])*4) + report=compare(measured(),measured(strategy='cell-atlas'),path,path,path) + self.assertFalse(report['hardwarePerformanceClaim']);self.assertEqual(report['rmse'],dict(pages=0,atlas=0)) + def test_matched_quality_identity_admission(self): pages=measured();atlas=measured(strategy='cell-atlas') # RGB decoding is separate from contract validation; no fake report is emitted. diff --git a/tools/hypercube/model_bindings.py b/tools/hypercube/model_bindings.py index 3328365..fec5e31 100644 --- a/tools/hypercube/model_bindings.py +++ b/tools/hypercube/model_bindings.py @@ -31,7 +31,7 @@ def _number(v,low,high): def model_density(path): data=path.read_bytes() if len(data)<112 or data[:4]!=b'DNDM' or struct.unpack_from('>I',data,4)[0] not in (1,2):raise ValueError('DNDM version') - + def fnv(raw): value=2166136261 for b in raw:value=((value^b)*16777619)&0xffffffff diff --git a/tools/progressive_acceptance.py b/tools/progressive_acceptance.py index 4d8ca23..73095db 100644 --- a/tools/progressive_acceptance.py +++ b/tools/progressive_acceptance.py @@ -12,7 +12,7 @@ import subprocess TIERS=('activeBytes','uploadBytes','readBufferBytes','retiredBytes','poolBytes','externalBytes') -MATCHED=('contentSha256','sceneSha256','cameraTraceSha256','shaderSha256','samplerSha256','width','height','format','texelsPerWorldUnit','physicalCeilingBytes','sdkRevision','platform','storage') +MATCHED=('contentSha256','sceneSha256','cameraTraceSha256','shaderSha256','samplerSha256','width','height','format','texelsPerWorldUnit','physicalCeilingBytes','measuredStartFrame','sdkRevision','platform','storage') CHECKS=('coarseBeforeDetail','noHoles','noStaleDraws','noUvJumps','noCorruptPixels','corruptNeverPublished','cancelNeverPublished','pairReachedHigh','noPromotionGpuWait','noSynchronousWholeTextureRead') STRESS=('camera-turn','multiple-objects','budget-pressure','camera-reversal','corrupt-read','cancel-read') @@ -42,9 +42,13 @@ def validate_run(run,hardware=False): require(isinstance(run.get('console'),str) and bool(run['console'].strip()),'Identify physical console and video mode') require(all(run['checks'].get(k) is True for k in CHECKS),'Missing/failed correctness check') require(set(STRESS)<=set(run['stressCases']),'Missing stress workload') + require(type(run['runId'])==str and bool(run['runId'].strip()),'Unique run identity required') + start=run['measuredStartFrame'];require(type(start)==int and start>=0,'Measured camera window start') frames=run['frames'];require(len(frames)>=300,'At least 300 measured frames required') + require(type(run['captureFrame'])==int and start<=run['captureFrame']=3,'Missing strategy/storage repeats') require(len({r['runId'] for r in group})==len(group),'Repeated run identity') first=group[0] - require(all(all(r[k]==first[k] for k in MATCHED) for r in group),'Repeat workload changed') + require(all(all(r[k]==first[k] for k in MATCHED) and len(r['frames'])==len(first['frames']) for r in group),'Repeat workload changed') for storage in ('sd2sp2','disc'): group=[r for r in runs if r['storage']==storage];first=group[0] - require(all(all(r[k]==first[k] for k in MATCHED) for r in group),'Hardware strategies do not match quality/workload/ceiling') + require(all(all(r[k]==first[k] for k in MATCHED) and len(r['frames'])==len(first['frames']) for r in group),'Hardware strategies do not match quality/workload/ceiling') for run in runs: result=validate_run(run,hardware=True) require(result['frame99Us']<=maximum_frame99_us and result['coarseReady95Us']<=maximum_coarse95_us and result['detailReady95Us']<=maximum_detail95_us,'Hardware performance threshold failed') From 3d3dd19344bb93b67f5f170aadb5bc1805415970 Mon Sep 17 00:00:00 2001 From: leroy ketelaars Date: Wed, 11 Nov 2026 13:20:02 +0000 Subject: [PATCH 3/7] Record final progressive texture Dolphin regression evidence --- .../progressive/phase-five-coarse.png | Bin 0 -> 7643 bytes .../progressive/phase-five-dolphin.json | 57 ++++++++++++++++++ .../progressive/phase-five-fine-dual-uv.png | Bin 0 -> 7776 bytes 3 files changed, 57 insertions(+) create mode 100644 docs/measurements/progressive/phase-five-coarse.png create mode 100644 docs/measurements/progressive/phase-five-dolphin.json create mode 100644 docs/measurements/progressive/phase-five-fine-dual-uv.png diff --git a/docs/measurements/progressive/phase-five-coarse.png b/docs/measurements/progressive/phase-five-coarse.png new file mode 100644 index 0000000000000000000000000000000000000000..d1132d69a9402b0634b1b97fecd99229d97148b6 GIT binary patch literal 7643 zcmeHMX;@R|woYOr3M$%ijmi-FaIRvd0<~>KhFI~4Q;CWd5h0>h^eQAWNJz+Ftp7%fWVDnvxaB!rnd0HPu?WCBAZkckj7?|n8$&hy}N?~mJi&X3dkm>=1DedBuH zwf48Z?_24+LxLCDuds)~U<wr}}nAE`ho#HHOzCZjcB3r}_}p10+fQeh8pEw(fn!_7IfYeD|5T|=It^z;k4 z^SXWD%-X(>eEM8mE(U&tA6ZO1@MB07YUD_K+OOyLv*#P1N2d*Mu>16E1Cz8Z-hT7; zm0uh&?>%|@rpI?lJv`gg3jsk-2y{dT;ga;0yI38yKI&xgmxpW_^!cz8r_-4vEL)dF&5k9!Coe8{Xb-`UT>3aCO`Z-5_c znR-3aKfmL;S^vlj7k7*x~b)NUJJaYzY=j zWoJizi_$zgdY}$e1i8{1d&azg8pPZi=PHq@NmS=dPl>=FRCn0gvF4&s0{TT>JC+hJ zzkAA_q*LNPmkf`wT*@b3E=9{rx_p2UI58#iFdXc~S}JOjsrQb54cWgW}!M?LpP-zU#o&IN*?sptG#B9g<2_z-S zHRe9>Hl>(pe(A6-ps7b*X}q(az8Kc|W<4@>0u{d$JlHux_*_(D6ozt$3guYYS%vFT zddjPr4oSFJp(CQPH zS$_-XBowoEpAd5It0LE8C-13`o7TdfWyq0p>E6In1&*V#dNIgW!?$Ba5)$k zLbJi^st+~nq&JyfVn60dao>aq>l2whoL=7;=K%f281vB{x$l{$L7$~O#__=vQO6?= zc9S93fdpI&U5yU?s+b|CBi}0eGN#@H6>nXt8Ny=@#^}v#T2@c!SL-A>DC<33tp0o7 z+D;?ee*}=Pk(dbV`JDNi#33N1VP}3~g0oGWjRi(oCTj_{CQ)Wmwi{Ua;`+ql=Rq6w zd3XN{Fzl~D1g6xtbELm96BTz8Czf8~bq;mAcSVwuLuiV7tqU4{{Q(g*IPC6VGy(}> zsGm}`ZMUJOUSW}S(>*g{>r9^(nm;S#A>BC@q{-hB@?=vu1*`sV?cL}Sf(o|At#>W@nV@8MTzAoTS&Oegh6`&WUDBewa zY@B{6T07@vNNka*dZgOR_FGSdl~5dtetuPu{xGp{KHc>Mt&klkOY-W!I+hJ)e1{7- zq1~txpdOmuuD2;~B&j>ECZ{yL_=vRC3GDs(YcXv?x7}yFu{%Q#(J; zNcu$kH}@!y@8Y6I&F&7Qra6B4+XI^2m_zb=Uac1qQ3e%Dr?-16im@j}QFQq0gAb%r zJ%rdAs1LHX*}VD)J_7jiMX%J^|C>CSh|tXisG$g9aiNFM0I zpTQR2V8B;yg3sB+K+I*pkPBdN`(-f1MHVm^24l2c?4oKd%M zJ~Vho-tXpN<^BdQ!9zP?+UnYJ6u$p_+^u-0wUTVsz`3{<=kjv-b?D`9MSb3zFhiNI z$!C_m=x&L>ym3Rl&&}PuB@teWj&A(RfvmlD?U+?gKVgoHwMOp>{^ZFXg~ZtQ<7VyF z)BFvWMo%2^-QnsYzPj5(lvavj`nZ1U8C-IH*!dItR_Aj4kauxWy38Yqn0j>cO_F<6 zluKLUm*poW$w#z@^(tY z962g>nfrF@XTkQ%?5E?RAn?7q<+JPXD`raq?0YTIv+L+z%&rso%$AZb!s~u{{0rhD z0*ayI)$d98MRO9}uec)@AfaIRn*Sc4!r2n#)Uq4SCDVWw6I$A6U(TL@?mb(=y1l2^ zdrE`Rvqu3)vnA1cfUvH!M*-g>2~3#X59osS6MKF4Htb$0O3MtGMc^J<^n)4Jf3-Fp zo6&xY>B6JGH=l6Gxjkb*8EVze8S6dv+Lb%Q*Jb=w&l!3=1m58@R{Pf)xVLfL5Oi^H zZu%|&^1VNp_8&A}^2c?(561i78n663mKJAMw}kL+`T6PJG*7&g-$3V}cgsrGAa>&% zM(jW5MEdR|7JP;8rK0Lz=S`KlS|&tc^$!fPqrFr0iQ{)Ec&kFBjf^6vs2$qts|+TN zlB$X^1RC^wR5tbe3hkes5HMN8nrYZj#R9`mD%T6j5rDDQb%&M1>Nlag z)c$o5p<8p$h7=c#7fKyw6Pfnm|b^OWX740FK1>=nDk1uTmT zbcy3(4^jMi8tHw@pKQf~TuZl(o6S+uh*X2LLuglhR-6)R0F=UN(QAJ4eF~9a z=*;IG~UO19k-d8KPPuF@h>y3ef^IjuFsUEt$?;A*TvjJKjX>e#6r{O9(9^CTa`P^yYDqV;Pog=n01WpvaF9NS5(~#pXvvYgzs)jvuX!7?++cJ7ebU(9ZP%OaCh{i}$KDq9W~s!1iL94y-3agLZ!l|zT*WzD;qP!^q zB^FfI^S}(eSYYa~sZ%K+DsCLKcB#e23pN5h2F%A81VDy`B6vtwaxHkX2td)k2V3z% zy0ph8n4td0-@g1a1#gEvK_y`v%w$=<L!I>h;poggyc+J4Lhlie0^MdaUdQQ z)zIAZ6aY^H_a7^gVmk+HH5hFt2 ztY{}qg*TJ}@sLavF(|hgUEcP_@MTC|iWAYYr-8q>A!ZrU>Ujwv^0%+Cc}SwxNY-CM z(3Wn66Wox4G$$g*Nwa(_9P3cTz+@xhu=X`!C;$;)80;KuM}wyF4m{_eO=V=2CdI#M zEfh`EdXGdNpthG#sY^MM9yi@*rS%R2o{%ak2zP7V)JPQ2mU%Q^5LIawYOxVxZ|-II z@ns3oPr3KaS)mrQXH?_efRXPGQc+dmJZV({eG8NWgRrA)+cEi^6riG?{`{4U57dQr z>guyig`xWGyM`e;TAl!-Ayj`rTV3r-^~3d9w_g~>wiC6vq~_RK2QNtm5VZ7NfACYz z6N)W~T_C5Tb$)DxBRmsS89(tFGNe8k9oA+G_#LAIDUXLBmPJ-!Qg1qDFgLWJsEmGn z?dh)Yk|g4F1NEu8KD#4Sks#|>JL2Cf2L9O7)jH76tuyDueT~;yV=BR0rSH+2bG410 zY)#Lzy(l!3tiuFCg8A(A2c{hRc6yd{=<^wu{zuH}3#LKY)8Bo0{ois2zk37;2p_$+ W`2{JwBy&1NyZz50Tgo;c{^75yF&uRO literal 0 HcmV?d00001 diff --git a/docs/measurements/progressive/phase-five-dolphin.json b/docs/measurements/progressive/phase-five-dolphin.json new file mode 100644 index 0000000..830de16 --- /dev/null +++ b/docs/measurements/progressive/phase-five-dolphin.json @@ -0,0 +1,57 @@ +{ + "artifacts": { + "out/native-progressive-Release/Game.dol": "f5f93cb2a37a779517e769ffc09dbeefa648adcc3519abdc3622f30d14fc452a", + "out/progressive/textures.hcp": "1ad90536c358b56ed773c9f822231fc0026b8765608460a723ba404d131bfcad" + }, + "audio": [ + { + "activeWindows": 254, + "file": "ID-Game_2026-11-11_13-17-52_dspdump1.wav", + "longestQuietMs": 0, + "seconds": 25.36424378668665 + } + ], + "coarsePixelFrames": 5, + "fineDualUvPixelFrames": 10, + "hardwarePerformanceClaim": false, + "implementationCommit": "a122bd5b7aad86ba211e6b3dbcd05e389ad6b732", + "last": { + "active32": 0, + "aram32": 0, + "aramHits": 25, + "audioActive": 1, + "cancelled": 26, + "checksum": 17963, + "coarseMask": 15, + "coarseMeanMs": 170, + "coarseReady": 5, + "coldKiB": 14, + "crcFailures": 1, + "deferred": 2, + "detailMask": 0, + "detailMeanMs": 210, + "detailReady": 7, + "failures": 2, + "frame": 1507, + "frame95ms": 17, + "frame99ms": 17, + "magic": 18516, + "maxPumpUs": 2382, + "peakKiB": 652, + "phase": 10, + "poolBytes": 0, + "publications": 8, + "readBufferKiB": 0, + "retired32": 0, + "retirements": 8, + "stale": 25, + "storageKiB": 1190, + "upload32": 0, + "uploadedKiB": 1171 + }, + "pixelPatches": 12, + "platform": "Dolphin", + "sampledFrames": 252, + "sdkRevision": "415476f42eddba4c3a01fc81902da60b06280486", + "validationScope": "Dolphin correctness and ownership regression; no physical-console performance or measured page/atlas strategy comparison claim." +} diff --git a/docs/measurements/progressive/phase-five-fine-dual-uv.png b/docs/measurements/progressive/phase-five-fine-dual-uv.png new file mode 100644 index 0000000000000000000000000000000000000000..d8e410f57ae83eafb41e84b27cf268c3ca491aef GIT binary patch literal 7776 zcmeHMdr*^C+J6&76jbb1r4|Go+oA={e$~pIcdd5OVxbjVph$uj#0nv31i~$Es`a*v zq`CsjHCno4sod0ta7}_JR3bz-KqP@ATtXlr5OXEB_x(cMopyGy&R`y?W4wey?K0068$cwqlw06Y)a(WC+7t7_=k>J-E1$f#c|`)_ zq{F!T8+Q_)O!;iy5*yJ;18(@buMOOdem%Bw738qL@dUm*@loLy9r2^@4;sba^NC`* zZrQHEQNm}@NQ768A);?^0%uMH5>7U(JT3S{l<@6;BAp4XK) zMT((*orynKeZ}8wZL3h|YO9s=23j^;lj#I-^J*!C)KsiVDUyiihHc}V^GJ=H_C?mz z%FLj_=$-y~$y&F$!bpI!(c5Lg2AQNhWE_LbF5f8?5&cQ9?np7lKORD4&Tmjwm-|Y4 z=%UldPzda0ecn%nRg!`^++T|^xE#x5kGC_4jQF@L$jr5}qp|mqTGj}p>vSO*C#wji>6~j2 z320}nYeX;KEIo;mz&B>YshDa?r0q4Gsz!XQvr6fc0PIPN;V{7)KUnU37smuYWFXIA zs;>(C0qOdjf3Z(6#udzM2sVG+0C_7>zu@@>Lg_5q(U8R zxpXCeeF1B78n%V(a@~;|o=JMrqAb^M@O)MiIW|ByNS#8A{9LsU@X3UljhaU`&dz~X zSysIT(RB91o9-9D-h(1JFwySVj z8&z71gxY$;ciYYvL8X~27|^eAS&({{Q{e;Zr;&TX0r;Z#At$ zsO)c;)?T?p&TrT1o&J&;!)eQj7!_BaIf$Bvou5r;^9_b~qFh`^NP~ODd-2G0UM=>^ z2{fxiDrYff5?kVvOmSF^)v`{VT0WQdG0?gR$>BIjzSH7$WBG10GlS=mH`dwPX2a0f z4@6L-R(h)d&QRQek)czLxfU`G+jWV4Uia=Od6|n~7}PzS@XQTlHUu*NgnD5F9NUo) zRwD%$3dvt$Z_LjJ`~SI$U6HMFUNG3^m^0Sxn>s#XdUZZbH@J4Fo1JL=Fp^Sgy5(iv zhdQkR9D}tFFBZCDNjj!eLmfssb3wZ}RRyEJg`Tb8f64)jXWJ=sWRC=1eWx1#LmM&x z!xxJ^f+2*qwS5%Q(GZsnYHJ)zyKyckyP@91%+TS^jnom-+UiRtJV*AXz|CP`MKrnB z6}|)Pjx|&NYgV#WaoSzwDO7=^Clf>%|J>c`+g{Fg#Qi)c!GLZ?Kwuc{dv|OszuHRPeIHRc-dE@t71^>m*b`Syz%R9V3Nbj-`|oPHVu33 zeK+hLn$_BJJWmkq=a;fsd{qZnz;C2yoSTC=8qvX~4|7t>U)eRu9VO4~0;xNxiE$j7K+rx>zGmXx!s#$&rQ^=H;SAUlA{jN-^pMUsqH z%)T`Bihp~+bz#zN0I+8H;$6U!-Iu(5DaMyF+p8+-KQ^B)mZGGI#ZvUK`#+YVaId_w z@fzX&SEQ}IKUb~}oFD(G%J+Z0L$H^|KD3;NLYD55H_fK_qaV9lF@CT}@@8bHagxi1 zC-ZS6%_U5T$y!XyHTXn^YTYBn);{YuZsD?RpK~zc=2LjYtWrvBV|sT?Onm!mGG;df zvS1_OD8d&BAfrG4>5T$N!xOOe6UrW>!b@}fr&Gg};tvd)Iw)FV{?z3Z1Sn=`K%oyT?9ItK+%wtl)w_vL5dc0%`y;*?$U`_?-22}ktnm$$`mB#C&? z+MmhRSIuKb8(T?RQ!6D&wUbq0JhEo!fl-oW=rC4xSZqlINIk0r^9<8fNzs0_Zx>c2 zpd^h=US#=H9QX&?5xJC*MPpErQDXO$10kXN;tWArT706j!^EsdLcRl(h1~1Dp-+E>V=Q9rZF7kB4G+&Rboc)V&|Ewe7KUjzw4W;YO; zUFZ}yJe`$053Nat^eL|o2xQ8H){3Bb{>B*A2R%)b&gE@F;r19poTZ$J)aSg|Ax?9P zKw*Rtc_uO9F$61y-}y@eSI~JrD6_SC5z7MnllA_Y3`SI=xXBPeK;n&!aqVL}M-*<; z*Zt}axCu0+!jmCQdwaifoe^#UG8HJ=GC(44*~4 z6P0y3RNhK*fJTv+%4Pc(CMfntjep*PAEP6`AgPy?n z0&+lD2^lOGPT|te*zC^h@7R92?@BNiPi6EK(mY=d1O(5P8gYJnltHnscTk|$TPXj2MignA(CFaKhK7B40-)7 z#A+%m21|uSxDqaiylr6ym}MS`W?R~sGH9?yKk>@t-}ln^GDC*;M3b{0Mak8T$zsSh zlgx`xf=yZ<478iEgZ*AH1cw_4L>)JvS7#X;*n@l@rR;g-{mG$|v8g$=_@y8lOckGwF*Z zIX>wCZKU&B-r_s)!4D($*ME?3;lBaA Cwnvr# literal 0 HcmV?d00001 From 415db10cd67cd8e944efb38b94b580788b9e5764 Mon Sep 17 00:00:00 2001 From: leroy ketelaars Date: Wed, 11 Nov 2026 15:05:50 +0000 Subject: [PATCH 4/7] Install FFmpeg before host capture acceptance tests --- .github/workflows/progressive-textures.yml | 4 ++++ .github/workflows/validation.yml | 4 ++++ README.md | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/progressive-textures.yml b/.github/workflows/progressive-textures.yml index 3abc9cd..c2b5985 100644 --- a/.github/workflows/progressive-textures.yml +++ b/.github/workflows/progressive-textures.yml @@ -21,6 +21,10 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' + - name: Install capture decoder for host tests + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg - run: python3 tests/progressive_fixtures.py - run: python3 -m unittest discover -s tests -p 'test_progressive*.py' - run: dotnet run --project tests/TextureFamilies.Tests -c Debug diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 87e33bc..d34cfd3 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -31,6 +31,10 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' + - name: Install capture decoder for host tests + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg - name: Test ordered mesh pass contracts run: dotnet run --project tests/OrderedMeshPasses.Tests -c Release - name: Build and test packed assets, scheduler and GameCube executable diff --git a/README.md b/README.md index 7a2f7da..151ec88 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Open **Hypercube.sln** for the engine, runtime adapter, host checks and console ## Build -Requirements: .NET 8, Python 3.10+, Git and Docker. +Requirements: .NET 8, Python 3.10+, Git, Docker and FFmpeg (for capture-decoding tests and comparison tools). ```sh mkdir -p vendor From d3f6668a461387e5a2de291852cd497a1a03acb5 Mon Sep 17 00:00:00 2001 From: leroy ketelaars Date: Wed, 11 Nov 2026 15:16:52 +0000 Subject: [PATCH 5/7] Use pinned FFmpeg wheel without Ubuntu mirror dependencies --- .github/actions/setup-capture-decoder/action.yml | 13 +++++++++++++ .github/workflows/progressive-textures.yml | 4 +--- .github/workflows/validation.yml | 4 +--- 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 .github/actions/setup-capture-decoder/action.yml diff --git a/.github/actions/setup-capture-decoder/action.yml b/.github/actions/setup-capture-decoder/action.yml new file mode 100644 index 0000000..a80c0aa --- /dev/null +++ b/.github/actions/setup-capture-decoder/action.yml @@ -0,0 +1,13 @@ +name: Set up capture decoder +description: Install a pinned FFmpeg binary for host capture tests without system package mirrors. +runs: + using: composite + steps: + - name: Install FFmpeg binary wheel + shell: bash + run: | + python3 -m pip install --disable-pip-version-check --only-binary=:all: --no-deps imageio-ffmpeg==0.6.0 + decoder_bin="$RUNNER_TEMP/hypercube-ffmpeg-bin" + mkdir -p "$decoder_bin" + ln -sf "$(python3 -c 'import imageio_ffmpeg; print(imageio_ffmpeg.get_ffmpeg_exe())')" "$decoder_bin/ffmpeg" + echo "$decoder_bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/progressive-textures.yml b/.github/workflows/progressive-textures.yml index c2b5985..cd2599e 100644 --- a/.github/workflows/progressive-textures.yml +++ b/.github/workflows/progressive-textures.yml @@ -22,9 +22,7 @@ jobs: with: python-version: '3.12' - name: Install capture decoder for host tests - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends ffmpeg + uses: ./.github/actions/setup-capture-decoder - run: python3 tests/progressive_fixtures.py - run: python3 -m unittest discover -s tests -p 'test_progressive*.py' - run: dotnet run --project tests/TextureFamilies.Tests -c Debug diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index d34cfd3..5350e3f 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -32,9 +32,7 @@ jobs: with: python-version: '3.12' - name: Install capture decoder for host tests - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends ffmpeg + uses: ./.github/actions/setup-capture-decoder - name: Test ordered mesh pass contracts run: dotnet run --project tests/OrderedMeshPasses.Tests -c Release - name: Build and test packed assets, scheduler and GameCube executable From 7f7f6ad187c498959a9c381ae1da7d09e1c1bfec Mon Sep 17 00:00:00 2001 From: leroy ketelaars Date: Wed, 11 Nov 2026 16:12:24 +0000 Subject: [PATCH 6/7] Run Release comparison suites in independent CI jobs --- .github/workflows/validation.yml | 119 ++++++++++++++++++------------- 1 file changed, 69 insertions(+), 50 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 5350e3f..c8d647c 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -11,7 +11,7 @@ permissions: jobs: validation: runs-on: ubuntu-24.04 - timeout-minutes: 50 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -63,63 +63,82 @@ jobs: --mount "type=bind,src=$PWD,dst=/work" -w /work hypercube-dolphin python3 tools/dolphin_validation.py out/native-${{ matrix.configuration }}/Game.dol --disc out/assets/world.iso --logs out/dolphin-${{ matrix.configuration }} --seconds 240 - - name: Build performance comparison executable - if: matrix.configuration == 'Release' - env: - PERFORMANCE_SUITE: '1' - run: tools/build_validation.sh - - name: Run nine performance comparisons and progress checks - if: matrix.configuration == 'Release' - run: >- - docker run --rm --init --shm-size=512m --user "$(id -u):$(id -g)" - --mount "type=bind,src=$PWD,dst=/work" -w /work hypercube-dolphin - python3 tools/dolphin_validation.py out/native-performance-Release/Game.dol - --suite performance --disc out/assets/world.iso - --logs out/dolphin-performance-Release --seconds 240 - - name: Build frame-phase profiling executable - if: matrix.configuration == 'Release' - env: - PROFILING_SUITE: '1' - run: tools/build_validation.sh - - name: Run sixteen engine/SDK throughput comparisons - if: matrix.configuration == 'Release' - run: >- - docker run --rm --init --shm-size=512m --user "$(id -u):$(id -g)" - --mount "type=bind,src=$PWD,dst=/work" -w /work hypercube-dolphin - python3 tools/dolphin_validation.py out/native-profiling-Release/Game.dol - --suite profiling --disc out/assets/world.iso - --logs out/dolphin-profiling-Release --seconds 360 - - name: Build SDK follow-up profiling executable - if: matrix.configuration == 'Release' - env: - SDK_FOLLOWUP_SUITE: '1' - run: tools/build_validation.sh - - name: Run sixteen SDK wait and direct-read comparisons - if: matrix.configuration == 'Release' - run: >- - docker run --rm --init --shm-size=512m --user "$(id -u):$(id -g)" - --mount "type=bind,src=$PWD,dst=/work" -w /work hypercube-dolphin - python3 tools/dolphin_validation.py out/native-sdk-followup-Release/Game.dol - --suite sdk-followup --disc out/assets/world.iso - --logs out/dolphin-sdk-followup-Release --seconds 360 - - name: Build texture follow-up profiling executable - if: matrix.configuration == 'Release' + - name: Upload executable, assets, measurements and rendered evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: hypercube-validation-${{ matrix.configuration }} + path: | + out/native-*/Game.dol + out/native-*/Game.elf + out/assets/ + out/dolphin-*/*.csv + out/dolphin-*/*.md + out/dolphin-*/*.json + out/dolphin-*/*.png + out/dolphin-*/*.log + out/dolphin-*/user/Logs/ + out/dolphin-*/user/Dump/Audio/ + retention-days: 14 + + release-comparisons: + name: release comparisons (${{ matrix.suite }}) + runs-on: ubuntu-24.04 + # Each capture and its decoding have their own budget. Running all four + # serially alongside baseline validation exceeded the former 50-minute job. + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - suite: performance + build_flag: PERFORMANCE_SUITE + seconds: 240 + - suite: profiling + build_flag: PROFILING_SUITE + seconds: 360 + - suite: sdk-followup + build_flag: SDK_FOLLOWUP_SUITE + seconds: 360 + - suite: texture-followup + build_flag: TEXTURE_FOLLOWUP_SUITE + seconds: 480 + steps: + - uses: actions/checkout@v4 + - id: sdk + run: echo "revision=$(cat .sdk-revision)" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@v4 + with: + repository: capz/DolphinDotNet + ref: ${{ steps.sdk.outputs.revision }} + path: vendor/sdk + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install capture decoder for host tests + uses: ./.github/actions/setup-capture-decoder + - name: Build and test Release comparison executable env: - TEXTURE_FOLLOWUP_SUITE: '1' - run: tools/build_validation.sh - - name: Run sixteen clear, wait and aligned-read comparisons - if: matrix.configuration == 'Release' + CONFIGURATION: Release + FIXTURE_RESOLUTION: '48' + run: env "${{ matrix.build_flag }}=1" tools/build_validation.sh + - name: Build Dolphin validation container + run: docker build -t hypercube-dolphin -f tools/Dockerfile.dolphin . + - name: Run comparison and progress checks run: >- docker run --rm --init --shm-size=512m --user "$(id -u):$(id -g)" --mount "type=bind,src=$PWD,dst=/work" -w /work hypercube-dolphin - python3 tools/dolphin_validation.py out/native-texture-followup-Release/Game.dol - --suite texture-followup --disc out/assets/world.iso - --logs out/dolphin-texture-followup-Release --seconds 480 + python3 tools/dolphin_validation.py out/native-${{ matrix.suite }}-Release/Game.dol + --suite ${{ matrix.suite }} --disc out/assets/world.iso + --logs out/dolphin-${{ matrix.suite }}-Release --seconds ${{ matrix.seconds }} - name: Upload executable, assets, measurements and rendered evidence if: always() uses: actions/upload-artifact@v4 with: - name: hypercube-validation-${{ matrix.configuration }} + name: hypercube-${{ matrix.suite }}-Release path: | out/native-*/Game.dol out/native-*/Game.elf From ee79c5f3af505777d9ae76e932ce2e418a7fc358 Mon Sep 17 00:00:00 2001 From: leroy ketelaars Date: Wed, 11 Nov 2026 16:17:24 +0000 Subject: [PATCH 7/7] Keep Release validation gated on every comparison suite --- .github/workflows/validation.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index c8d647c..aaee11b 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -9,7 +9,8 @@ concurrency: permissions: contents: read jobs: - validation: + baseline-validation: + name: ${{ matrix.configuration == 'Debug' && 'validation (Debug)' || 'baseline validation (Release)' }} runs-on: ubuntu-24.04 timeout-minutes: 30 strategy: @@ -151,3 +152,18 @@ jobs: out/dolphin-*/user/Logs/ out/dolphin-*/user/Dump/Audio/ retention-days: 14 + + release-validation: + name: validation (Release) + needs: [baseline-validation, release-comparisons] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require baseline validation and every Release comparison + env: + BASELINE_RESULT: ${{ needs.baseline-validation.result }} + COMPARISONS_RESULT: ${{ needs.release-comparisons.result }} + run: | + test "$BASELINE_RESULT" = success + test "$COMPARISONS_RESULT" = success