diff --git a/docs/measurements/memory/world-payload.json b/docs/measurements/memory/world-payload.json new file mode 100644 index 0000000..cb9c4a0 --- /dev/null +++ b/docs/measurements/memory/world-payload.json @@ -0,0 +1,57 @@ +{ + "caveats": [ + "Selected array element payload only; object field changes, headers and allocator rounding excluded.", + "Managed heap reservation is unchanged; these are occupancy reductions.", + "Cold ARAM, permanent geometry, active/idle texture pairs, framebuffers and SDK heaps are separate.", + "Unsupported large records retain their existing demand-time rejection." + ], + "hardwareMeasurement": false, + "idleAramMetadata": { + "disabled": { + "after": 0, + "before": 5200, + "saved": 5200 + }, + "enabledAfterFirstStore": { + "after": 5200, + "before": 5200, + "saved": 0 + }, + "targetPointerBytes": 4 + }, + "largestSupportedStreamRecord": 73984, + "maximumReadBatchBytes": 65536, + "nativeStagingBytes": { + "after": 73984, + "before": 131072, + "saved": 57088 + }, + "records": 649, + "retainedManagedPayload": { + "catalogIds": { + "after": 2862, + "before": 5724, + "saved": 2862 + }, + "saved": 4162, + "schedulerLookup": { + "after": 1300, + "before": 2600, + "saved": 1300 + } + }, + "schema": 1, + "scope": "validated-v1-world-payload-accounting", + "sourceSha256": "0a782f8137b426528a185cb8bbb5a67ae50d8fc5448ef6a88d56016daec49024", + "transientCatalogTailPayload": { + "after": 192, + "before": 384, + "saved": 192 + }, + "unchanged": { + "metadataInputBytes": 41536, + "parsedRecordPayloadBytes": 36344, + "textureCopiesPerPoolSlot": 2 + }, + "unsupportedStreamRecords": 0 +} diff --git a/docs/memory-and-validation.md b/docs/memory-and-validation.md index 3daef5b..4954beb 100644 --- a/docs/memory-and-validation.md +++ b/docs/memory-and-validation.md @@ -48,3 +48,7 @@ Acceptance requires a representative **playable** 30 FPS original-console test, ## Major risks Texture page splitting can cause excessive draws; fix via hierarchical merging. Atlas overfetch can overload storage; compare equal-budget misses **and** bytes transferred. ARAM promotions can compete with audio; prioritize safely. Small MEM1 budgets constrain whole-game utility; test during animation and gameplay rather than static demos. + +See [the memory reduction audit](memory-reduction-audit.md) for measured host +allocation changes, reproducible world payload accounting, implemented staging +and metadata reductions, and larger opportunities requiring API/hardware work. diff --git a/docs/memory-reduction-audit.md b/docs/memory-reduction-audit.md new file mode 100644 index 0000000..48fc805 --- /dev/null +++ b/docs/memory-reduction-audit.md @@ -0,0 +1,162 @@ +# Memory reduction audit + +Baseline: `b6eee61` (SDK 318). This audit covers current `main`'s v1 packed-world +loader, streaming adapter, catalog and rendering helpers. It does not assume +unmerged progressive-runtime branches are present. The target has 24 MiB MEM1; +managed objects, native assets and framebuffers all compete for that physical +memory. ARAM is a separate DMA-backed tier, not directly GX-addressable memory. + +## Implemented reductions + +The deterministic 48-resolution validation pack has 649 records and a largest +supported streamed record of 73,984 bytes. The new accounting tool checks pack +CRCs/ranges and reports its source SHA256; engine load-time admission still +validates descriptor semantics. The checked-in JSON is **payload accounting**, +not a physical-console trace. + +| Allocation | Before | After | Reduction | +| --- | ---: | ---: | ---: | +| Native streaming source buffer | 131,072 B | 73,984 B | 57,088 B | +| Catalog ID indexes/links, retained managed payload | 5,724 B | 2,862 B | 2,862 B | +| Scheduler ID-to-slot lookup, retained managed payload | 2,600 B | 1,300 B | 1,300 B | +| ARAM metadata while unused, managed payload on 32-bit target | 5,200 B | 0 B | 5,200 B | +| Temporary catalog mesh-tail payload | 384 B | 192 B | 192 B | + +The native staging reduction is **57,088 bytes**. The selected retained arrays +save **9,362 B of element payload** while the cold cache is unused, or 4,162 B +after its first successful cold allocation. These figures exclude object field +changes, object/array headers and allocator rounding; they are not a whole +managed-heap net total. The SDK's reserved managed heap is unchanged: lower live occupancy +reduces pressure but does not automatically return that reservation to the +native allocator. No heap-size default or texture quality is reduced. + +### Bounded native staging + +`StreamingBufferPolicy` reserves the aligned maximum of a configured coalescing +ceiling and every supported streamed record. The default ceiling is 65,536 bytes; +existing validation targets of 32/64 KiB fit. A single larger supported record +still fits even when the requested coalescing target is smaller. Unsupported +records above 131,040 bytes retain their existing demand-time rejection, so an +unused unsupported fine level does not newly reject an otherwise usable pack. + +`ReadBatchBytes` remains a soft target and is capped by `MaximumReadBatchBytes`. +For the former maximum coalescing range, explicitly construct: + +```csharp +new WorldStreamer(world, path, budgetBytes, maximumReadBatchBytes: 131040); +``` + +The original three-parameter constructor signature is retained for compiled +clients. Capacity is reserved before streaming begins; `Pump` never resizes or +disposes the source buffer. This avoids introducing a GPU wait through the SDK's +conservative native-buffer disposal. Source holds, cancellation drain, transfer +padding, CRC and texture publication are unchanged. `StagingBytes` reports this +separate allocation; `PhysicalBytes` retains its existing dynamic-residency +meaning. Scenario teardown checks now use the actual staging size. + +### Compact IDs and lazy ARAM metadata + +Catalog record IDs are bounded by 4,096, so private indexes/links use `ushort`. +Scheduler capacity is bounded by 256; its lookup stores slot+1 in `ushort`, with +zero as the missing sentinel. Public return types, generation checks, record +ordering and the cached `AssetRecord[]` representation are unchanged. Maximum-ID +and slot-255 tests exercise relocation, stale mappings, cancellation and drain. + +Combined retained catalog/scheduler payload savings are `4*recordCount+1566` +bytes: 4,162 B at 649 records and 17,950 B at 4,096. The catalog constructor also +uses smaller temporary mesh-tail IDs without retaining or mutating its caller's +metadata buffer. + +An unused `AramCache` no longer reserves per-record entry/age arrays. Metadata is +created only after the first native cold allocation succeeds; empty probes, +over-budget stores and failed native allocation do not allocate it. Once +active, the cache uses the same bounds and LRU/protected-ID behavior. On the +32-bit target this avoids `8*(recordCount+1)` payload bytes while unused, up to +32,776 B at 4,096 records. This one-time first-use allocation may occur during +streaming; normal stores already allocate transfer/ARAM ownership. Callers still drain +transfers before disposal. Zero-ID entries are now evicted and freed correctly. + +### Bootstrap and failed construction + +The synchronous bootstrap file is scoped to world construction. Its handle and +SDK/stdio resources are no longer retained for the world's lifetime; the +independent asynchronous `AssetStream` remains owned by the world. Failed world +construction also releases already-created coarse meshes, textures and asset +stream ownership. This prevents malformed input or allocation failure from +leaving native resources behind. Cleanup is on the bootstrap/error path, not the +steady-state promotion path. SDK-free allocation-failure/corruption tests verify +partial ownership is released. + +## DolphinDotNet audit + +The SDK changes are delivered separately in +[DolphinDotNet PR #171](https://github.com/capz/DolphinDotNet/pull/171). This engine +keeps its source pin and published NuGet dependency on SDK 318 until a matching +SDK release is available; source and package consumers must use the same reviewed +SDK revision. The engine reductions above work with SDK 318 independently. + +The SDK renderer previously reserved two 4,096-entry model submission tables, +although it admits at most 4,096 submissions in total. A shared table preserves +opaque insertion order, transparent depth sorting and stable ties while removing +65,536 B of native static storage. This saving applies when the executable links +the model submission path. A raw-mesh-only Hypercube executable may discard that +section, so this is not added to the world staging savings above. + +SDK transforms now reserve a local matrix only when first parented, and a child +list only when a child is first attached. A never-parented leaf keeps its existing +world-matrix cache and computes TRS directly into it. `MeshBatch`'s 128 root poses +therefore avoid 8,192 B of local-matrix payload plus child-list objects, empty +backing arrays and headers. Hierarchy caches are reserved during attachment and +retained for reuse; matrix updates and draws remain allocation-free after setup. +These are managed occupancy savings, not a change to the reserved heap size. +The SDK's host CLR probe measures 47,128 B before versus 28,696 B after for 128 +root transforms, including the array that holds them: an 18,432 B reduction. + +Native skinning workspace now holds the largest validated skin in a model +instead of always reserving room for 256 joints. A two-joint instance uses +288 B rather than 32,800 B including the native allocation header and alignment. +A cached stride avoids scanning skins per update; its extra 4 B per PPC instance +leaves a net 32,508 B saving in that example. A 256-joint model keeps its former +workspace size and pays the 4 B stride field. Budget checks, inverse-transpose +normal matrices, unload/recreation and cleanup remain covered by native tests. + +## Other opportunities and why they are deferred + +| Opportunity | Potential | Required follow-up | +| --- | --- | --- | +| Dense permanent-patch metadata | The 649-record fixture reserves 7,800 B of 32-bit array payload for 89 permanent patches; dense storage would use about 1,068 B plus a mapping | `Patches`, `TriangleCounts` and `VertexCounts` are public arrays indexed by record ID. Introduce/version an accessor API before changing that contract. | +| Compact or streamed catalog metadata | Parsed records alone occupy 56 B per record; input metadata adds another 64 B per record during construction | Reconstructing records in `Get` adds work to a hot path; a streaming parser needs bounded-read/CRC and v2-family tests. Keep cached records for this pass. The 4,096-record format limit is not a promise that a full world fits the default 256 KiB managed heap. | +| Further flatten `MeshBatch` transforms | After SDK transform compaction, 128 root poses still retain 13,312 B of float-array payload plus headers | A flat-matrix/native batching path needs transform, ordering, allocation and throughput parity. Keep reusable preallocation. | +| Drop decoded collision source copies | Up to 24 extra 44-byte native payloads, plus allocator overhead, duplicate decoded heights/masks | Residency and cell-readiness currently use their ownership during eviction. Change the accounting and lifecycle together, with movement/hole tests. | +| Shrink decode scratch | A valid maximum mesh needs 13,068 B versus the 16,384 B scratch reservation | Small gain relative to source staging; preserve public scratch/decode behavior until a separate bounded workspace API is agreed. | +| Reduce texture-pool copies or idle retention | Double-buffered textures and reusable unbound allocations dominate some working sets | Active GPU readers and async staging require their storage. Use measured residency/quality policy or the independent progressive-level runtime; do not free a front/back allocation or reduce quality implicitly. | +| Lower heap/framebuffer/ARAM reservations | Potentially larger physical MEM1 savings than live-object compaction | Requires representative console high-water, video mode, stacks, audio and pressure measurements. No physical-console performance or reserve-sizing claim is made here. | + +## Reproduction and acceptance + +```sh +python3 tools/build_assets.py --resolution 48 +python3 tools/memory_audit.py out/assets/world.hcp --output out/memory-audit/world.json +python3 tools/measure_core_allocations.py --revision b6eee61 +python3 tools/measure_core_allocations.py +python3 tools/test_host.py +python3 tests/progressive_fixtures.py +dotnet run --project tests/TextureFamilies.Tests -c Release +``` + +The CLR probe exports baseline or working-tree Core sources to an isolated +project, warms constructors, excludes input-buffer creation and counts allocated +bytes for catalog and a capacity-256 scheduler (the streaming adapter uses 128). +It is a host measurement, not a +GameCube heap measurement. On this executor, totals were: + +| Record count | Baseline allocated | Updated allocated | Reduction | +| --- | ---: | ---: | ---: | +| 649 | 55,008 B | 50,664 B | 4,344 B | +| 4,096 | 275,624 B | 257,488 B | 18,136 B | + +Validation includes maximum record/slot boundaries, coalescing/alignment and +unsupported levels, unchanged catalog ownership, ARAM LRU/protection/copy failure, +empty-cache allocation bounds, constructor cleanup, both ordered-rendering paths +and full console-scene CI. Console checks must continue to cover stale/canceled +work, pixel consistency, audio, movement readiness and zero-resource teardown. diff --git a/samples/Validation/Program.cs b/samples/Validation/Program.cs index 04f6723..0f0e722 100644 --- a/samples/Validation/Program.cs +++ b/samples/Validation/Program.cs @@ -152,8 +152,8 @@ private void Transition() { streamer.Frame=frame; if(!streamer.Stop())return; - streamer.Dispose(); - if(device.NativeBytes!=baseline-131072)throw new InvalidOperationException("Native resources leaked across a scenario transition."); + int stagingBytes=streamer.StagingBytes;streamer.Dispose(); + if(device.NativeBytes!=baseline-stagingBytes)throw new InvalidOperationException("Native resources leaked across a scenario transition."); if(Aram.Statistics.AllocatedBytes!=0)throw new InvalidOperationException("ARAM transfers or allocations leaked across a scenario transition."); scenario=requested;requested=-1;completedMask&=~(1<=cols*((4+r.Group-1)/r.Group)||textures[key]!=0)throw new IOException("Duplicate or invalid texture tile."); - textures[key] = i; + textures[key] = (ushort)i; } else if (r.Kind == 2) { if(r.Length<12||r.Length>16384||r.Cell<0||r.Cell>=24||r.Mip<0||r.Mip>2||r.Variant<0||r.Variant>1)throw new IOException("Invalid geometry key."); int key=r.Cell*4+(r.Variant==0?3:r.Mip); - if(meshHeads[key]==0)meshHeads[key]=i;else next[meshTails[key]]=i; - meshTails[key]=i; + if(meshHeads[key]==0)meshHeads[key]=(ushort)i;else next[meshTails[key]]=(ushort)i; + meshTails[key]=(ushort)i; } else if(r.Kind==3) - {if(r.Cell<0||r.Cell>=24||r.Length!=44||collisions[r.Cell]!=0)throw new IOException("Invalid collision record.");collisions[r.Cell]=i;} + {if(r.Cell<0||r.Cell>=24||r.Length!=44||collisions[r.Cell]!=0)throw new IOException("Invalid collision record.");collisions[r.Cell]=(ushort)i;} else if(r.Kind==4) - {if(r.Length<12||r.Length>16384||r.Cell<0||r.Cell>=85||r.Material<0||r.Material>1)throw new IOException("Invalid patch record.");next[i]=patchHeads[r.Cell];patchHeads[r.Cell]=i;} + {if(r.Length<12||r.Length>16384||r.Cell<0||r.Cell>=85||r.Material<0||r.Material>1)throw new IOException("Invalid patch record.");next[i]=patchHeads[r.Cell];patchHeads[r.Cell]=(ushort)i;} else if(r.Kind==5&& (r.Length!=56||++scenes>1))throw new IOException("Invalid scene metadata."); } for (int i = 1; i <= Count; i++) diff --git a/src/Hypercube.Core/ResourceScheduler.cs b/src/Hypercube.Core/ResourceScheduler.cs index b6661af..195f590 100644 --- a/src/Hypercube.Core/ResourceScheduler.cs +++ b/src/Hypercube.Core/ResourceScheduler.cs @@ -11,7 +11,9 @@ public sealed class ResourceScheduler { private readonly int[] ids, priorities, generations, retries, references, pins, touched, fences; private readonly ResourceState[] states; - private readonly int[] lookup; + // Slot+1 fits in ushort even for slot 255. Zero means no mapping; + // stale mappings remain harmless until the old slot is reused. + private readonly ushort[] lookup; private readonly uint[] freeMasks; private readonly AssetCatalog catalog; private int nextGeneration = 1, order; @@ -26,8 +28,7 @@ public ResourceScheduler(AssetCatalog catalog, int capacity, int budgetBytes) this.catalog = catalog; BudgetBytes = budgetBytes; ids = new int[capacity]; priorities = new int[capacity]; generations = new int[capacity]; retries = new int[capacity]; references = new int[capacity]; pins = new int[capacity]; touched = new int[capacity]; - states = new ResourceState[capacity]; fences = new int[capacity]; lookup = new int[catalog.Count+1]; - for(int i=0;i>5]|=1u<<(i&31); } @@ -36,7 +37,11 @@ public ResourceScheduler(AssetCatalog catalog, int capacity, int budgetBytes) public ResourceState State(int slot) => states[slot]; public ResourcePriority Priority(int slot) => (ResourcePriority)priorities[slot]; public int Find(int id) - { return id>0&&id=0&&states[lookup[id]]!=ResourceState.Missing?lookup[id]:-1; } + { + if(id<=0||id>=lookup.Length)return -1; + int slot=lookup[id]-1; + return slot>=0&&states[slot]!=ResourceState.Missing?slot:-1; + } public void BeginDemand() { for (int i = 0; i < ids.Length; i++) references[i] = 0; } private int FirstFree() @@ -73,8 +78,8 @@ public int Demand(int id, ResourcePriority priority) if(free>=0) { int i=free; - if(ids[i]>0&&lookup[ids[i]]==i)lookup[ids[i]]=-1; - ids[i] = id; lookup[id]=i; states[i] = ResourceState.Queued; priorities[i] = (int)priority; + if(ids[i]>0&&lookup[ids[i]]==i+1)lookup[ids[i]]=0; + ids[i] = id; lookup[id]=(ushort)(i+1); states[i] = ResourceState.Queued; priorities[i] = (int)priority; freeMasks[i>>5]&=~(1u<<(i&31)); references[i] = 1; touched[i] = ++order; generations[i] = nextGeneration++; retries[i] = 0; return i; diff --git a/src/Hypercube.Core/StreamingBufferPolicy.cs b/src/Hypercube.Core/StreamingBufferPolicy.cs new file mode 100644 index 0000000..04ebcf0 --- /dev/null +++ b/src/Hypercube.Core/StreamingBufferPolicy.cs @@ -0,0 +1,33 @@ +using System; + +namespace Hypercube; + +/// Bounded storage for V1 streaming batches. Coalescing is a soft target; +/// a larger supported single record still gets enough aligned source storage. +public static class StreamingBufferPolicy +{ + public const int MaximumRecordBytes=131040; + public const int DefaultMaximumReadBatchBytes=65536; + private static void ValidateMaximum(int bytes) + {if(bytes<32||bytes>MaximumRecordBytes)throw new ArgumentOutOfRangeException("maximumReadBatchBytes");} + public static int Capacity(AssetCatalog catalog,int maximumReadBatchBytes=DefaultMaximumReadBatchBytes) + { + if(catalog==null)throw new ArgumentNullException("catalog");ValidateMaximum(maximumReadBatchBytes); + int bytes=maximumReadBatchBytes; + for(int id=1;id<=catalog.Count;id++) + { + var record=catalog.Get(id); + // WorldStreamer requests textures, detailed meshes and collisions. + // Larger unsupported levels retain the existing demand-time error; + // an unused fine level must not reject an otherwise usable pack. + if(record.Kind>=1&&record.Kind<=3&&record.Length<=MaximumRecordBytes&&record.Length>bytes)bytes=record.Length; + } + return (bytes+31)&~31; + } + public static int CoalescingLimit(int requestedReadBatchBytes,int maximumReadBatchBytes) + { + ValidateMaximum(maximumReadBatchBytes); + if(requestedReadBatchBytes<32)throw new ArgumentOutOfRangeException("requestedReadBatchBytes"); + return Math.Min(requestedReadBatchBytes,maximumReadBatchBytes); + } +} diff --git a/src/Hypercube.GameCube/AramCache.cs b/src/Hypercube.GameCube/AramCache.cs index d03ad19..43443cc 100644 --- a/src/Hypercube.GameCube/AramCache.cs +++ b/src/Hypercube.GameCube/AramCache.cs @@ -7,44 +7,62 @@ namespace Hypercube.GameCube; public sealed class AramCache : IDisposable { - private readonly AramBuffer?[] buffers; - private readonly int[] touched; + private readonly int capacity; + // The default streamer disables ARAM. Reserve per-record metadata only when + // a native cold allocation succeeds, not for Has probes or rejected stores. + private AramBuffer?[]? buffers; + private int[]? touched; private int order; public int Bytes {get;private set;} public int Hits,Stores,Evictions; public int Budget {get;} - public AramCache(int records,int budget){buffers=new AramBuffer?[records+1];touched=new int[records+1];Budget=budget;} - public bool Has(int id)=>buffers[id]!=null; + public AramCache(int records,int budget) + {if(records<0||records==int.MaxValue||budget<0)throw new ArgumentOutOfRangeException("records");capacity=records+1;Budget=budget;} + private void ValidateId(int id) + {if((uint)id>=(uint)capacity)throw new IndexOutOfRangeException();} + public bool Has(int id){ValidateId(id);return buffers!=null&&buffers[id]!=null;} public Task Fetch(int id,NativeBuffer target,int targetOffset,int bytes) { - if(buffers[id]==null)throw new InvalidOperationException("ARAM cache miss."); - touched[id]=++order;Hits++; - return AramTransferQueue.CopyToMainAsync(buffers[id]!,0,target,targetOffset,(bytes+31)&~31); + if(!Has(id))throw new InvalidOperationException("ARAM cache miss."); + touched![id]=++order;Hits++; + return AramTransferQueue.CopyToMainAsync(buffers![id]!,0,target,targetOffset,(bytes+31)&~31); } public Task? Store(int id,NativeBuffer source,int sourceOffset,int bytes,int[] protectedIds,int count) { - if(buffers[id]!=null)return null; + ValidateId(id); + if(buffers!=null&&buffers[id]!=null)return null; + if(bytes<1||bytes>int.MaxValue-31)throw new ArgumentOutOfRangeException("bytes"); int size=(bytes+31)&~31; if(size>Budget)return null; - while(Bytes+size>Budget) + while(size>Budget-Bytes) { int victim=-1; - for(int i=1;i4096)throw new IOException("Pack record count exceeds validation bound."); - var table=new byte[count*64];ReadExact(bootstrap,table,table.Length); - Catalog=new AssetCatalog(header,table,bootstrap.Length); - Patches=new Mesh?[count+1];TriangleCounts=new int[count+1];VertexCounts=new int[count+1]; - for(int i=0;i<85;i++){NodeMinY[i]=100;NodeMaxY[i]=-100;} - for(int tile=0;tile<2;tile++) + try { - var r=Catalog.Get(Catalog.Texture(14,4,3,tile)); - var bytes=ReadPayload(r); - var texture=Texture2D.CreateNative(r.Width,r.Width,NativeTextureFormat.Cmpr,true); - texture.ReplaceNative(bytes);CoarseTextures[tile]=texture; - } - int native=0; - for(int id=1;id<=count;id++) - { - var r=Catalog.Get(id); - if(r.Kind==4) + using var bootstrap=new FileStream(path,FileMode.Open,FileAccess.Read,FileShare.Read); + var header=new byte[32];ReadExact(bootstrap,header,32); + int count=Binary.I32(header,8); + if(count<1||count>4096)throw new IOException("Pack record count exceeds validation bound."); + var table=new byte[count*64];ReadExact(bootstrap,table,table.Length); + Catalog=new AssetCatalog(header,table,bootstrap.Length); + Patches=new Mesh?[count+1];TriangleCounts=new int[count+1];VertexCounts=new int[count+1]; + for(int i=0;i<85;i++){NodeMinY[i]=100;NodeMaxY[i]=-100;} + for(int tile=0;tile<2;tile++) { - var payload=ReadPayload(r); - var texture=Catalog.Get(Catalog.Texture(14,4,3,r.Material)); - var mesh=DecodeMesh(payload,r,texture,4); - Patches[id]=mesh;ParentMeshCount++;ParentTriangles+=Binary.U16(payload,6)/3; - TriangleCounts[id]=Binary.U16(payload,6)/3;VertexCounts[id]=Binary.U16(payload,4); - for(int k=0;kNodeMaxY[r.Cell])NodeMaxY[r.Cell]=y; - } - native+=mesh.NativeBytes; - if(native>1024*1024)throw new IOException("Coarse patch hierarchy exceeds its 1 MiB native budget."); + var r=Catalog.Get(Catalog.Texture(14,4,3,tile)); + var bytes=ReadPayload(bootstrap,r); + var texture=Texture2D.CreateNative(r.Width,r.Width,NativeTextureFormat.Cmpr,true); + CoarseTextures[tile]=texture;texture.ReplaceNative(bytes); } - else if(r.Kind==5) + int native=0; + for(int id=1;id<=count;id++) { - var payload=ReadPayload(r); - if(r.Length!=56||Binary.U32(payload,0)!=0x44574348)throw new IOException("Invalid scene metadata."); - MinY=Binary.I32(payload,8)/65536f;MaxY=Binary.I32(payload,20)/65536f; - VerifiedOccluder=Binary.I32(payload,28)==1; - for(int k=0;k<6;k++)Occluder[k]=Binary.I32(payload,32+k*4)/65536f; + var r=Catalog.Get(id); + if(r.Kind==4) + { + var payload=ReadPayload(bootstrap,r); + var texture=Catalog.Get(Catalog.Texture(14,4,3,r.Material)); + var mesh=DecodeMesh(payload,r,texture,4); + Patches[id]=mesh;ParentMeshCount++;ParentTriangles+=Binary.U16(payload,6)/3; + TriangleCounts[id]=Binary.U16(payload,6)/3;VertexCounts[id]=Binary.U16(payload,4); + for(int k=0;kNodeMaxY[r.Cell])NodeMaxY[r.Cell]=y; + } + native+=mesh.NativeBytes; + if(native>1024*1024)throw new IOException("Coarse patch hierarchy exceeds its 1 MiB native budget."); + } + else if(r.Kind==5) + { + var payload=ReadPayload(bootstrap,r); + if(r.Length!=56||Binary.U32(payload,0)!=0x44574348)throw new IOException("Invalid scene metadata."); + MinY=Binary.I32(payload,8)/65536f;MaxY=Binary.I32(payload,20)/65536f; + VerifiedOccluder=Binary.I32(payload,28)==1; + for(int k=0;k<6;k++)Occluder[k]=Binary.I32(payload,32+k*4)/65536f; + } } + if(ParentMeshCount==0)throw new IOException("The pack has no permanent coarse hierarchy."); + } + catch + { + // A failed constructor cannot hand ownership back to the caller. + // Reclaim the source and every GPU resource created so far. + Dispose(); + throw; } - if(ParentMeshCount==0)throw new IOException("The pack has no permanent coarse hierarchy."); } public bool HasCell(int cell)=>Catalog.FirstPatch(Tree.Leaf(cell))!=0; - private byte[] ReadPayload(AssetRecord r) + private byte[] ReadPayload(FileStream bootstrap,AssetRecord r) { if(r.Length>16384)throw new IOException("Bootstrap payload exceeds its managed scratch limit."); var bytes=Scratch;bootstrap.Seek(r.Offset,SeekOrigin.Begin);ReadExact(bootstrap,bytes,r.Length); @@ -122,8 +131,8 @@ public Mesh DecodeMesh(byte[] bytes,AssetRecord record,AssetRecord texture,int g } public void Dispose() { - Assets.Dispose();bootstrap.Dispose(); - for(int i=0;iTextures.PhysicalBytes+otherBytes; public int Active=>count; + public int MaximumReadBatchBytes {get;} + public int StagingBytes=>disposed?0:staging.Length; + // Retain the original constructor's binary signature for existing clients. public WorldStreamer(PackedWorld world,string path,int budgetBytes=384*1024) + :this(world,path,budgetBytes,StreamingBufferPolicy.DefaultMaximumReadBatchBytes){} + /// Reserve the configured coalescing limit and every supported + /// single record up front. ReadBatchBytes may change within that limit; + /// use 131040 to restore the previous maximum coalescing range. + public WorldStreamer(PackedWorld world,string path,int budgetBytes,int maximumReadBatchBytes) { this.world=world;scratch=world.Scratch;catalog=world.Catalog;stream=world.Assets; + int stagingBytes=StreamingBufferPolicy.Capacity(catalog,maximumReadBatchBytes); + MaximumReadBatchBytes=maximumReadBatchBytes; Policy=new ResourceScheduler(catalog,ResourceCapacity,budgetBytes); Cold=new AramCache(catalog.Count,256*1024); for(int i=0;iPolicy.BeginDemand(); private void Demand(int id,ResourcePriority priority) @@ -159,8 +172,9 @@ private bool TextureRoom(AssetRecord record,ResourcePriority priority) } private void Queue() { - int limit=Policy.RequiredBatchBytes(Frame,ReadBatchBytes); - if(limit>131040)throw new IOException("Asset exceeds the bounded native staging size."); + int coalescing=StreamingBufferPolicy.CoalescingLimit(ReadBatchBytes,MaximumReadBatchBytes); + int limit=Policy.RequiredBatchBytes(Frame,coalescing); + if(limit>StreamingBufferPolicy.MaximumRecordBytes)throw new IOException("Asset exceeds the bounded native staging size."); count=Policy.BeginBatch(Frame,limit,MaxBatchRecords,batch,out batchOffset,out batchLength); if(count==0)return; // A large record may exceed the requested coalescing size; SDK reads it @@ -320,8 +334,9 @@ public bool Stop() } public void Dispose() { + if(disposed)return; if(count!=0)throw new InvalidOperationException("Drain streaming work before disposal."); for(int i=0;iData.Length; + internal AramBuffer(int size){Data=new byte[size];Aram.LiveBytes+=size;} + public void Dispose(){if(disposed)throw new Exception("ARAM allocation disposed twice.");disposed=true;Aram.LiveBytes-=Length;} + } + public static class Aram + { + public static bool Fail; + public static int Attempts,LiveBytes; + public static bool TryAllocate(int bytes,out AramBuffer allocation) + {Attempts++;if(Fail){allocation=null!;return false;}allocation=new AramBuffer(bytes);return true;} + } + public static class AramTransferQueue + { + public static bool Fail;public static int LastBytes; + public static Task CopyToAramAsync(Dolphin.Graphics.NativeBuffer source,int sourceOffset,AramBuffer target,int targetOffset,int bytes) + {if(Fail)throw new InvalidOperationException("Injected copy failure.");LastBytes=bytes;Array.Copy(source.Data,sourceOffset,target.Data,targetOffset,bytes);return Task.CompletedTask;} + public static Task CopyToMainAsync(AramBuffer source,int sourceOffset,Dolphin.Graphics.NativeBuffer target,int targetOffset,int bytes) + {LastBytes=bytes;Array.Copy(source.Data,sourceOffset,target.Data,targetOffset,bytes);return Task.CompletedTask;} + } +} diff --git a/tests/Hypercube.GameCube.Tests/Hypercube.GameCube.Tests.csproj b/tests/Hypercube.GameCube.Tests/Hypercube.GameCube.Tests.csproj index 90b8987..6c781da 100644 --- a/tests/Hypercube.GameCube.Tests/Hypercube.GameCube.Tests.csproj +++ b/tests/Hypercube.GameCube.Tests/Hypercube.GameCube.Tests.csproj @@ -2,5 +2,7 @@ Exe + + diff --git a/tests/Hypercube.GameCube.Tests/PackedWorldChecks.cs b/tests/Hypercube.GameCube.Tests/PackedWorldChecks.cs new file mode 100644 index 0000000..bb34511 --- /dev/null +++ b/tests/Hypercube.GameCube.Tests/PackedWorldChecks.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using Dolphin.Graphics; +using Dolphin.Storage; +using Hypercube; +using Hypercube.GameCube; + +internal static class PackedWorldChecks +{ + private static void Check(bool value,string message){if(!value)throw new Exception(message);} + private static void Empty(string phase) + {Check(AssetStream.Open==0&&Texture2D.Live==0&&Mesh.Live==0,"PackedWorld leaked partial ownership: "+phase);} + internal static void Run() + { + const string fixture="out/assets/world.hcp"; + Empty("before checks"); + using(var world=new PackedWorld(fixture)) + {Check(AssetStream.Open==1&&Texture2D.Live==2&&Mesh.Live==world.ParentMeshCount&&Mesh.Live>1,"Successful world bootstrap did not retain its expected owners.");} + Empty("successful dispose"); + Texture2D.FailOnUpload=Texture2D.Uploads+2; + try{new PackedWorld(fixture);throw new Exception("Injected texture upload failure was swallowed.");} + catch(InvalidOperationException error){Check(error.Message=="Injected texture upload failure.","Cleanup masked the original texture error.");} + finally{Texture2D.FailOnUpload=-1;} + Empty("second coarse texture upload failure"); + byte[] original=File.ReadAllBytes(fixture);int records=Binary.I32(original,8); + string temporary=Path.Combine(Path.GetTempPath(),"hypercube-bootstrap-"+Guid.NewGuid().ToString("N")+".hcp"); + try + { + var truncated=new byte[16];File.WriteAllBytes(temporary,truncated); + try{new PackedWorld(temporary);throw new Exception("Truncated bootstrap accepted.");}catch(EndOfStreamException){} + Empty("truncated header"); + var badMetadata=(byte[])original.Clone();badMetadata[32]^=1;File.WriteAllBytes(temporary,badMetadata); + try{new PackedWorld(temporary);throw new Exception("Bad bootstrap metadata accepted.");}catch(IOException){} + Empty("invalid metadata"); + int seen=0,failedPatch=-1; + for(int i=0;i=0,"Fixture must include at least two bootstrap patches."); + var badPayload=(byte[])original.Clone();badPayload[failedPatch]^=128;File.WriteAllBytes(temporary,badPayload); + int created=Mesh.Creates; + try{new PackedWorld(temporary);throw new Exception("Corrupt bootstrap payload accepted.");} + catch(IOException error){Check(error.Message=="Bootstrap asset checksum mismatch.","Cleanup masked the original bootstrap error.");} + Check(Mesh.Creates==created+1,"Corrupt fixture did not reach a partially created mesh hierarchy."); + Empty("corrupt second patch"); + Mesh.FailOnCreate=Mesh.Creates+2; + try{new PackedWorld(fixture);throw new Exception("Injected mesh allocation failure was swallowed.");}catch(OutOfMemoryException){} + finally{Mesh.FailOnCreate=-1;} + Empty("allocation failure after first patch"); + } + finally{File.Delete(temporary);Mesh.FailOnCreate=-1;Texture2D.FailOnUpload=-1;} + Console.WriteLine("PackedWorld bootstrap: source/texture/mesh ownership reclaimed after truncated header, bad index, failed texture upload, corrupt partial hierarchy and allocation failure."); + } +} diff --git a/tests/Hypercube.GameCube.Tests/Program.cs b/tests/Hypercube.GameCube.Tests/Program.cs index ad1e1f0..dbffd08 100644 --- a/tests/Hypercube.GameCube.Tests/Program.cs +++ b/tests/Hypercube.GameCube.Tests/Program.cs @@ -1,6 +1,7 @@ using Hypercube; using Hypercube.GameCube; using Dolphin.Graphics; +AramCacheChecks.Run(); var cache=new IndexedMeshCache();var counters=new RenderCounters(); var vertices=new VertexPositionNormalColorTexture[12];var indices=new ushort[12]; if(!cache.Commit(vertices,4,indices,6,1,counters)||Mesh.Creates!=1||cache.VertexCount!=4||cache.IndexCount!=6)throw new Exception("Exact prefix upload"); @@ -16,11 +17,35 @@ var last=cache.Mesh;cache.Dispose();cache.Dispose();if(!last!.Disposed||cache.Mesh!=null||cache.VertexCount!=0)throw new Exception("Dispose"); try{cache.Commit(vertices,3,indices,3,4);throw new Exception("Disposed cache reused");}catch(ObjectDisposedException){} Console.WriteLine("Indexed cache: exact prefixes, revision/count invalidation, allocation-free hits, failed replacement and disposal passed."); +PackedWorldChecks.Run(); namespace Dolphin.Graphics { - public struct VertexPositionNormalColorTexture{} + public readonly struct Vector2 {public readonly float X,Y;public Vector2(float x,float y){X=x;Y=y;}} + public readonly struct Vector3 {public readonly float X,Y,Z;public Vector3(float x,float y,float z){X=x;Y=y;Z=z;}} + public readonly struct Color32 {public readonly byte R,G,B,A;public Color32(byte r,byte g,byte b,byte a){R=r;G=g;B=b;A=a;}} + public struct VertexPositionNormalColorTexture {public VertexPositionNormalColorTexture(Vector3 p,Vector3 n,Color32 c,Vector2 uv){}} + public enum NativeTextureFormat {Cmpr=14} + public sealed class Texture2D:IDisposable { + public static int Live,Uploads,FailOnUpload=-1;private bool disposed; + private Texture2D(){Live++;} + public static Texture2D CreateNative(int width,int height,NativeTextureFormat format,bool linear)=>new Texture2D(); + public void ReplaceNative(byte[] data){Uploads++;if(Uploads==FailOnUpload)throw new InvalidOperationException("Injected texture upload failure.");} + public void Dispose(){if(disposed)throw new Exception("Texture disposed twice");disposed=true;Live--;} + } public sealed class Mesh:IDisposable { - public static int Creates,LastVertices,LastIndices;public static bool Fail;public bool Disposed; - public static Mesh Create(VertexPositionNormalColorTexture[] vertices,int vertexCount,ushort[] indices,int indexCount){if(Fail)throw new OutOfMemoryException();Creates++;LastVertices=vertexCount;LastIndices=indexCount;return new Mesh();} - public void Dispose(){if(Disposed)throw new Exception("Double disposal");Disposed=true;} + public static int Creates,LastVertices,LastIndices,Live,FailOnCreate=-1;public static bool Fail;public bool Disposed; + public int NativeBytes{get;private set;} + public static Mesh Create(VertexPositionNormalColorTexture[] vertices,int vertexCount,ushort[] indices,int indexCount){if(Fail||Creates+1==FailOnCreate)throw new OutOfMemoryException();Creates++;Live++;LastVertices=vertexCount;LastIndices=indexCount;return new Mesh{NativeBytes=vertexCount*36+indexCount*2};} + public void Dispose(){if(Disposed)throw new Exception("Double disposal");Disposed=true;Live--;} + } +} + +namespace Dolphin.Storage { + public sealed class AssetStream:IDisposable { + public static int Open;private FileStream? stream; + public AssetStream(string path){stream=new FileStream(path,FileMode.Open,FileAccess.Read,FileShare.Read);Open++;} + public void Dispose(){if(stream==null)return;stream.Dispose();stream=null;Open--;} } } +namespace Dolphin.Timing { + public static class Clock {public static long Timestamp=>System.Diagnostics.Stopwatch.GetTimestamp();} +} diff --git a/tests/Hypercube.Tests/CompactIndexChecks.cs b/tests/Hypercube.Tests/CompactIndexChecks.cs new file mode 100644 index 0000000..f78ff5a --- /dev/null +++ b/tests/Hypercube.Tests/CompactIndexChecks.cs @@ -0,0 +1,63 @@ +using System; +using Hypercube; + +internal static class CompactIndexChecks +{ + private static void Check(bool condition,string message) + {if(!condition)throw new Exception(message);} + private static void Word(byte[] bytes,int at,int value)=>BitConverter.GetBytes(value).CopyTo(bytes,at); + + internal static void Run() + { + // The highest IDs exercise every compact lookup/link category, and the + // 4092 mesh records retain their original forward insertion order. + const int count=4096,geometryCount=4092; + var index=new byte[count*64];var header=new byte[32];int offset=32+index.Length; + for(int id=1;id<=count;id++) + { + int at=(id-1)*64,kind=id<=geometryCount?2:id<=4094?4:id==4095?3:1; + int length=kind==3?44:kind==1?128:12; + Word(index,at,id);Word(index,at+4,kind);Word(index,at+8,offset);Word(index,at+12,length); + if(kind==4)Word(index,at+32,84); + if(kind==3)Word(index,at+32,23); + if(kind==1) + {Word(index,at+20,14);Word(index,at+24,4);Word(index,at+28,3);Word(index,at+32,1);Word(index,at+40,16);Word(index,at+44,8);} + offset+=(length+31)&~31; + } + Word(header,0,0x4b504348);Word(header,4,1);Word(header,8,count);Word(header,12,32); + Word(header,16,32+index.Length);Word(header,20,offset); + Word(header,24,unchecked((int)Binary.Crc(index,0,index.Length))); + var catalog=new AssetCatalog(header,index,offset); + Check(catalog.Count==4096&&catalog.Get(4096).Id==4096,"Maximum record ID survives compact indexes"); + Check(catalog.FirstMesh(0,0,true)==1,"Compact mesh head preserves first record"); + for(int id=1;id<=geometryCount;id++)Check(catalog.Next(id)==(id==geometryCount?0:id+1),"Compact forward mesh links preserve ordering and terminal zero"); + Check(catalog.FirstPatch(84)==4094&&catalog.Next(4094)==4093&&catalog.Next(4093)==0,"Compact reverse patch links preserve highest node/record IDs"); + Check(catalog.Collision(23)==4095&&catalog.Texture(14,4,3,1)==4096,"Compact collision/texture indexes preserve maximum record IDs"); + var last=catalog.Get(4096);Array.Clear(index);Array.Clear(header); + Check(catalog.Get(4096).Equals(last)&&catalog.Texture(14,4,3,1)==4096&&catalog.Next(4094)==4093,"Catalog owns immutable validated metadata after caller buffers change"); + + var scheduler=new ResourceScheduler(catalog,256,1024*1024); + for(int slot=0;slot<256;slot++) + { + int id=3841+slot; + Check(scheduler.Demand(id,ResourcePriority.Visible)==slot&&scheduler.Find(id)==slot,"Compact lookup reaches every scheduler slot"); + } + Check(scheduler.Find(4096)==255&&scheduler.Find(0)==-1&&scheduler.Find(4097)==-1,"Slot 255 uses a nonzero 16-bit lookup sentinel"); + int oldGeneration=scheduler.Generation(255); + scheduler.Cancel(4096);scheduler.Cancel(3841); + Check(scheduler.Find(4096)==-1&&scheduler.Find(3841)==-1,"Cancelled mappings remain missing"); + // Relocate an ID before its previous slot is reused. Reclaiming that + // stale slot must not erase the ID's newer mapping. + Check(scheduler.Demand(4096,ResourcePriority.Visible)==0,"Relocated maximum ID uses first free slot"); + Check(scheduler.Demand(1,ResourcePriority.Visible)==255&&scheduler.Find(4096)==0&&scheduler.Find(1)==255,"Stale slot reuse cannot clear a relocated lookup"); + Check(scheduler.Generation(255)!=oldGeneration,"Slot reuse renews generation"); + scheduler.Cancel(1);Check(scheduler.Demand(2,ResourcePriority.Gameplay)==255&&scheduler.Find(1)==-1,"Reclaimed slot clears its own obsolete mapping"); + var batch=new int[1]; + Check(scheduler.BeginBatch(0,32,1,batch,out _,out _)==1&&batch[0]==255,"Highest compact slot can issue a prioritized read"); + int generation=scheduler.Generation(255); + scheduler.Cancel(2);Check(scheduler.State(255)==ResourceState.Retiring,"Issued highest-slot cancellation waits for drain"); + Check(!scheduler.Complete(255,generation,true,0)&&scheduler.Find(2)==-1,"Drained highest-slot generation cannot publish"); + Check(scheduler.Demand(3,ResourcePriority.Gameplay)==255&&scheduler.Find(3)==255,"Highest slot remains reusable after drain"); + Console.WriteLine("Compact indexes: maximum 4096 records, slot 255, immutable metadata, stale relocation and cancellation/drain passed."); + } +} diff --git a/tests/Hypercube.Tests/Program.cs b/tests/Hypercube.Tests/Program.cs index 4ef0ee1..7cadec5 100644 --- a/tests/Hypercube.Tests/Program.cs +++ b/tests/Hypercube.Tests/Program.cs @@ -12,6 +12,7 @@ static void Reject(Action action, string label) int count = Binary.I32(header, 8); var index = raw[32..(32 + count * 64)]; var catalog = new AssetCatalog(header, index, raw.Length); +StreamingBufferChecks.Run(catalog); Check(Binary.Crc(System.Text.Encoding.ASCII.GetBytes("123456789"), 0, 9) == 0xcbf43926u, "CRC known vector"); foreach (int format in new[] { 6, 14 }) foreach (int group in new[] { 1, 2, 4 }) { @@ -341,3 +342,5 @@ static void Reject(Action action, string label) var beforeMutation=stableCatalog.Get(1);Array.Clear(stableIndex); Check(stableCatalog.Get(1).Equals(beforeMutation),"Validated metadata remains immutable after input mutation"); Console.WriteLine("Free-slot transitions, stable demand ordering, cached metadata and execution accounting passed."); + +CompactIndexChecks.Run(); diff --git a/tests/Hypercube.Tests/StreamingBufferChecks.cs b/tests/Hypercube.Tests/StreamingBufferChecks.cs new file mode 100644 index 0000000..717ae06 --- /dev/null +++ b/tests/Hypercube.Tests/StreamingBufferChecks.cs @@ -0,0 +1,60 @@ +using System; +using Hypercube; + +internal static class StreamingBufferChecks +{ + private static void Check(bool value,string message){if(!value)throw new Exception(message);} + private static void Reject(Action action) + {try{action();}catch(ArgumentException){return;}throw new Exception("Invalid staging policy accepted.");} + public static void Run(AssetCatalog catalog) + { + int capacity=StreamingBufferPolicy.Capacity(catalog); + Check(capacity==73984,"Validation fixture staging reserves its largest RGBA8 image, not 128 KiB."); + Check(131072-capacity==57088,"Fixture staging saving is 57,088 native payload bytes."); + Check(StreamingBufferPolicy.Capacity(catalog,131040)==131040,"Explicit legacy coalescing cap."); + Check(StreamingBufferPolicy.CoalescingLimit(32768,65536)==32768,"32 KiB coalescing retained."); + Check(StreamingBufferPolicy.CoalescingLimit(65536,65536)==65536,"64 KiB coalescing retained."); + Check(StreamingBufferPolicy.CoalescingLimit(131040,65536)==65536,"Runtime target is bounded by reserved coalescing storage."); + foreach(int requested in new[]{32768,65536,131040}) + { + var scheduler=new ResourceScheduler(catalog,8,1024*1024);scheduler.BeginDemand(); + scheduler.Demand(catalog.Texture(6,4,0,0),ResourcePriority.Visible); + int limit=scheduler.RequiredBatchBytes(0,StreamingBufferPolicy.CoalescingLimit(requested,65536)); + Check(limit==73984,"A large single image overrides the coalescing target."); + var slots=new int[8];Check(scheduler.BeginBatch(0,limit,8,slots,out _,out int actual)==1,"Large image remains schedulable."); + Check(((actual+31)&~31)<=capacity,"Aligned large-image read fits its reserved buffer."); + } + // Adjacent 44-byte collision records are separated by 20 alignment bytes. + // Coalescing them reads 108 bytes and needs 128 bytes after SDK padding. + var collisions=Catalog(3,44,0,2); + Check(StreamingBufferPolicy.Capacity(collisions,32)==64,"A small single record includes native read padding."); + Check(StreamingBufferPolicy.Capacity(collisions,108)==128,"An unaligned coalescing cap reserves padding."); + var batch=new ResourceScheduler(collisions,2,1024);batch.BeginDemand(); + batch.Demand(1,ResourcePriority.Gameplay);batch.Demand(2,ResourcePriority.Gameplay); + var ids=new int[2];Check(batch.BeginBatch(0,108,2,ids,out _,out int bytes)==2&&bytes==108,"Near-adjacent record coalescing remains intact."); + Check(((bytes+31)&~31)==StreamingBufferPolicy.Capacity(collisions,108),"Rounded batch ends exactly at reserved capacity."); + var supported=Catalog(1,176*176*4,176,1); + Check(StreamingBufferPolicy.Capacity(supported)==123904,"A supported image larger than the default cap reserves its full source."); + var unsupported=Catalog(1,1024*1024*4,1024,1); + Check(StreamingBufferPolicy.Capacity(unsupported)==65536,"Unused unsupported fine levels do not force a large allocation."); + var unsupportedDemand=new ResourceScheduler(unsupported,1,8*1024*1024);unsupportedDemand.BeginDemand();unsupportedDemand.Demand(1,ResourcePriority.Visible); + Check(unsupportedDemand.RequiredBatchBytes(0,65536)>StreamingBufferPolicy.MaximumRecordBytes,"Demanded unsupported images retain the existing Queue rejection."); + foreach(int maximum in new[]{-1,0,31,131041,int.MaxValue})Reject(()=>StreamingBufferPolicy.Capacity(catalog,maximum)); + Reject(()=>StreamingBufferPolicy.Capacity(null!));Reject(()=>StreamingBufferPolicy.CoalescingLimit(31,65536));Reject(()=>StreamingBufferPolicy.CoalescingLimit(32,0)); + Console.WriteLine("Streaming staging: catalog sizing, explicit caps, dynamic targets, oversized records and read padding passed."); + } + private static AssetCatalog Catalog(int kind,int length,int width,int count) + { + var header=new byte[32];var table=new byte[count*64];int offset=32+table.Length; + void Word(byte[] data,int at,int value)=>BitConverter.GetBytes(value).CopyTo(data,at); + for(int i=0;iExenet8.0enableenable') + print('Source:',a.revision or 'working tree',flush=True) + subprocess.run(['dotnet','run','--project',str(temp/'Probe.csproj'),'-c','Release'],check=True) diff --git a/tools/memory_audit.py b/tools/memory_audit.py new file mode 100644 index 0000000..547eaab --- /dev/null +++ b/tools/memory_audit.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Reproducible payload accounting for a validated HCPK v1 world (not a hardware trace).""" +import argparse +import hashlib +import json +from pathlib import Path +import struct +from hypercube import pack + + +def audit(path, maximum_read_batch_bytes=65536): + if not 32 <= maximum_read_batch_bytes <= 131040: + raise ValueError('Maximum read batch must be 32..131040 bytes') + path = Path(path) + count = pack.inspect(path) + raw = path.read_bytes() + if struct.unpack_from('slot lookup also has a sentinel. + catalog = change(4*(count+782), 2*(count+782)) + scheduler = change(4*(count+1), 2*(count+1)) + native = change(131072, staging) + return dict(schema=1,scope='validated-v1-world-payload-accounting',hardwareMeasurement=False, + sourceSha256=hashlib.sha256(raw).hexdigest(),records=count, + maximumReadBatchBytes=maximum_read_batch_bytes, + largestSupportedStreamRecord=max(supported, default=0), + unsupportedStreamRecords=sum(size>131040 for size in lengths), + nativeStagingBytes=native, + retainedManagedPayload=dict(catalogIds=catalog,schedulerLookup=scheduler, + saved=catalog['saved']+scheduler['saved']), + idleAramMetadata=dict(targetPointerBytes=4,disabled=change(8*(count+1),0), + enabledAfterFirstStore=change(8*(count+1),8*(count+1))), + transientCatalogTailPayload=change(96*4,96*2), + unchanged=dict(parsedRecordPayloadBytes=count*56,metadataInputBytes=count*64, + textureCopiesPerPoolSlot=2), + caveats=['Selected array element payload only; object field changes, headers and allocator rounding excluded.', + 'Managed heap reservation is unchanged; these are occupancy reductions.', + 'Cold ARAM, permanent geometry, active/idle texture pairs, framebuffers and SDK heaps are separate.', + 'Unsupported large records retain their existing demand-time rejection.']) + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('pack',type=Path) + parser.add_argument('--maximum-read-batch-bytes',type=int,default=65536) + parser.add_argument('--output',type=Path) + args=parser.parse_args() + result=json.dumps(audit(args.pack,args.maximum_read_batch_bytes),indent=2,sort_keys=True)+'\n' + if args.output: + args.output.parent.mkdir(parents=True,exist_ok=True) + args.output.write_text(result) + else: + print(result,end='') +if __name__=='__main__':main() diff --git a/tools/memory_probe.cs b/tools/memory_probe.cs new file mode 100644 index 0000000..3a02a87 --- /dev/null +++ b/tools/memory_probe.cs @@ -0,0 +1,31 @@ +// Host CLR construction-allocation probe. Never a GameCube heap measurement. +using System; +using Hypercube; + +internal static class Program +{ + private static void Word(byte[] data,int at,int value)=>BitConverter.GetBytes(value).CopyTo(data,at); + private static void Main() + { + foreach(int count in new[]{649,4096}) + { + // Synthetic, valid metadata isolates the record-count allocation cost. + // Input buffers and JIT warmup are outside both measured windows. + var index=new byte[count*64];var header=new byte[32];int offset=32+index.Length; + for(int i=0;i