From a2a32390d1fde4e14c98449f5f0357e79edfb481 Mon Sep 17 00:00:00 2001 From: "Lt. Sammy" <26645712+ltsammy@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:11:23 +0000 Subject: [PATCH 1/2] feat(model3d): read Source engine models (MDL) with their VTF textures A Source engine model (versions 44 to 49, with the .vvd and .dx90.vtx beside it, all three of the same build) becomes a 3D model in its bind pose with its default body groups, first skin and LOD 0, turned to Filee's Y up, millimetres, counter-clockwise and bottom-left UVs, so it converts to OBJ, GLB, STL and the other 3D formats and to a picture. Its textures are found as Garry's Mod finds them: the .vmt in the materials folder beside the models folder, its $basetexture decoded from VTF. Every triangle is counted against one budget for the whole model (10 million), in strips and lists alike, so strip groups that point at the same table can't multiply it; a larger model is refused as too large. A material file over 1 MB is not read (the material stays plain). MDL is in the Games field; .mdl files learn the models donut (schema 100). Part of #1526. --- README.md | 2 +- docs/ENGINES.md | 24 +- src/Filee.Core/Formats/FormatFields.cs | 4 +- src/Filee.Core/Formats/FormatRegistry.cs | 3 + src/Filee.Core/Settings/AppSettings.cs | 2 +- src/Filee.Core/Settings/UserDataStore.cs | 3 + src/Filee.Engines/Archives/Vpk.cs | 3 +- src/Filee.Engines/Models3D/ModelConverter.cs | 11 +- src/Filee.Engines/Models3D/SourceModel.cs | 622 ++++++++++++++++++ tests/Filee.Core.Tests/FormatFieldsTests.cs | 8 +- tests/Filee.Core.Tests/SourceFormatTests.cs | 25 +- tests/Filee.Core.Tests/UserDataStoreTests.cs | 6 +- .../Filee.Engines.Tests/SourceModelBuilder.cs | 261 ++++++++ tests/Filee.Engines.Tests/SourceModelTests.cs | 189 ++++++ 14 files changed, 1141 insertions(+), 22 deletions(-) create mode 100644 src/Filee.Engines/Models3D/SourceModel.cs create mode 100644 tests/Filee.Engines.Tests/SourceModelBuilder.cs create mode 100644 tests/Filee.Engines.Tests/SourceModelTests.cs diff --git a/README.md b/README.md index 5418132f0..127610b50 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ To undo it, remove the entries in Windows Security → App & browser control → | **CAD**: DWG, DXF
Gerber PCB layers (GBR, GTL, GBL, GTO, …) read;
G-code of 3D printers and CNC machines (GCODE, GCO, NGC) read | PDF, SVG, PNG, DWG ↔ DXF;
G-code → toolpath (from above or isometric) or the slicer's preview | | **Fonts**: TTF, OTF, WOFF, WOFF2, EOT;
FON (Windows bitmap fonts) read | each other;
FON → one font per size | | **Data**: JSON, JSON Lines, YAML, TOML, XML
vCard, iCalendar | each other, CSV and XLSX (tables);
vCard, iCalendar ↔ CSV | -| **3D models**: GLB, glTF, STL, OBJ (+ MTL), 3MF, PLY;
FBX read | each other, PNG preview | +| **3D models**: GLB, glTF, STL, OBJ (+ MTL), 3MF, PLY;
FBX, Source engine model (MDL, with its VTF textures) read | each other, PNG preview | | **Geospatial vectors**: Shapefile, GeoJSON, KML, KMZ, GPX
CSV with coordinates; Garmin FIT, OpenStreetMap (.osm) read | each other and CSV / XLSX / JSON;
projected data (UTM, Web Mercator, Lambert, Albers) comes out as WGS 84 longitude / latitude | Unmarked formats work out of the box, without Office, Hancom Office or LibreOffice. diff --git a/docs/ENGINES.md b/docs/ENGINES.md index b2cb79974..18b36017e 100644 --- a/docs/ENGINES.md +++ b/docs/ENGINES.md @@ -1434,8 +1434,8 @@ to draw). G-code is a read-only CAD format (fields: 3D models and CAD). ## 3D models -`Filee.Engines/Models3D` converts GLB, glTF, STL, OBJ, 3MF and PLY into each other, and FBX into any of them, in -managed code: every reader +`Filee.Engines/Models3D` converts GLB, glTF, STL, OBJ, 3MF and PLY into each other, and FBX and Source engine models +(MDL) into any of them, in managed code: every reader fills one in-memory model (`Model3D`: parts with positions, normals, UVs, vertex colours and triangles; materials with a base colour and a base colour texture) and every writer takes it, so each pair is one step. glTF and GLB are read and written with [SharpGLTF](https://github.com/vpenades/SharpGLTF) (MIT); STL, OBJ/MTL, PLY, 3MF and the FBX @@ -1469,6 +1469,7 @@ What each format keeps: | OBJ + MTL | `o` / `g` / `usemtl` as parts, `v` (+ r g b), `vt`, `vn`, faces of any size; `mtllib` with `Kd`, `d` / `Tr`, `map_Kd` (options skipped) next to the file | **the `.mtl` and the textures are written next to the `.obj`** (`name.obj`, `name.mtl`, `name_texture.png`, named like the output); vertex colours as `v x y z r g b` | | 3MF | objects, components (also in other model parts, as Bambu Studio writes them), build transforms, base materials, colour groups (materials extension); `support` / `other` objects skipped | one object per part, base materials from material colours, a colour group from vertex colours | | FBX (read only) | binary 6.x–7.x (64-bit records from 7.5, zlib-compressed arrays) and ASCII; mesh geometry with polygons of any size, the first normal, UV, colour and material layer (by polygon vertex, control point or polygon, or all the same; direct or indexed); the model hierarchy from the connections with translation, rotation (every Euler order, pre- / post-rotation, pivots and offsets), scaling and geometric transforms; property templates from Definitions; material `DiffuseColor` and `Opacity`; the diffuse texture embedded (Video `Content`) or named next to the file; animation, skins, blend shapes, NURBS, cameras and lights skipped (the rest pose is read) | — (not written) | +| MDL (Source engine, read only) | versions 44–49 with the `.vvd` and `.dx90.vtx` (else `.vtx`, `.dx80.vtx`, `.sw.vtx`) beside it (see below) | — (needs Valve's studiomdl) | | PLY | ASCII, binary little / big endian; vertex position, normal, UV (`s`/`t`, `u`/`v`, `texture_u`/`texture_v`), colours of any type; face lists with per-corner `texcoord` and face colours; `comment TextureFile`; other elements skipped | binary little endian, one mesh; material colours become vertex colours | Lost on the way (no place in the target): **STL** keeps only the triangles (no colours, UVs, normals or materials); @@ -1483,6 +1484,25 @@ FBX's texture names) are read only from the model's folder or a folder below it. points anywhere else is looked up by its name in the model's folder instead, so a downloaded model cannot pull a picture from elsewhere on the disk into its output. +**Source engine models** (`.mdl`, #1526, the Games field; `Models3D/SourceModel.cs`): Garry's Mod, Counter-Strike: +Source, Team Fortress 2 and Half-Life 2 models are three files studiomdl writes side by side: the `.mdl` (bones, +textures, texture folders, the skin table, body parts → models → meshes), the `.vvd` (vertices with up to three bone +weights; fixups say which runs make LOD 0) and the `.dx90.vtx` (per mesh, strip groups of triangle lists or strips +indexing the mesh's vertices). All three must have the `.mdl`'s checksum, else the files belong to different builds and +the model is refused; a missing `.vvd` or `.vtx` is named. The model is read in its bind pose with its default body +groups (the first model of each body part), the first skin and LOD 0; animations, flexes, attachments and the physics +mesh (`.phy`) are not read. Source is Z up and measured in inches (25.4 mm), its triangles are clockwise and its UVs +start at the top left, so the model is turned to Filee's Y up, millimetres, counter-clockwise and bottom-left UVs; each +mesh becomes a part named after its model, with one material per texture. Later branches' 8-byte-longer strip groups +(CS:GO) are read when the indices of the usual layout don't fit. +The textures are found as Garry's Mod finds them: in the addon's or game's own layout, the `materials` folder beside the +`models` folder the model is in, `materials//.vmt` for each of the model's texture folders +(`$cdmaterials`), letter case aside, then the material's `$basetexture` (also through a `patch` material's `include`) as +a `.vtf`, decoded by the VTF reader into a PNG texture; `$nocull` makes the material double-sided. Only `.vmt` and +`.vtf` files below that `materials` folder are read, never through `..`. A model outside such a layout, or one whose +materials come from the game itself (the HL2 textures), keeps plain materials. OBJ output writes the textures beside the +`.obj`, GLB embeds them. + **3D → pictures**: a small software rasteriser (`MeshRenderer`) draws an isometric view from the front right, above, with one light and flat shading per triangle, vertex colours or the texture interpolated across it, and a z-buffer, at 2048 px and scaled down to 1024 px for smooth edges, on a transparent background. The preset's image options diff --git a/src/Filee.Core/Formats/FormatFields.cs b/src/Filee.Core/Formats/FormatFields.cs index 94e6aade5..01aed0a92 100644 --- a/src/Filee.Core/Formats/FormatFields.cs +++ b/src/Filee.Core/Formats/FormatFields.cs @@ -83,7 +83,7 @@ .. Enum.GetValues().Select(category => category.ToString().ToLow "dts", "tar", "tgz", "tbz2", "txz", "tar7z", "tlz", "gz", "bz2", "xz", "lz", "lzma", "jar", "zxp", "nupkg", "apks", "aab", "msix", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", "cab", "gerber", "cpio", "deb", "rpm", "dwg", "ttf", "otf", "woff", "woff2", "ass", "ssa", "ttml", "scc", "glb", "gltf", - "stl", "obj", "3mf", "ply", "fbx", "gcode", "geojson", "kml", "kmz", "gpx", "shp", "fit", "osm", "xm", "it", "ani", "vtf", "gma", "vpk", "bsp" + "stl", "obj", "3mf", "ply", "fbx", "gcode", "geojson", "kml", "kmz", "gpx", "shp", "fit", "osm", "xm", "it", "ani", "vtf", "gma", "vpk", "bsp", "mdl" ]), (FormatGrade.Legacy, [ @@ -100,7 +100,7 @@ .. Enum.GetValues().Select(category => category.ToString().ToLow // Layout, print, photo post-production, signs and embroidery. (["psd", "psb", "exr", "hdr", "xcf", "kra", "pdn", "raw", "eps", "ps", "ai", "cdr", "plt", "dst"], ["design"]), // Game textures and cutscenes, Minecraft packs and modpacks. - (["dds", "vtf", "usm", "bik", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", "gma", "vpk", "bsp"], ["game"]), + (["dds", "vtf", "usm", "bik", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", "gma", "vpk", "bsp", "mdl"], ["game"]), // Medical imaging and astronomy. (["dcm", "fits"], ["science"]), // Software work: raw image formats of graphics tools, app icons and cursors, config and data files, flat XML diff --git a/src/Filee.Core/Formats/FormatRegistry.cs b/src/Filee.Core/Formats/FormatRegistry.cs index 6ec8847e1..6233f1568 100644 --- a/src/Filee.Core/Formats/FormatRegistry.cs +++ b/src/Filee.Core/Formats/FormatRegistry.cs @@ -419,6 +419,9 @@ public static class FormatRegistry new("3mf", "3MF", FormatCategory.Model3D, ["3mf"]), new("ply", "PLY", FormatCategory.Model3D, ["ply"]), new("fbx", "FBX", FormatCategory.Model3D, ["fbx"], Writable: false), + // Source engine models (Garry's Mod and other Source games), read with the .vvd and .vtx beside them and their + // textures from the "materials" folder above them. + new("mdl", "Source model (MDL)", FormatCategory.Model3D, ["mdl"], Writable: false), // Geospatial vector data (CSV with coordinates is a Data format and reaches these through the geo engine) new("geojson", "GeoJSON", FormatCategory.Geo, ["geojson"]), diff --git a/src/Filee.Core/Settings/AppSettings.cs b/src/Filee.Core/Settings/AppSettings.cs index 6455d0eed..4ba80706c 100644 --- a/src/Filee.Core/Settings/AppSettings.cs +++ b/src/Filee.Core/Settings/AppSettings.cs @@ -170,7 +170,7 @@ public static bool IsFlipKeyAllowed(string key, IEnumerable gest public sealed class AppSettings { /// Current schema version. Increase when the format changes and add a migration. - public const int CurrentSchemaVersion = 99; + public const int CurrentSchemaVersion = 100; public int SchemaVersion { get; set; } = CurrentSchemaVersion; diff --git a/src/Filee.Core/Settings/UserDataStore.cs b/src/Filee.Core/Settings/UserDataStore.cs index e963a79db..942a61128 100644 --- a/src/Filee.Core/Settings/UserDataStore.cs +++ b/src/Filee.Core/Settings/UserDataStore.cs @@ -1193,6 +1193,9 @@ public static bool ApplyToLibrary(int from, List presets, ListThe 3D formats, each readable and writable. public static readonly string[] Formats = ["glb", "gltf", "stl", "obj", "3mf", "ply"]; - /// Every 3D format that can be read: and FBX. - public static readonly string[] InputFormats = [.. Formats, "fbx"]; + /// Every 3D format that can be read: , FBX and Source engine models (MDL). + public static readonly string[] InputFormats = [.. Formats, "fbx", "mdl"]; /// Long edge of the preview picture before the preset's resize. internal const int PreviewSize = 1024; @@ -42,7 +42,7 @@ public sealed class ModelConverter(ILogger? logger = null) : IConverter, IImageT .. from source in InputFormats from target in ImageEncoder.Writable select ImageEncoder.Edge(source, target), ]; - public EngineStatus GetStatus() => EngineStatus.Available("GLB, glTF, STL, OBJ, 3MF, PLY ↔ each other; FBX → each of them", + public EngineStatus GetStatus() => EngineStatus.Available("GLB, glTF, STL, OBJ, 3MF, PLY ↔ each other; FBX, MDL → each of them", $"{EngineVersions.BuiltIn} · {EngineVersions.Library("SharpGLTF", typeof(SharpGLTF.Schema2.ModelRoot))}"); public Task> ConvertAsync(ConversionStep step, IProgress? progress, CancellationToken cancellationToken) => @@ -80,6 +80,7 @@ internal static Model3D Read(string path, string format, IProgress? prog "3mf" => ThreeMfFormat.Read(path, progress, cancellationToken), "glb" or "gltf" => GltfFormat.Read(path, progress, cancellationToken), "fbx" => FbxFormat.Read(path, progress, cancellationToken), + "mdl" => SourceModel.Load(path, cancellationToken).ToModel3D(path, cancellationToken), _ => throw new NotSupportedException($"Unknown 3D format '{format}'."), }; model.Validate(); diff --git a/src/Filee.Engines/Models3D/SourceModel.cs b/src/Filee.Engines/Models3D/SourceModel.cs new file mode 100644 index 000000000..9d64659cf --- /dev/null +++ b/src/Filee.Engines/Models3D/SourceModel.cs @@ -0,0 +1,622 @@ +// Source engine models (.mdl, versions 44 to 49: Garry's Mod, Counter-Strike: Source, Team Fortress 2, Half-Life 2), read +// by Filee with the two files studiomdl writes beside them: the vertices (.vvd) and the triangles (.dx90.vtx, else .vtx, +// .dx80.vtx or .sw.vtx). Layouts (little endian), as in Valve's studio.h / optimize.h: +// +// .mdl studiohdr_t (408 bytes): bones (216 bytes each: name, parent, position, rotation), textures (64 bytes each: +// name), texture folders ($cdmaterials), the skin table (shorts: family × reference), body parts (16 bytes: name, +// models) → models (148 bytes: name, meshes, first vertex) → meshes (116 bytes: material, vertex offset) +// .vvd "IDSV", version 4, the checksum of its .mdl, vertices per LOD, fixups (which runs of vertices LOD 0 uses); +// 48 bytes a vertex: three bone weights, three bones, bone count, position, normal, UV +// .vtx version 7: body parts → models → LODs → meshes (9 bytes) → strip groups (25 bytes; 33 in later branches) +// → their vertices (9 bytes: the index of the mesh's vertex) and triangle lists (or strips) of u16 +// +// The model is read in its bind pose with the default body groups (the first model of each body part), the first skin +// and LOD 0; animations, flexes and physics (.phy) are not read. Source is Z up, one unit an inch, triangles clockwise, +// UVs from the top left: Filee turns that into its own conventions (Y up, millimetres, counter-clockwise, bottom left). +// The textures are found as Garry's Mod finds them: the model's material (a .vmt) in a "materials" folder above the +// model, its $basetexture (a .vtf) decoded by the VTF reader. + +using System.Buffers.Binary; +using System.Numerics; +using System.Runtime.ExceptionServices; +using System.Text; +using Filee.Core.Conversion; +using Filee.Engines.Magick; +using ImageMagick; + +namespace Filee.Engines.Models3D; + +/// A Source engine model as read from its .mdl, .vvd and .vtx. +internal sealed class SourceModel +{ + /// One inch, Source's unit, in Filee's millimetres. + public const float MillimetresPerUnit = 25.4f; + + /// The largest .mdl, .vvd or .vtx Filee reads (a damaged or made-up file otherwise). + private const long MaxFileSize = 256L * 1024 * 1024; + + private const int MaxBones = 1024; + private const int MaxTextures = 4096; + private const int MaxBodyParts = 512; + private const int MaxMeshes = 16384; + private const int MaxVertices = 4_000_000; + + /// A bone in its bind pose, relative to its parent (Source's axes and units). + internal readonly record struct Bone(string Name, int Parent, Vector3 Position, Vector3 Rotation); + + /// A vertex in Source's axes and units, UV from the top left, with up to three bone weights. + internal readonly record struct Vertex(Vector3 Position, Vector3 Normal, Vector2 Uv, int BoneCount, + int Bone0, int Bone1, int Bone2, float Weight0, float Weight1, float Weight2); + + /// One mesh: its part name, material and triangles (indices into , Source's winding). + internal sealed record Mesh(string Part, string Material, List Triangles); + + public string Name { get; private init; } = ""; + public List Bones { get; } = []; + public List Vertices { get; } = []; + public List Meshes { get; } = []; + + /// The folders the model's materials are looked up in ($cdmaterials, "/" separated, without the end slash). + public List MaterialFolders { get; } = []; + + /// The most triangles Filee reads from one model, over all its meshes (as many as an SMD may have). + public const int MaxTriangles = 10_000_000; + + /// + /// Reads the model at with its .vvd and .vtx, refusing one with more than + /// triangles. + /// + public static SourceModel Load(string mdlPath, CancellationToken cancellationToken, int maxTriangles = MaxTriangles) + { + var mdl = new Bytes(ReadFile(mdlPath), Path.GetFileName(mdlPath)); + var basePath = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(mdlPath))!, Path.GetFileNameWithoutExtension(mdlPath)); + var vvdPath = Companion(basePath, [".vvd"]) + ?? throw new FileNotFoundException($"{Path.GetFileName(mdlPath)} needs its vertex file {Path.GetFileName(basePath)}.vvd next to it."); + var vtxPath = Companion(basePath, [".dx90.vtx", ".vtx", ".dx80.vtx", ".sw.vtx"]) + ?? throw new FileNotFoundException($"{Path.GetFileName(mdlPath)} needs its triangle file {Path.GetFileName(basePath)}.dx90.vtx next to it."); + var vvd = new Bytes(ReadFile(vvdPath), Path.GetFileName(vvdPath)); + var vtx = new Bytes(ReadFile(vtxPath), Path.GetFileName(vtxPath)); + + if (!mdl.Data.AsSpan().StartsWith("IDST"u8)) + throw new InvalidDataException($"{mdl.File} is not a Source engine model (IDST)."); + var version = mdl.I32(4); + if (version is < 44 or > 49) + throw new InvalidDataException($"{mdl.File} is a model of version {version}; Filee reads Source engine models of versions 44 to 49."); + var checksum = mdl.I32(8); + var model = new SourceModel { Name = mdl.String(12, 64) }; + + ReadBones(mdl, model); + var textures = ReadTextures(mdl, model); + ReadVertices(vvd, checksum, model); + cancellationToken.ThrowIfCancellationRequested(); + if (!vtx.Data.AsSpan().StartsWith((ReadOnlySpan)[7, 0, 0, 0])) + throw new InvalidDataException($"{vtx.File} is not a triangle file of version 7."); + if (vtx.I32(16) != checksum) + throw new InvalidDataException($"{vtx.File} belongs to another build of {mdl.File}: its checksum differs."); + // Strip groups and strips grew by 8 bytes in later branches (CS:GO); the layout whose indices all fit is used. + // A model too large in the first layout stays refused when the second one doesn't fit it at all. + try + { + ReadMeshes(mdl, vtx, textures, model, extended: false, new TriangleBudget(maxTriangles), cancellationToken); + } + catch (Exception first) when (first is InvalidDataException or ConversionRefusedException) + { + model.Meshes.Clear(); + try + { + ReadMeshes(mdl, vtx, textures, model, extended: true, new TriangleBudget(maxTriangles), cancellationToken); + } + catch (InvalidDataException) when (first is ConversionRefusedException) + { + ExceptionDispatchInfo.Throw(first); + } + } + if (model.Meshes.Count == 0) + throw new InvalidDataException($"{mdl.File} has no triangles in its default body groups."); + return model; + } + + private static byte[] ReadFile(string path) + { + var length = new FileInfo(path).Length; + if (length > MaxFileSize) + throw new InvalidDataException($"{Path.GetFileName(path)} is too large to be a Source engine model file."); + return File.ReadAllBytes(path); + } + + /// The first of " + suffix" that exists, letter case aside. + private static string? Companion(string basePath, string[] suffixes) + { + var folder = Path.GetDirectoryName(basePath)!; + var name = Path.GetFileName(basePath); + var files = Directory.EnumerateFiles(folder).ToList(); + foreach (var suffix in suffixes) + { + var match = files.FirstOrDefault(f => string.Equals(Path.GetFileName(f), name + suffix, StringComparison.OrdinalIgnoreCase)); + if (match is not null) + return match; + } + return null; + } + + private static void ReadBones(Bytes mdl, SourceModel model) + { + var count = mdl.Count(156, MaxBones, "bones"); + var start = mdl.I32(160); + for (var i = 0; i < count; i++) + { + var at = start + i * 216; + var parent = mdl.I32(at + 4); + model.Bones.Add(new Bone(mdl.String(at + mdl.I32(at)), parent is >= -1 && parent < count ? parent : -1, + mdl.Vector(at + 32), mdl.Vector(at + 60))); + } + } + + /// The texture name of every skin reference of the first skin family. + private static List ReadTextures(Bytes mdl, SourceModel model) + { + var count = mdl.Count(204, MaxTextures, "textures"); + var start = mdl.I32(208); + var names = new List(count); + for (var i = 0; i < count; i++) + { + var at = start + i * 64; + names.Add(mdl.String(at + mdl.I32(at)).Replace('\\', '/')); + } + + var folders = mdl.Count(212, 256, "texture folders"); + var folderTable = mdl.I32(216); + for (var i = 0; i < folders; i++) + model.MaterialFolders.Add(mdl.String(mdl.I32(folderTable + i * 4)).Replace('\\', '/').Trim('/')); + + // The skin table's first row maps each mesh's material reference to a texture. + var references = mdl.Count(220, MaxTextures, "skin references"); + var families = mdl.Count(224, MaxTextures, "skins"); + var skinTable = mdl.I32(228); + var skin = new List(references); + for (var i = 0; i < references; i++) + { + var texture = families > 0 ? mdl.I16(skinTable + i * 2) : i; + skin.Add(texture >= 0 && texture < names.Count ? names[texture] : i < names.Count ? names[i] : ""); + } + return skin.Count > 0 ? skin : names; + } + + private static void ReadVertices(Bytes vvd, int checksum, SourceModel model) + { + if (!vvd.Data.AsSpan().StartsWith("IDSV"u8)) + throw new InvalidDataException($"{vvd.File} is not a Source engine vertex file (IDSV)."); + if (vvd.I32(8) != checksum) + throw new InvalidDataException($"{vvd.File} belongs to another build of the model: its checksum differs."); + var lod0 = vvd.Count(16, MaxVertices, "vertices"); + var fixups = vvd.Count(48, MaxVertices, "fixups"); + var fixupTable = vvd.I32(52); + var vertexData = vvd.I32(56); + // The tables are checked against the file before a single vertex is read or added. + bool Fits(long offset, long count, int size) => offset >= 0 && count >= 0 && offset + count * size <= vvd.Data.Length; + if (!Fits(vertexData, lod0, 48) || !Fits(fixupTable, fixups, 12)) + throw vvd.Damaged("the vertex table"); + + void Add(int index) + { + var at = vertexData + index * 48; + model.Vertices.Add(new Vertex(vvd.Vector(at + 16), vvd.Vector(at + 28), new Vector2(vvd.F32(at + 40), vvd.F32(at + 44)), + Math.Clamp((int)vvd.Byte(at + 15), 1, 3), vvd.Byte(at + 12), vvd.Byte(at + 13), vvd.Byte(at + 14), + vvd.F32(at), vvd.F32(at + 4), vvd.F32(at + 8))); + } + + if (fixups == 0) + { + for (var i = 0; i < lod0; i++) + Add(i); + return; + } + // With fixups, LOD 0 is every run of vertices whose LOD is 0 or above, in order. + for (var f = 0; f < fixups; f++) + { + var at = fixupTable + f * 12; + var lod = vvd.I32(at); + var source = vvd.I32(at + 4); + var length = vvd.I32(at + 8); + if (lod < 0) + continue; + if (source < 0 || length < 0 || model.Vertices.Count + length > MaxVertices || !Fits(vertexData, (long)source + length, 48)) + throw vvd.Damaged("a vertex fixup"); + for (var i = 0; i < length; i++) + Add(source + i); + } + } + + /// + /// The triangles a model may have in all. Strip groups and strips are reached through offsets, so many of them can + /// point at the same table; every triangle is counted as it is added, wherever it comes from. + /// + internal sealed class TriangleBudget(int max) + { + private long _used; + + public void Take(long triangles) + { + _used += triangles; + if (_used > max) + throw new ConversionRefusedException("error.document_too_large", $"more than {max} triangles"); + } + } + + private static void ReadMeshes(Bytes mdl, Bytes vtx, List materials, SourceModel model, bool extended, + TriangleBudget budget, CancellationToken cancellationToken) + { + var stripGroupSize = extended ? 33 : 25; + var stripSize = extended ? 35 : 27; + var bodyParts = Math.Min(mdl.Count(232, MaxBodyParts, "body parts"), vtx.Count(28, MaxBodyParts, "body parts")); + var mdlBodyParts = mdl.I32(236); + var vtxBodyParts = vtx.I32(32); + for (var b = 0; b < bodyParts; b++) + { + var mdlPart = mdlBodyParts + b * 16; + var vtxPart = vtxBodyParts + b * 8; + if (mdl.I32(mdlPart + 4) <= 0 || vtx.I32(vtxPart) <= 0) + continue; + // The default body group: the first model of the part. + var mdlModel = mdlPart + mdl.I32(mdlPart + 12); + var vtxModel = vtxPart + vtx.I32(vtxPart + 4); + var modelName = mdl.String(mdlModel, 64); + var firstVertex = mdl.I32(mdlModel + 84) / 48; + var meshes = mdl.Count(mdlModel + 72, MaxMeshes, "meshes"); + var mdlMeshes = mdlModel + mdl.I32(mdlModel + 76); + if (vtx.I32(vtxModel) <= 0) + continue; + var lod = vtxModel + vtx.I32(vtxModel + 4); // LOD 0 + meshes = Math.Min(meshes, vtx.Count(lod, MaxMeshes, "meshes")); + var vtxMeshes = lod + vtx.I32(lod + 4); + + for (var m = 0; m < meshes; m++) + { + cancellationToken.ThrowIfCancellationRequested(); + var mdlMesh = mdlMeshes + m * 116; + var material = mdl.I32(mdlMesh); + var meshFirst = firstVertex + mdl.I32(mdlMesh + 12); + var vtxMesh = vtxMeshes + m * 9; + var groups = vtx.Count(vtxMesh, MaxMeshes, "strip groups"); + var groupTable = vtxMesh + vtx.I32(vtxMesh + 4); + var triangles = new List(); + for (var g = 0; g < groups; g++) + { + var group = groupTable + g * stripGroupSize; + var vertexCount = vtx.Count(group, MaxVertices, "vertices"); + var vertexTable = group + vtx.I32(group + 4); + var indexCount = vtx.Count(group + 8, MaxVertices * 3, "indices"); + var indexTable = group + vtx.I32(group + 12); + var strips = vtx.Count(group + 16, MaxMeshes, "strips"); + var stripTable = group + vtx.I32(group + 20); + + int MeshVertex(int index) + { + if ((uint)index >= (uint)vertexCount) + throw vtx.Damaged("a triangle"); + var vertex = meshFirst + vtx.U16(vertexTable + index * 9 + 4); + if ((uint)vertex >= (uint)model.Vertices.Count) + throw vtx.Damaged("a triangle"); + return vertex; + } + + var anyStrip = false; + for (var s = 0; s < strips; s++) + { + var strip = stripTable + s * stripSize; + if ((vtx.Byte(strip + 18) & 0x02) == 0) + continue; + // A triangle strip: every next index makes a triangle with the two before it, alternating. + anyStrip = true; + var first = vtx.I32(strip + 4); + var count = vtx.I32(strip); + if (first < 0 || count < 0 || first + count > indexCount) + throw vtx.Damaged("a triangle strip"); + for (var i = 2; i < count; i++) + { + var a = MeshVertex(vtx.U16(indexTable + (first + i - 2) * 2)); + var c = MeshVertex(vtx.U16(indexTable + (first + i - 1) * 2)); + var d = MeshVertex(vtx.U16(indexTable + (first + i) * 2)); + if (a == c || c == d || a == d) + continue; + budget.Take(1); + if (i % 2 == 0) + triangles.AddRange([a, c, d]); + else + triangles.AddRange([c, a, d]); + } + } + if (anyStrip) + continue; + if (indexCount % 3 != 0) + throw vtx.Damaged("a triangle list"); + budget.Take(indexCount / 3); + for (var i = 0; i < indexCount; i++) + triangles.Add(MeshVertex(vtx.U16(indexTable + i * 2))); + } + if (triangles.Count == 0) + continue; + var name = material >= 0 && material < materials.Count ? materials[material] : $"material{material}"; + model.Meshes.Add(new Mesh(modelName, name, triangles)); + } + } + } + + /// + /// The model in Filee's conventions: one part per mesh, its textures looked up in the "materials" folder above the + /// model (none when there is no such folder or the game's own textures are meant). + /// + public Model3D ToModel3D(string mdlPath, CancellationToken cancellationToken) + { + var result = new Model3D(); + var textures = new SourceMaterials(mdlPath, MaterialFolders); + var materialIndex = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var mesh in Meshes) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!materialIndex.TryGetValue(mesh.Material, out var index)) + { + index = result.Materials.Count; + materialIndex[mesh.Material] = index; + var (texture, doubleSided) = textures.Find(mesh.Material); + result.Materials.Add(new Material3D + { + Name = Path.GetFileName(mesh.Material), + BaseColorTexture = texture, + DoubleSided = doubleSided, + }); + } + + var part = new MeshPart { Name = mesh.Part, Material = index, Normals = [], TexCoords = [] }; + var local = new Dictionary(); + int Local(int vertex) + { + if (local.TryGetValue(vertex, out var at)) + return at; + var v = Vertices[vertex]; + at = part.AddVertex(MeshMath.ToYUp(v.Position) * MillimetresPerUnit, MeshMath.ToYUp(v.Normal), new Vector2(v.Uv.X, 1 - v.Uv.Y)); + local[vertex] = at; + return at; + } + // Source's triangles are clockwise: reversed, they face the same way counter-clockwise. + for (var t = 0; t + 2 < mesh.Triangles.Count; t += 3) + part.AddTriangle(Local(mesh.Triangles[t + 2]), Local(mesh.Triangles[t + 1]), Local(mesh.Triangles[t])); + result.Parts.Add(part); + } + return result; + } + + /// A model file's bytes with bounds-checked little-endian reads. + private sealed class Bytes(byte[] data, string file) + { + public byte[] Data { get; } = data; + public string File { get; } = file; + + public InvalidDataException Damaged(string what) => new($"{File} is damaged ({what} points outside the file)."); + + private ReadOnlySpan At(int offset, int length) => + offset >= 0 && (long)offset + length <= Data.Length ? Data.AsSpan(offset, length) : throw Damaged("an offset"); + + public int I32(int offset) => BinaryPrimitives.ReadInt32LittleEndian(At(offset, 4)); + public int I16(int offset) => BinaryPrimitives.ReadInt16LittleEndian(At(offset, 2)); + public int U16(int offset) => BinaryPrimitives.ReadUInt16LittleEndian(At(offset, 2)); + public byte Byte(int offset) => At(offset, 1)[0]; + public float F32(int offset) => BinaryPrimitives.ReadSingleLittleEndian(At(offset, 4)); + public Vector3 Vector(int offset) => new(F32(offset), F32(offset + 4), F32(offset + 8)); + + /// A count at , at most . + public int Count(int offset, int max, string what) + { + var count = I32(offset); + if (count < 0 || count > max) + throw new InvalidDataException($"{File} is damaged (it claims {count} {what})."); + return count; + } + + /// A NUL-terminated name at , at most bytes. + public string String(int offset, int max = 1024) + { + if (offset < 0 || offset >= Data.Length) + throw Damaged("a name"); + var span = Data.AsSpan(offset, Math.Min(max, Data.Length - offset)); + var end = span.IndexOf((byte)0); + return Encoding.UTF8.GetString(end < 0 ? span : span[..end]); + } + } +} + +/// +/// Finds a Source model's textures as Garry's Mod does: "materials/<texture folder>/<texture>.vmt" beside +/// the "models" folder the model is in, then that material's $basetexture as a .vtf. Only .vmt and .vtf files below that +/// "materials" folder are read, never through "..". +/// +internal sealed class SourceMaterials +{ + /// How far above the model its "models" folder is looked for (models/a/b/c/model.mdl is four). + private const int MaxLevels = 10; + + private readonly string? _root; + private readonly IReadOnlyList _folders; + private readonly Dictionary _textures = new(StringComparer.OrdinalIgnoreCase); + + public SourceMaterials(string mdlPath, IReadOnlyList folders) + { + _folders = folders.Count > 0 ? folders : [""]; + // Only the game's or addon's own layout: a "models" folder above the model, with "materials" beside it. + var folder = new DirectoryInfo(Path.GetDirectoryName(Path.GetFullPath(mdlPath))!); + for (var level = 0; level < MaxLevels && folder.Parent is { } parent; level++, folder = parent) + { + if (!folder.Name.Equals("models", StringComparison.OrdinalIgnoreCase)) + continue; + try + { + _root = parent.EnumerateDirectories() + .FirstOrDefault(d => d.Name.Equals("materials", StringComparison.OrdinalIgnoreCase))?.FullName; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // a folder Filee may not list: no textures + } + break; + } + } + + /// The texture of material , and whether the material is drawn from both sides. + public (Texture3D? Texture, bool DoubleSided) Find(string name) + { + if (_root is null) + return (null, false); + foreach (var folder in _folders) + { + var relative = folder.Length > 0 ? folder + "/" + name : name; + if (Resolve(relative + ".vmt") is { } vmt) + { + var material = Vmt.Read(vmt, path => Resolve(path.StartsWith("materials/", StringComparison.OrdinalIgnoreCase) ? path[10..] : path)); + var texture = material.BaseTexture is { } baseTexture ? Texture(StripExtension(baseTexture) + ".vtf") : null; + return (texture, material.NoCull); + } + if (Resolve(relative + ".vtf") is not null) + return (Texture(relative + ".vtf"), false); + } + return (null, false); + } + + private static string StripExtension(string path) => + path.EndsWith(".vtf", StringComparison.OrdinalIgnoreCase) ? path[..^4] : path; + + /// The decoded texture at (below "materials"), as PNG, once per file. + private Texture3D? Texture(string relative) + { + if (_textures.TryGetValue(relative, out var known)) + return known; + Texture3D? texture = null; + if (Resolve(relative) is { } path) + { + try + { + using var frames = Vtf.Read(path); + if (frames.Count > 0) + texture = new Texture3D(frames[0].ToByteArray(MagickFormat.Png), "png", Path.GetFileNameWithoutExtension(path)); + } + catch (Exception ex) when (ex is InvalidDataException or MagickException or IOException) + { + // A texture Filee can't read leaves the material plain. + } + } + _textures[relative] = texture; + return texture; + } + + /// + /// The file at below "materials" ("/" or "\" separated), letter case aside (Source's file + /// system ignores it), or null. + /// + private string? Resolve(string relative) + { + var parts = relative.Replace('\\', '/').Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0 || parts.Any(p => p is "." or "..")) + return null; + var current = _root!; + for (var i = 0; i < parts.Length; i++) + { + var exact = Path.Combine(current, parts[i]); + var last = i == parts.Length - 1; + if (last ? File.Exists(exact) : Directory.Exists(exact)) + { + current = exact; + continue; + } + var entries = last ? Directory.EnumerateFiles(current) : Directory.EnumerateDirectories(current); + var match = entries.FirstOrDefault(e => string.Equals(Path.GetFileName(e), parts[i], StringComparison.OrdinalIgnoreCase)); + if (match is null) + return null; + current = match; + } + return current; + } +} + +/// The few things Filee reads from a Valve material (.vmt, KeyValues text): $basetexture and $nocull. +internal static class Vmt +{ + internal sealed record Material(string? BaseTexture, bool NoCull); + + /// + /// Reads ; a "patch" material takes what it doesn't set from the material it includes + /// (found through , one level). + /// + public static Material Read(string path, Func resolve) + { + var tokens = Tokens(Text(path)); + var material = Parse(tokens); + if (tokens.Count > 0 && tokens[0].Equals("patch", StringComparison.OrdinalIgnoreCase) + && Value(tokens, "include") is { } include && include.EndsWith(".vmt", StringComparison.OrdinalIgnoreCase) + && resolve(include.Replace('\\', '/')) is { } included && included != path) + { + var parent = Parse(Tokens(Text(included))); + material = new Material(material.BaseTexture ?? parent.BaseTexture, material.NoCull || parent.NoCull); + } + return material; + } + + /// The largest material file Filee reads; real ones are a few hundred bytes. + public const long MaxBytes = 1024 * 1024; + + /// The text of a material, or "" when it is larger than (the material stays plain). + private static string Text(string path) => new FileInfo(path).Length > MaxBytes ? "" : File.ReadAllText(path); + + private static Material Parse(List tokens) => + new(Value(tokens, "$basetexture"), Value(tokens, "$nocull") is "1"); + + /// The value after the first key , at any depth. + private static string? Value(List tokens, string key) + { + for (var i = 0; i + 1 < tokens.Count; i++) + { + if (tokens[i].Equals(key, StringComparison.OrdinalIgnoreCase) && tokens[i + 1] is not ("{" or "}")) + return tokens[i + 1]; + } + return null; + } + + /// KeyValues tokens: quoted or bare words and braces; "//" comments are skipped. + internal static List Tokens(string text) + { + var tokens = new List(); + var i = 0; + while (i < text.Length) + { + var c = text[i]; + if (char.IsWhiteSpace(c)) + { + i++; + } + else if (c == '/' && i + 1 < text.Length && text[i + 1] == '/') + { + while (i < text.Length && text[i] != '\n') + i++; + } + else if (c is '{' or '}') + { + tokens.Add(c.ToString()); + i++; + } + else if (c == '"') + { + var end = text.IndexOf('"', i + 1); + if (end < 0) + end = text.Length; + tokens.Add(text[(i + 1)..end]); + i = end + 1; + } + else + { + var start = i; + while (i < text.Length && !char.IsWhiteSpace(text[i]) && text[i] is not ('{' or '}' or '"')) + i++; + tokens.Add(text[start..i]); + } + } + return tokens; + } +} diff --git a/tests/Filee.Core.Tests/FormatFieldsTests.cs b/tests/Filee.Core.Tests/FormatFieldsTests.cs index 3cc6e8466..7c61bd9bd 100644 --- a/tests/Filee.Core.Tests/FormatFieldsTests.cs +++ b/tests/Filee.Core.Tests/FormatFieldsTests.cs @@ -30,7 +30,7 @@ private static string[] Hidden(IReadOnlySet turnedOff) => "dff", "dot", "dsf", "dss", "dst", "dts", "dv", "dwg", "eac3", "eot", "eps", "exr", "fbx", "fit", "fits", "flc", "flv", "fodg", "fodp", "fods", "fodt", "fon", "gcode", "geojson", "gerber", "glb", "gltf", "gma", "gpx", "gz", "h264", "h265", "har", "hdr", "htmlz", "icns", "ipynb", "it", "jar", "jp2", "jsonl", "kml", "kmz", "kra", "lha", "lit", "lrf", "lwp", "lz", - "lzma", "m4s", "man", "mcaddon", "mcpack", "mctemplate", "mcw", "mcworld", "mediawiki", "miff", "mng", "mobi", + "lzma", "m4s", "man", "mcaddon", "mcpack", "mctemplate", "mcw", "mcworld", "mdl", "mediawiki", "miff", "mng", "mobi", "mod", "mp2", "mpc", "mrpack", "msix", "mxf", "nov", "nupkg", "obj", "oeb", "org", "osm", "otf", "pam", "pcd", "pcx", "pdb", "pdn", "pfm", "plist", "plt", "ply", "pml", "ppm", "prc", "ps", "psb", "psd", "psw", "qoi", "ras", "raw", "rb", "rm", "rpm", "rst", "s3m", "scc", "sda", "sdc", "sdw", "sgi", "shp", "snb", "spx", @@ -45,7 +45,7 @@ private static string[] Hidden(IReadOnlySet turnedOff) => { "model3d", ["fbx", "glb", "gltf", "ply"] }, { "geo", ["fit", "geojson", "gpx", "kml", "kmz", "osm", "shp"] }, { "font", ["otf", "ttf", "woff", "woff2"] }, - { "game", ["bik", "bsp", "dds", "gma", "mcaddon", "mcpack", "mctemplate", "mcworld", "mrpack", "usm", "vpk", "vtf"] }, + { "game", ["bik", "bsp", "dds", "gma", "mcaddon", "mcpack", "mctemplate", "mcworld", "mdl", "mrpack", "usm", "vpk", "vtf"] }, { "design", ["ai", "cdr", "dst", "eps", "exr", "hdr", "kra", "pdn", "plt", "ps", "psb", "psd", "raw", "xcf"] }, { "science", ["dcm", "fits"] }, { "developer", [ @@ -72,7 +72,7 @@ private static string[] Hidden(IReadOnlySet turnedOff) => { "model3d", ["3mf", "fbx", "gcode", "glb", "gltf", "obj", "ply", "stl"] }, { "geo", ["fit", "geojson", "gpx", "kml", "kmz", "osm", "shp"] }, { "font", ["eot", "fon", "otf", "ttf", "woff", "woff2"] }, - { "game", ["bik", "bsp", "dds", "flc", "gma", "it", "mcaddon", "mcpack", "mctemplate", "mcworld", "mrpack", "tga", "usm", "vpk", "vtf", "xm"] }, + { "game", ["bik", "bsp", "dds", "flc", "gma", "it", "mcaddon", "mcpack", "mctemplate", "mcworld", "mdl", "mrpack", "tga", "usm", "vpk", "vtf", "xm"] }, { "design", ["ai", "cdr", "dst", "eps", "exr", "hdr", "jp2", "kra", "pcd", "pdn", "plt", "ps", "psb", "psd", "raw", "xcf"] }, { "science", ["dcm", "fits", "ipynb", "jp2"] }, { "developer", [ @@ -145,7 +145,7 @@ public void Grade_counts_match_the_published_table() { [FormatGrade.Everyday] = 19, [FormatGrade.General] = 99, - [FormatGrade.Specialist] = 131, + [FormatGrade.Specialist] = 132, [FormatGrade.Legacy] = 52, }, counts); diff --git a/tests/Filee.Core.Tests/SourceFormatTests.cs b/tests/Filee.Core.Tests/SourceFormatTests.cs index 1e095e797..c005fd27f 100644 --- a/tests/Filee.Core.Tests/SourceFormatTests.cs +++ b/tests/Filee.Core.Tests/SourceFormatTests.cs @@ -1,6 +1,6 @@ // Source engine files (#1526): Garry's Mod addons, Valve paks and maps (for their packed files) open the archives donut, -// which existing libraries learn without taking a donut of the user's own; they unpack only where the Games field is -// on. +// which existing libraries learn without taking a donut of the user's own, and Source models the 3D models donut; each +// is a target only where the Games field is on. using Filee.Core.Formats; using Filee.Core.Presets; @@ -16,6 +16,7 @@ public class SourceFormatTests [InlineData("WIRE.GMA", "gma", "archives")] [InlineData("pak01_dir.vpk", "vpk", "archives")] [InlineData("gm_construct.bsp", "bsp", "archives")] + [InlineData("gate_e2.mdl", "mdl", "models")] public void Source_files_use_their_donut(string file, string format, string donut) { Assert.Same(FormatRegistry.Get(format), FormatRegistry.Detect(file)); @@ -26,6 +27,7 @@ public void Source_files_use_their_donut(string file, string format, string donu [InlineData("gma", FormatCategory.Archive, true)] [InlineData("vpk", FormatCategory.Archive, false)] [InlineData("bsp", FormatCategory.Archive, false)] + [InlineData("mdl", FormatCategory.Model3D, false)] public void Source_files_are_game_formats(string id, FormatCategory category, bool writable) { var format = FormatRegistry.Get(id)!; @@ -71,4 +73,23 @@ public void Version_96_libraries_get_the_gma_preset_after_the_game_packs() // Not on a donut. Assert.DoesNotContain(BuiltInData.CreateProfiles(), p => p.PresetIds.Contains("to-gma") || p.BackPresetIds?.Contains("to-gma") == true); } + + [Fact] + public void Version_99_libraries_learn_source_models_without_taking_custom_claims() + { + var presets = BuiltInData.CreatePresets(); + var profiles = BuiltInData.CreateProfiles(); + var models = profiles.Single(p => p.Id == "models"); + models.Extensions.RemoveAll(e => e is "mdl"); + + Assert.True(SettingsMigrations.ApplyToLibrary(99, presets, profiles)); + Assert.Contains("mdl", models.Extensions); + Assert.Equal(BuiltInData.CreatePresets().Select(p => p.Id), presets.Select(p => p.Id)); // no preset added + Assert.False(SettingsMigrations.ApplyToLibrary(99, presets, profiles)); + + models.Extensions.Remove("mdl"); + profiles.Insert(0, new ToolbarProfile { Id = "my-models", Extensions = ["MDL"] }); + Assert.False(SettingsMigrations.ApplyToLibrary(99, presets, profiles)); + Assert.DoesNotContain("mdl", models.Extensions); + } } diff --git a/tests/Filee.Core.Tests/UserDataStoreTests.cs b/tests/Filee.Core.Tests/UserDataStoreTests.cs index 887570306..f2081cf41 100644 --- a/tests/Filee.Core.Tests/UserDataStoreTests.cs +++ b/tests/Filee.Core.Tests/UserDataStoreTests.cs @@ -1060,7 +1060,7 @@ public void Version_14_libraries_get_3d_model_presets_and_their_donut_after_data Assert.Equal(ids.IndexOf("models") + 1, ids.IndexOf("geo")); // and, from schema 25, the geospatial donut Assert.Equal(ids.IndexOf("mixed") - 1, ids.IndexOf("geo")); var models = profiles.Single(p => p.Id == "models"); - Assert.Equal(["glb", "gltf", "stl", "obj", "3mf", "ply", "fbx"], models.Extensions); + Assert.Equal(["glb", "gltf", "stl", "obj", "3mf", "ply", "fbx", "mdl"], models.Extensions); Assert.Equal(BuiltInData.ModelProfile().PresetIds, models.PresetIds); Assert.False(SettingsMigrations.ApplyToLibrary(14, presets, profiles)); // once only } @@ -1074,7 +1074,7 @@ public void The_3d_donut_leaves_out_extensions_a_profile_of_the_user_takes() Assert.True(SettingsMigrations.ApplyToLibrary(14, presets, profiles)); - Assert.Equal(["glb", "gltf", "obj", "ply", "fbx"], profiles.Single(p => p.Id == "models").Extensions); + Assert.Equal(["glb", "gltf", "obj", "ply", "fbx", "mdl"], profiles.Single(p => p.Id == "models").Extensions); } [Fact] @@ -1088,7 +1088,7 @@ public void Version_21_libraries_learn_fbx_on_the_3d_donut_and_nothing_else() Assert.True(SettingsMigrations.ApplyToLibrary(21, presets, profiles)); - Assert.Equal(["glb", "gltf", "stl", "obj", "3mf", "fbx"], models.Extensions); + Assert.Equal(["glb", "gltf", "stl", "obj", "3mf", "mdl", "fbx"], models.Extensions); Assert.False(SettingsMigrations.ApplyToLibrary(21, presets, profiles)); // once only // A donut of the user that already opens FBX files keeps them. diff --git a/tests/Filee.Engines.Tests/SourceModelBuilder.cs b/tests/Filee.Engines.Tests/SourceModelBuilder.cs new file mode 100644 index 000000000..943fce31b --- /dev/null +++ b/tests/Filee.Engines.Tests/SourceModelBuilder.cs @@ -0,0 +1,261 @@ +// Builds Source engine models (.mdl version 48 with its .vvd and .dx90.vtx) byte by byte for the tests: bones, textures +// with one texture folder, one body part with one model, and one mesh per material, each a triangle list in one strip +// group. Layouts as in Valve's studio.h and optimize.h (see Models3D/SourceModel.cs). + +using System.Buffers.Binary; +using System.Numerics; +using System.Text; + +namespace Filee.Engines.Tests; + +internal static class SourceModelBuilder +{ + /// A vertex in Source's axes and units (inches, Z up), UV from the top left, weighted to up to two bones. + public sealed record Vertex(Vector3 Position, Vector3 Normal, Vector2 Uv, int Bone = 0, int Bone2 = -1, float Weight2 = 0); + + /// A mesh: its material (an index into the textures), vertices and triangles in Source's (clockwise) order. + public sealed record Mesh(int Material, Vertex[] Vertices, int[] Triangles); + + /// A bone in its bind pose relative to its parent. + public sealed record Bone(string Name, int Parent, Vector3 Position, Vector3 Rotation); + + public const int Checksum = 0x1EE7; + + /// + /// Writes ".mdl", ".vvd" and ".dx90.vtx" into and returns the .mdl; + /// sets a different checksum in the .vvd; stores LOD 0 as + /// two runs of vertices. + /// + public static string Write(string folder, string name, string textureFolder, string[] textures, Mesh[] meshes, + Bone[]? bones = null, int vvdChecksum = Checksum, bool fixups = false) + { + bones ??= [new Bone("static_prop", -1, Vector3.Zero, Vector3.Zero)]; + Directory.CreateDirectory(folder); + var path = Path.Combine(folder, name + ".mdl"); + File.WriteAllBytes(path, Mdl(name, textureFolder, textures, meshes, bones)); + File.WriteAllBytes(Path.Combine(folder, name + ".vvd"), Vvd(meshes, vvdChecksum, fixups)); + File.WriteAllBytes(Path.Combine(folder, name + ".dx90.vtx"), Vtx(meshes)); + return path; + } + + private static byte[] Mdl(string name, string textureFolder, string[] textures, Mesh[] meshes, Bone[] bones) + { + const int header = 408, bone = 216, texture = 64, bodyPart = 16, model = 148, mesh = 116; + var boneAt = header; + var textureAt = boneAt + bones.Length * bone; + var folderTableAt = textureAt + textures.Length * texture; + var skinAt = folderTableAt + 4; + var bodyPartAt = skinAt + textures.Length * 2 + 2; + var modelAt = bodyPartAt + bodyPart; + var meshAt = modelAt + model; + var stringsAt = meshAt + meshes.Length * mesh; + + var strings = new MemoryStream(); + int Str(string text) + { + var at = stringsAt + (int)strings.Length; + strings.Write(Encoding.ASCII.GetBytes(text + "\0")); + return at; + } + + var data = new byte[stringsAt]; + var span = data.AsSpan(); + "IDST"u8.CopyTo(span); + I32(span, 4, 48); + I32(span, 8, Checksum); + Encoding.ASCII.GetBytes($"test/{name}.mdl").CopyTo(span[12..]); + I32(span, 156, bones.Length); + I32(span, 160, boneAt); + I32(span, 204, textures.Length); + I32(span, 208, textureAt); + I32(span, 212, 1); + I32(span, 216, folderTableAt); + I32(span, 220, textures.Length); + I32(span, 224, 1); + I32(span, 228, skinAt); + I32(span, 232, 1); + I32(span, 236, bodyPartAt); + + for (var i = 0; i < bones.Length; i++) + { + var at = boneAt + i * bone; + I32(span, at, Str(bones[i].Name) - at); + I32(span, at + 4, bones[i].Parent); + Vec(span, at + 32, bones[i].Position); + F32(span, at + 56, 1); // quaternion w + Vec(span, at + 60, bones[i].Rotation); + } + for (var i = 0; i < textures.Length; i++) + { + var at = textureAt + i * texture; + I32(span, at, Str(textures[i]) - at); + BinaryPrimitives.WriteInt16LittleEndian(span[(skinAt + i * 2)..], (short)i); + } + I32(span, folderTableAt, Str(textureFolder)); + I32(span, bodyPartAt, Str("body") - bodyPartAt); + I32(span, bodyPartAt + 4, 1); + I32(span, bodyPartAt + 8, 1); + I32(span, bodyPartAt + 12, modelAt - bodyPartAt); + Encoding.ASCII.GetBytes(name).CopyTo(span[modelAt..]); + I32(span, modelAt + 72, meshes.Length); + I32(span, modelAt + 76, meshAt - modelAt); + I32(span, modelAt + 80, meshes.Sum(m => m.Vertices.Length)); + I32(span, modelAt + 84, 0); + var first = 0; + for (var i = 0; i < meshes.Length; i++) + { + var at = meshAt + i * mesh; + I32(span, at, meshes[i].Material); + I32(span, at + 4, modelAt - at); + I32(span, at + 8, meshes[i].Vertices.Length); + I32(span, at + 12, first); + first += meshes[i].Vertices.Length; + } + var result = data.Concat(strings.ToArray()).ToArray(); + I32(result, 76, result.Length); + return result; + } + + private static byte[] Vvd(Mesh[] meshes, int checksum, bool fixups) + { + var vertices = meshes.SelectMany(m => m.Vertices).ToArray(); + var fixupCount = fixups ? 2 : 0; + var fixupAt = 64; + var vertexAt = fixupAt + fixupCount * 12; + var data = new byte[vertexAt + vertices.Length * 48]; + var span = data.AsSpan(); + "IDSV"u8.CopyTo(span); + I32(span, 4, 4); + I32(span, 8, checksum); + I32(span, 12, 1); + I32(span, 16, vertices.Length); + I32(span, 48, fixupCount); + I32(span, 52, fixupAt); + I32(span, 56, vertexAt); + I32(span, 60, data.Length); + if (fixups) + { + var half = vertices.Length / 2; + I32(span, fixupAt, 0); + I32(span, fixupAt + 4, 0); + I32(span, fixupAt + 8, half); + I32(span, fixupAt + 12, 0); + I32(span, fixupAt + 16, half); + I32(span, fixupAt + 20, vertices.Length - half); + } + for (var i = 0; i < vertices.Length; i++) + { + var at = vertexAt + i * 48; + var v = vertices[i]; + var two = v.Bone2 >= 0; + F32(span, at, two ? 1 - v.Weight2 : 1); + F32(span, at + 4, two ? v.Weight2 : 0); + span[at + 12] = (byte)v.Bone; + span[at + 13] = (byte)Math.Max(0, v.Bone2); + span[at + 15] = (byte)(two ? 2 : 1); + Vec(span, at + 16, v.Position); + Vec(span, at + 28, v.Normal); + F32(span, at + 40, v.Uv.X); + F32(span, at + 44, v.Uv.Y); + } + return data; + } + + private static byte[] Vtx(Mesh[] meshes) + { + const int header = 36, bodyPart = 8, model = 8, lod = 12, mesh = 9, group = 25, strip = 27; + var bodyPartAt = header; + var modelAt = bodyPartAt + bodyPart; + var lodAt = modelAt + model; + var meshAt = lodAt + lod; + var groupAt = meshAt + meshes.Length * mesh; + var dataAt = groupAt + meshes.Length * group; + + var tail = new MemoryStream(); + var bytes = new byte[dataAt]; + var span = bytes.AsSpan(); + I32(span, 0, 7); + I32(span, 4, 24); + I32(span, 16, Checksum); + I32(span, 20, 1); + I32(span, 28, 1); + I32(span, 32, bodyPartAt); + I32(span, bodyPartAt, 1); + I32(span, bodyPartAt + 4, modelAt - bodyPartAt); + I32(span, modelAt, 1); + I32(span, modelAt + 4, lodAt - modelAt); + I32(span, lodAt, meshes.Length); + I32(span, lodAt + 4, meshAt - lodAt); + for (var i = 0; i < meshes.Length; i++) + { + var m = meshes[i]; + var meshPos = meshAt + i * mesh; + var groupPos = groupAt + i * group; + I32(span, meshPos, 1); + I32(span, meshPos + 4, groupPos - meshPos); + + var vertexPos = dataAt + (int)tail.Length; + foreach (var index in Enumerable.Range(0, m.Vertices.Length)) + { + var vertex = new byte[9]; + vertex[3] = 1; + BinaryPrimitives.WriteUInt16LittleEndian(vertex.AsSpan(4), (ushort)index); + tail.Write(vertex); + } + var indexPos = dataAt + (int)tail.Length; + var u16 = new byte[2]; + foreach (var index in m.Triangles) + { + BinaryPrimitives.WriteUInt16LittleEndian(u16, (ushort)index); + tail.Write(u16); + } + var stripPos = dataAt + (int)tail.Length; + var stripBytes = new byte[strip]; + I32(stripBytes, 0, m.Triangles.Length); + I32(stripBytes, 8, m.Vertices.Length); + BinaryPrimitives.WriteInt16LittleEndian(stripBytes.AsSpan(16), 1); + stripBytes[18] = 0x01; // triangle list + tail.Write(stripBytes); + + I32(span, groupPos, m.Vertices.Length); + I32(span, groupPos + 4, vertexPos - groupPos); + I32(span, groupPos + 8, m.Triangles.Length); + I32(span, groupPos + 12, indexPos - groupPos); + I32(span, groupPos + 16, 1); + I32(span, groupPos + 20, stripPos - groupPos); + } + return [.. bytes, .. tail.ToArray()]; + } + + private static void I32(Span span, int at, int value) => BinaryPrimitives.WriteInt32LittleEndian(span[at..], value); + + private static void F32(Span span, int at, float value) => BinaryPrimitives.WriteSingleLittleEndian(span[at..], value); + + private static void Vec(Span span, int at, Vector3 v) + { + F32(span, at, v.X); + F32(span, at + 4, v.Y); + F32(span, at + 8, v.Z); + } + + /// + /// A 10 × 10 inch floor tile facing up (material 0, four vertices, two triangles) and a triangle 5 inches above it + /// (material 1), both clockwise as Source stores them. + /// + public static Mesh[] TileAndTriangle(bool weighted = false) => + [ + new(0, + [ + new(new(0, 0, 0), Vector3.UnitZ, new(0, 1)), + new(new(0, 10, 0), Vector3.UnitZ, new(0, 0)), + new(new(10, 10, 0), Vector3.UnitZ, new(1, 0)), + new(new(10, 0, 0), Vector3.UnitZ, new(1, 1), weighted ? 0 : 0, weighted ? 1 : -1, weighted ? 0.25f : 0), + ], [0, 1, 2, 0, 2, 3]), + new(1, + [ + new(new(0, 0, 5), Vector3.UnitZ, new(0, 0), weighted ? 1 : 0), + new(new(0, 10, 5), Vector3.UnitZ, new(0, 1), weighted ? 1 : 0), + new(new(10, 10, 5), Vector3.UnitZ, new(1, 1), weighted ? 1 : 0), + ], [0, 1, 2]), + ]; +} diff --git a/tests/Filee.Engines.Tests/SourceModelTests.cs b/tests/Filee.Engines.Tests/SourceModelTests.cs new file mode 100644 index 000000000..498f69f35 --- /dev/null +++ b/tests/Filee.Engines.Tests/SourceModelTests.cs @@ -0,0 +1,189 @@ +// Source engine models (.mdl with .vvd and .dx90.vtx, #1526): read in their bind pose into Filee's 3D model (Y up, +// millimetres, counter-clockwise), with their textures found through the .vmt files in the "materials" folder above them +// and decoded from VTF. The test models are built by SourceModelBuilder. + +using System.Buffers.Binary; +using System.Numerics; +using Filee.Core.Conversion; +using Filee.Core.Formats; +using Filee.Core.Presets; +using Filee.Engines.Magick; +using Filee.Engines.Models3D; +using ImageMagick; + +namespace Filee.Engines.Tests; + +public class SourceModelTests(EngineFixture fx) : IClassFixture +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private async Task> ConvertOkAsync(string source, string target) + { + var job = await fx.ConvertAsync([source], new Preset { TargetFormat = target }); + Assert.True(job.State == JobState.Completed, string.Join("; ", job.Files.Select(f => $"{f.ErrorKey} {f.ErrorDetail}"))); + return [.. job.Outputs]; + } + + /// + /// An addon folder as Garry's Mod mounts it: models/props/tile.mdl (two materials, "brick" and "glass"), the brick's + /// material (in a differently cased folder) pointing at a red 16 × 16 VTF, and no material for the glass. + /// + private string Addon(bool fixups = false) + { + var root = fx.NewFolder(); + var mdl = SourceModelBuilder.Write(Path.Combine(root, "models", "props"), "tile", "models\\props\\", ["brick", "glass"], + SourceModelBuilder.TileAndTriangle(), fixups: fixups); + var materials = Directory.CreateDirectory(Path.Combine(root, "materials", "Models", "Props")).FullName; + File.WriteAllText(Path.Combine(materials, "brick.vmt"), """ + // a comment + "VertexLitGeneric" + { + "$basetexture" "models\props\brick_color" + "$nocull" "1" + } + """); + using var red = new MagickImage(MagickColors.Red, 16, 16); + VtfWriter.Write([red], Path.Combine(materials, "brick_color.vtf")); + return mdl; + } + + [Fact] + public void Mdl_is_a_3d_model_that_converts_in_one_step() + { + Assert.Equal("mdl", FormatRegistry.Detect("Tile.MDL")?.Id); + var planner = fx.Catalog.CreatePlanner(["model3d"]); + foreach (var target in new[] { "obj", "glb", "stl", "3mf", "ply", "gltf" }) + Assert.Single(planner.Plan("mdl", target)!.Steps); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_model_is_read_in_filees_axes_units_and_winding(bool fixups) + { + var model = ModelConverter.Read(Addon(fixups), "mdl", null, Ct); + + Assert.Equal(2, model.Parts.Count); + Assert.Equal(3, model.TriangleCount); + Assert.Equal(7, model.VertexCount); + // 10 inches is 254 mm; Source's Z up becomes Y up, its Y becomes −Z. + var (min, max) = model.Bounds(); + Assert.Equal(new Vector3(0, 0, -254), min, new Vector3Comparer()); + Assert.Equal(new Vector3(254, 127, 0), max, new Vector3Comparer()); + foreach (var part in model.Parts) + { + // Counter-clockwise: every face points the way its vertex normals (up) do. + for (var t = 0; t < part.TriangleCount; t++) + Assert.True(Vector3.Dot(part.FaceNormal(t), Vector3.UnitY) > 0.99f); + Assert.All(Enumerable.Range(0, part.VertexCount), v => Assert.Equal(Vector3.UnitY, part.Normal(v), new Vector3Comparer())); + } + // UVs from the bottom left: the tile's corner at the origin had v = 1 from the top. + var tile = model.Parts[0]; + var origin = Enumerable.Range(0, tile.VertexCount).Single(v => tile.Position(v) == Vector3.Zero); + Assert.Equal(Vector2.Zero, tile.TexCoord(origin)); + } + + [Fact] + public void Its_textures_are_found_through_the_materials() + { + var model = ModelConverter.Read(Addon(), "mdl", null, Ct); + + var brick = model.Materials.Single(m => m.Name == "brick"); + Assert.True(brick.DoubleSided); + var texture = Assert.IsType(brick.BaseColorTexture); + Assert.Equal("png", texture.Extension); + using var image = new MagickImage(texture.Data); + Assert.Equal((16u, 16u), (image.Width, image.Height)); + using var pixels = image.GetPixels(); + var pixel = pixels.GetPixel(8, 8)!.ToColor()!; + Assert.InRange(pixel.R, 240, 255); + Assert.InRange(pixel.G, 0, 15); + + // The game's own texture (no material in the addon) leaves the material plain. + Assert.Null(model.Materials.Single(m => m.Name == "glass").BaseColorTexture); + } + + [Fact] + public async Task A_model_becomes_an_obj_with_its_texture_and_a_glb() + { + var mdl = Addon(); + + var obj = await ConvertOkAsync(mdl, "obj"); + Assert.Contains(obj, p => p.EndsWith(".obj", StringComparison.Ordinal)); + Assert.Contains(obj, p => p.EndsWith(".mtl", StringComparison.Ordinal)); + Assert.Contains(obj, p => p.EndsWith(".png", StringComparison.Ordinal)); + + var glb = Assert.Single(await ConvertOkAsync(mdl, "glb")); + var back = ModelConverter.Read(glb, "glb", null, Ct); + Assert.Equal(3, back.TriangleCount); + Assert.Contains(back.Materials, m => m.BaseColorTexture is not null); + } + + [Fact] + public void A_model_without_its_vvd_or_with_another_builds_vvd_is_refused_clearly() + { + var folder = fx.NewFolder(); + var mdl = SourceModelBuilder.Write(folder, "box", "", ["brick", "glass"], SourceModelBuilder.TileAndTriangle()); + File.Delete(Path.ChangeExtension(mdl, ".vvd")); + var missing = Assert.Throws(() => ModelConverter.Read(mdl, "mdl", null, Ct)); + Assert.Contains("box.vvd", missing.Message); + + mdl = SourceModelBuilder.Write(folder, "box2", "", ["brick", "glass"], SourceModelBuilder.TileAndTriangle(), vvdChecksum: 42); + var other = Assert.Throws(() => ModelConverter.Read(mdl, "mdl", null, Ct)); + Assert.Contains("checksum", other.Message); + + var notModel = Path.Combine(folder, "quake.mdl"); + File.WriteAllBytes(notModel, "IDPO"u8.ToArray()); + File.WriteAllBytes(Path.Combine(folder, "quake.vvd"), []); + File.WriteAllBytes(Path.Combine(folder, "quake.dx90.vtx"), []); + Assert.Throws(() => ModelConverter.Read(notModel, "mdl", null, Ct)); + } + + [Fact] + public void Triangles_are_counted_against_one_budget_even_where_meshes_share_a_table() + { + var folder = fx.NewFolder(); + var mdl = SourceModelBuilder.Write(folder, "shared", "", ["brick", "glass"], SourceModelBuilder.TileAndTriangle()); + // The first mesh (at 64 in the .vtx) now claims two strip groups: its own and the second mesh's, which follows it. + // The second mesh's triangle is read twice, so the model has 4 triangles though its file holds 3. + var vtx = Path.ChangeExtension(mdl, ".dx90.vtx"); + var bytes = File.ReadAllBytes(vtx); + BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(64), 2); + File.WriteAllBytes(vtx, bytes); + + Assert.Equal(4, SourceModel.Load(mdl, Ct, maxTriangles: 4).Meshes.Sum(m => m.Triangles.Count / 3)); + var refused = Assert.Throws(() => SourceModel.Load(mdl, Ct, maxTriangles: 3)); + Assert.Equal("error.document_too_large", refused.ErrorKey); + } + + [Fact] + public void A_material_file_larger_than_the_cap_leaves_the_material_plain() + { + var folder = fx.NewFolder(); + var vmt = Path.Combine(folder, "huge.vmt"); + File.WriteAllText(vmt, "\"VertexLitGeneric\" { $basetexture \"a/b\" }" + new string(' ', (int)Vmt.MaxBytes)); + Assert.Null(Vmt.Read(vmt, _ => null).BaseTexture); + + File.WriteAllText(vmt, "\"VertexLitGeneric\" { $basetexture \"a/b\" }"); + Assert.Equal("a/b", Vmt.Read(vmt, _ => null).BaseTexture); + } + + [Fact] + public void Material_files_are_read_as_keyvalues() + { + var tokens = Vmt.Tokens(""" + "UnlitGeneric" // shader + { + $basetexture "a/b c" + "$alphatest" 1 + } + """); + Assert.Equal(["UnlitGeneric", "{", "$basetexture", "a/b c", "$alphatest", "1", "}"], tokens); + } + + private sealed class Vector3Comparer : IEqualityComparer + { + public bool Equals(Vector3 a, Vector3 b) => Vector3.Distance(a, b) < 0.01f; + public int GetHashCode(Vector3 v) => 0; + } +} From 46a421d72fec3d0073685f10be0f9b0e2f865463 Mon Sep 17 00:00:00 2001 From: "Lt. Sammy" <26645712+ltsammy@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:15:19 +0000 Subject: [PATCH 2/2] feat(model3d): read and write studiomdl data (SMD), from MDL with its skeleton An MDL becomes a reference SMD studiomdl can compile again: its bones with their bind pose, its triangles counter-clockwise with up to three bone weights per vertex, UVs from the bottom left, in inches, Z up. Any other 3D model becomes a static prop with one root bone, ready for studiomdl; a reference SMD's triangles become a model again, so SMD converts to GLB, OBJ, STL and the rest. SMD is in the Games field; .smd files learn the models donut and the SMD preset joins the 3D models in the palette (schema 100, with MDL). Part of #1526. --- README.md | 2 +- docs/ENGINES.md | 15 +- src/Filee.Core/Formats/FormatFields.cs | 4 +- src/Filee.Core/Formats/FormatRegistry.cs | 2 + src/Filee.Core/Presets/BuiltInData.cs | 10 + src/Filee.Core/Settings/UserDataStore.cs | 22 +- src/Filee.Engines/Models3D/ModelConverter.cs | 39 +++- src/Filee.Engines/Models3D/SmdFormat.cs | 206 ++++++++++++++++++ tests/Filee.Core.Tests/FormatFieldsTests.cs | 8 +- tests/Filee.Core.Tests/SourceFormatTests.cs | 23 +- .../TranslatePresetMigrationTests.cs | 2 +- tests/Filee.Core.Tests/UserDataStoreTests.cs | 6 +- tests/Filee.Engines.Tests/SmdTests.cs | 149 +++++++++++++ 13 files changed, 463 insertions(+), 25 deletions(-) create mode 100644 src/Filee.Engines/Models3D/SmdFormat.cs create mode 100644 tests/Filee.Engines.Tests/SmdTests.cs diff --git a/README.md b/README.md index 127610b50..c11c5613e 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ To undo it, remove the entries in Windows Security → App & browser control → | **CAD**: DWG, DXF
Gerber PCB layers (GBR, GTL, GBL, GTO, …) read;
G-code of 3D printers and CNC machines (GCODE, GCO, NGC) read | PDF, SVG, PNG, DWG ↔ DXF;
G-code → toolpath (from above or isometric) or the slicer's preview | | **Fonts**: TTF, OTF, WOFF, WOFF2, EOT;
FON (Windows bitmap fonts) read | each other;
FON → one font per size | | **Data**: JSON, JSON Lines, YAML, TOML, XML
vCard, iCalendar | each other, CSV and XLSX (tables);
vCard, iCalendar ↔ CSV | -| **3D models**: GLB, glTF, STL, OBJ (+ MTL), 3MF, PLY;
FBX, Source engine model (MDL, with its VTF textures) read | each other, PNG preview | +| **3D models**: GLB, glTF, STL, OBJ (+ MTL), 3MF, PLY, studiomdl data (SMD);
FBX, Source engine model (MDL, with its VTF textures) read | each other, PNG preview | | **Geospatial vectors**: Shapefile, GeoJSON, KML, KMZ, GPX
CSV with coordinates; Garmin FIT, OpenStreetMap (.osm) read | each other and CSV / XLSX / JSON;
projected data (UTM, Web Mercator, Lambert, Albers) comes out as WGS 84 longitude / latitude | Unmarked formats work out of the box, without Office, Hancom Office or LibreOffice. diff --git a/docs/ENGINES.md b/docs/ENGINES.md index 18b36017e..5a7025adf 100644 --- a/docs/ENGINES.md +++ b/docs/ENGINES.md @@ -1434,8 +1434,8 @@ to draw). G-code is a read-only CAD format (fields: 3D models and CAD). ## 3D models -`Filee.Engines/Models3D` converts GLB, glTF, STL, OBJ, 3MF and PLY into each other, and FBX and Source engine models -(MDL) into any of them, in managed code: every reader +`Filee.Engines/Models3D` converts GLB, glTF, STL, OBJ, 3MF and PLY into each other, FBX and Source engine models (MDL) +into any of them, and studiomdl data (SMD) to and from them, in managed code: every reader fills one in-memory model (`Model3D`: parts with positions, normals, UVs, vertex colours and triangles; materials with a base colour and a base colour texture) and every writer takes it, so each pair is one step. glTF and GLB are read and written with [SharpGLTF](https://github.com/vpenades/SharpGLTF) (MIT); STL, OBJ/MTL, PLY, 3MF and the FBX @@ -1469,6 +1469,7 @@ What each format keeps: | OBJ + MTL | `o` / `g` / `usemtl` as parts, `v` (+ r g b), `vt`, `vn`, faces of any size; `mtllib` with `Kd`, `d` / `Tr`, `map_Kd` (options skipped) next to the file | **the `.mtl` and the textures are written next to the `.obj`** (`name.obj`, `name.mtl`, `name_texture.png`, named like the output); vertex colours as `v x y z r g b` | | 3MF | objects, components (also in other model parts, as Bambu Studio writes them), build transforms, base materials, colour groups (materials extension); `support` / `other` objects skipped | one object per part, base materials from material colours, a colour group from vertex colours | | FBX (read only) | binary 6.x–7.x (64-bit records from 7.5, zlib-compressed arrays) and ASCII; mesh geometry with polygons of any size, the first normal, UV, colour and material layer (by polygon vertex, control point or polygon, or all the same; direct or indexed); the model hierarchy from the connections with translation, rotation (every Euler order, pre- / post-rotation, pivots and offsets), scaling and geometric transforms; property templates from Definitions; material `DiffuseColor` and `Opacity`; the diffuse texture embedded (Video `Content`) or named next to the file; animation, skins, blend shapes, NURBS, cameras and lights skipped (the rest pose is read) | — (not written) | +| SMD (studiomdl data) | the triangles of a reference SMD, one part per material (bones and weights not kept) | from an MDL: its bones, bind pose and weights (see below); from the others: a static prop with one root bone | | MDL (Source engine, read only) | versions 44–49 with the `.vvd` and `.dx90.vtx` (else `.vtx`, `.dx80.vtx`, `.sw.vtx`) beside it (see below) | — (needs Valve's studiomdl) | | PLY | ASCII, binary little / big endian; vertex position, normal, UV (`s`/`t`, `u`/`v`, `texture_u`/`texture_v`), colours of any type; face lists with per-corner `texcoord` and face colours; `comment TextureFile`; other elements skipped | binary little endian, one mesh; material colours become vertex colours | @@ -1503,6 +1504,16 @@ a `.vtf`, decoded by the VTF reader into a PNG texture; `$nocull` makes the mate materials come from the game itself (the HL2 textures), keeps plain materials. OBJ output writes the textures beside the `.obj`, GLB embeds them. +**Studiomdl data** (`.smd`, #1526, the Games field; `Models3D/SmdFormat.cs`): the text Valve's studiomdl compiles models +from (`nodes`, a `skeleton` at `time 0`, `triangles`: a material line and three vertex lines of parent bone, position, +normal, UV and bone weights), Z up, in inches, counter-clockwise, UVs from the bottom left. An MDL becomes a reference +SMD that studiomdl can compile again: its bones with their bind pose (position and rotation relative to the parent), its +triangles with up to three bone weights per vertex and the textures as materials, read straight from the model so the +skeleton is kept (Filee's own 3D model has no place for it). Any other model becomes a static prop: one `root` bone, +every vertex weighted to it, the material's (or texture's) name as the material line, normals computed where the model +has none. Reading takes the triangles of a reference SMD as a static model; an animation SMD (no triangles) is refused +with "no triangles", a broken line with its number. The "SMD" preset is in the palette, after glTF. + **3D → pictures**: a small software rasteriser (`MeshRenderer`) draws an isometric view from the front right, above, with one light and flat shading per triangle, vertex colours or the texture interpolated across it, and a z-buffer, at 2048 px and scaled down to 1024 px for smooth edges, on a transparent background. The preset's image options diff --git a/src/Filee.Core/Formats/FormatFields.cs b/src/Filee.Core/Formats/FormatFields.cs index 01aed0a92..1ea004f72 100644 --- a/src/Filee.Core/Formats/FormatFields.cs +++ b/src/Filee.Core/Formats/FormatFields.cs @@ -83,7 +83,7 @@ .. Enum.GetValues().Select(category => category.ToString().ToLow "dts", "tar", "tgz", "tbz2", "txz", "tar7z", "tlz", "gz", "bz2", "xz", "lz", "lzma", "jar", "zxp", "nupkg", "apks", "aab", "msix", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", "cab", "gerber", "cpio", "deb", "rpm", "dwg", "ttf", "otf", "woff", "woff2", "ass", "ssa", "ttml", "scc", "glb", "gltf", - "stl", "obj", "3mf", "ply", "fbx", "gcode", "geojson", "kml", "kmz", "gpx", "shp", "fit", "osm", "xm", "it", "ani", "vtf", "gma", "vpk", "bsp", "mdl" + "stl", "obj", "3mf", "ply", "fbx", "gcode", "geojson", "kml", "kmz", "gpx", "shp", "fit", "osm", "xm", "it", "ani", "vtf", "gma", "vpk", "bsp", "mdl", "smd" ]), (FormatGrade.Legacy, [ @@ -100,7 +100,7 @@ .. Enum.GetValues().Select(category => category.ToString().ToLow // Layout, print, photo post-production, signs and embroidery. (["psd", "psb", "exr", "hdr", "xcf", "kra", "pdn", "raw", "eps", "ps", "ai", "cdr", "plt", "dst"], ["design"]), // Game textures and cutscenes, Minecraft packs and modpacks. - (["dds", "vtf", "usm", "bik", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", "gma", "vpk", "bsp", "mdl"], ["game"]), + (["dds", "vtf", "usm", "bik", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", "gma", "vpk", "bsp", "mdl", "smd"], ["game"]), // Medical imaging and astronomy. (["dcm", "fits"], ["science"]), // Software work: raw image formats of graphics tools, app icons and cursors, config and data files, flat XML diff --git a/src/Filee.Core/Formats/FormatRegistry.cs b/src/Filee.Core/Formats/FormatRegistry.cs index 6233f1568..ccc70c370 100644 --- a/src/Filee.Core/Formats/FormatRegistry.cs +++ b/src/Filee.Core/Formats/FormatRegistry.cs @@ -422,6 +422,8 @@ public static class FormatRegistry // Source engine models (Garry's Mod and other Source games), read with the .vvd and .vtx beside them and their // textures from the "materials" folder above them. new("mdl", "Source model (MDL)", FormatCategory.Model3D, ["mdl"], Writable: false), + // Studiomdl data, the text the Source engine's model compiler reads (from an MDL with its bones and weights). + new("smd", "Studiomdl data (SMD)", FormatCategory.Model3D, ["smd"]), // Geospatial vector data (CSV with coordinates is a Data format and reaches these through the geo engine) new("geojson", "GeoJSON", FormatCategory.Geo, ["geojson"]), diff --git a/src/Filee.Core/Presets/BuiltInData.cs b/src/Filee.Core/Presets/BuiltInData.cs index 8f130f15a..55eb837a9 100644 --- a/src/Filee.Core/Presets/BuiltInData.cs +++ b/src/Filee.Core/Presets/BuiltInData.cs @@ -172,6 +172,7 @@ .. CatalogPresets(), .. DataPresets(), .. CodecAndSubtitlePresets(), .. ModelPresets(), + .. SourceModelPresets(), .. GeoPresets(), .. TracePresets(), ]; @@ -603,6 +604,15 @@ public static List ModelPresets() => new() { Id = "to-gltf", Name = "glTF", TargetFormat = "gltf" }, ]; + /// + /// Studiomdl data for Source engine models (#1526, added to the library in settings schema 100): in the palette only, + /// after glTF, in the Games field (off in a new install). + /// + public static List SourceModelPresets() => + [ + new() { Id = "to-smd", Name = "SMD", TargetFormat = "smd" }, + ]; + /// /// Geospatial vector targets (added in settings schema 25, see SettingsMigrations). CSV, the sixth slice of the /// donut, is the spreadsheet preset. KMZ is a KML in a ZIP and is on no donut. diff --git a/src/Filee.Core/Settings/UserDataStore.cs b/src/Filee.Core/Settings/UserDataStore.cs index 942a61128..0a45e6495 100644 --- a/src/Filee.Core/Settings/UserDataStore.cs +++ b/src/Filee.Core/Settings/UserDataStore.cs @@ -1193,9 +1193,13 @@ public static bool ApplyToLibrary(int from, List presets, List presets) return true; } + /// "SMD" after the last 3D model preset of the library (glTF), or at the end. + private static bool AddSourcePresets(List presets) + { + var changed = false; + foreach (var preset in BuiltInData.SourceModelPresets().Where(p => presets.All(existing => existing.Id != p.Id))) + { + var models = BuiltInData.ModelPresets().Select(p => p.Id).ToHashSet(); + var last = presets.FindLastIndex(p => models.Contains(p.Id)); + presets.Insert(last < 0 ? presets.Count : last + 1, preset); + changed = true; + } + return changed; + } + private static bool AddCropVideo(List presets) { var at = presets.FindIndex(p => p.Id == "tool-clip"); diff --git a/src/Filee.Engines/Models3D/ModelConverter.cs b/src/Filee.Engines/Models3D/ModelConverter.cs index 68dc889ab..5139dc9b0 100644 --- a/src/Filee.Engines/Models3D/ModelConverter.cs +++ b/src/Filee.Engines/Models3D/ModelConverter.cs @@ -1,6 +1,7 @@ // 3D models in managed code only (SharpGLTF for glTF, Filee's own readers and writers for the rest): GLB, glTF, STL, // OBJ, 3MF and PLY into each other through one in-memory model (Model3D), FBX and Source engine models (MDL, with their -// textures; read only) into any of them, and an isometric preview picture. +// textures; read only) into any of them, studiomdl data (SMD) to and from them (from an MDL with its bones and weights), +// and an isometric preview picture. // See docs/ENGINES.md, "3D models", for units, axes and what each format keeps. using Filee.Core.Conversion; @@ -19,8 +20,8 @@ public sealed class ModelConverter(ILogger? logger = null) : IConverter, IImageT /// The 3D formats, each readable and writable. public static readonly string[] Formats = ["glb", "gltf", "stl", "obj", "3mf", "ply"]; - /// Every 3D format that can be read: , FBX and Source engine models (MDL). - public static readonly string[] InputFormats = [.. Formats, "fbx", "mdl"]; + /// Every 3D format that can be read: , FBX, Source engine models (MDL) and SMD. + public static readonly string[] InputFormats = [.. Formats, "fbx", "mdl", "smd"]; /// Long edge of the preview picture before the preset's resize. internal const int PreviewSize = 1024; @@ -39,15 +40,20 @@ public sealed class ModelConverter(ILogger? logger = null) : IConverter, IImageT public IReadOnlyList Edges { get; } = [ .. from source in InputFormats from target in Formats where source != target select new ConversionEdge(source, target), + // SMD, the source studiomdl compiles: from an MDL with its skeleton, from the others as a static prop. + .. from source in InputFormats where source != "smd" select new ConversionEdge(source, "smd"), .. from source in InputFormats from target in ImageEncoder.Writable select ImageEncoder.Edge(source, target), ]; - public EngineStatus GetStatus() => EngineStatus.Available("GLB, glTF, STL, OBJ, 3MF, PLY ↔ each other; FBX, MDL → each of them", + public EngineStatus GetStatus() => EngineStatus.Available("GLB, glTF, STL, OBJ, 3MF, PLY, SMD ↔ each other; FBX, MDL → each of them", $"{EngineVersions.BuiltIn} · {EngineVersions.Library("SharpGLTF", typeof(SharpGLTF.Schema2.ModelRoot))}"); public Task> ConvertAsync(ConversionStep step, IProgress? progress, CancellationToken cancellationToken) => Task.Run>(() => { + // A Source model keeps its skeleton and weights in an SMD, which Filee's static model has no place for. + if (step.From == "mdl" && step.To == "smd") + return WriteSkeletalSmd(step, cancellationToken); var reading = progress is null ? null : new SubProgress(progress, 0, 0.6); var model = Read(step.InputPath, step.From, reading, cancellationToken); _logger.LogDebug("{File}: {Parts} parts, {Vertices} vertices, {Triangles} triangles, {Materials} materials", @@ -81,6 +87,7 @@ internal static Model3D Read(string path, string format, IProgress? prog "glb" or "gltf" => GltfFormat.Read(path, progress, cancellationToken), "fbx" => FbxFormat.Read(path, progress, cancellationToken), "mdl" => SourceModel.Load(path, cancellationToken).ToModel3D(path, cancellationToken), + "smd" => SmdFormat.Read(path, progress, cancellationToken), _ => throw new NotSupportedException($"Unknown 3D format '{format}'."), }; model.Validate(); @@ -98,7 +105,7 @@ internal static void Write(Model3D model, string format, IOutputAllocator output if (output.Allocate(format) is not { } path) return; outputs.Add(path); - if (!Formats.Contains(format)) + if (!Formats.Contains(format) && format != "smd") { WriteImage(model, format, imageOptions, path, cancellationToken); return; @@ -115,6 +122,9 @@ internal static void Write(Model3D model, string format, IOutputAllocator output case "3mf": ThreeMfFormat.Write(model, stream, cancellationToken); break; + case "smd": + SmdFormat.Write(model, stream, cancellationToken); + break; case "glb": case "gltf": GltfFormat.Write(model, stream, binary: format == "glb", cancellationToken); @@ -122,6 +132,25 @@ internal static void Write(Model3D model, string format, IOutputAllocator output } } + /// A Source model as a reference SMD with its bones, bind pose and weights. + private static IReadOnlyList WriteSkeletalSmd(ConversionStep step, CancellationToken cancellationToken) + { + var model = SourceModel.Load(step.InputPath, cancellationToken); + if (step.Output.Allocate("smd") is not { } path) + return []; + try + { + using var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 16); + SmdFormat.WriteSkeletal(model, stream, cancellationToken); + } + catch + { + TryDelete(path); + throw; + } + return [path]; + } + /// The .obj, the .mtl next to it (when the model has materials) and the textures next to that. private static void WriteObj(Model3D model, IOutputAllocator output, List outputs, CancellationToken cancellationToken) { diff --git a/src/Filee.Engines/Models3D/SmdFormat.cs b/src/Filee.Engines/Models3D/SmdFormat.cs new file mode 100644 index 000000000..5eff85e10 --- /dev/null +++ b/src/Filee.Engines/Models3D/SmdFormat.cs @@ -0,0 +1,206 @@ +// Studiomdl data (.smd), the text format Valve's studiomdl compiles Source engine models from: +// +// version 1 +// nodes one line per bone: index "name" parent (−1 for a root) +// skeleton / time 0 one line per bone: index, position x y z, rotation x y z (radians), relative to its parent +// triangles per triangle a material line, then three vertex lines: +// parent bone, position x y z, normal x y z, u v [, link count, (bone weight) per link] +// end after each section +// +// Positions are in model space, Z up, in inches (Source's unit), triangles counter-clockwise, UVs from the bottom left. +// Reading takes the triangles of a reference SMD as a static model (an animation SMD has none). Writing from another +// model gives a static prop: one root bone, every vertex weighted to it. Writing from a Source model (MDL) keeps its +// bones, bind pose and weights, as a reference mesh studiomdl can compile again. + +using System.Globalization; +using System.Numerics; +using System.Text; + +namespace Filee.Engines.Models3D; + +/// Reads and writes studiomdl data files. +internal static class SmdFormat +{ + /// The most triangles Filee reads from one file (about 1 GB of text). + private const int MaxTriangles = 10_000_000; + + private static readonly CultureInfo Invariant = CultureInfo.InvariantCulture; + + /// The triangles of a reference SMD, one part per material. + public static Model3D Read(string path, IProgress? progress, CancellationToken cancellationToken) + { + var model = new Model3D(); + var parts = new Dictionary(StringComparer.Ordinal); + var length = Math.Max(1, new FileInfo(path).Length); + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); + var lineNumber = 0; + string? Next() + { + while (reader.ReadLine() is { } line) + { + lineNumber++; + var text = line.Trim(); + if (text.Length > 0 && !text.StartsWith("//", StringComparison.Ordinal)) + return text; + } + return null; + } + + var triangles = 0; + var corners = new int[3]; + string? section = null; + while (Next() is { } line) + { + if (section is null) + { + section = line.Split((char[]?)null, 2, StringSplitOptions.RemoveEmptyEntries)[0].ToLowerInvariant(); + if (section is "version") + section = null; + continue; + } + if (line.Equals("end", StringComparison.OrdinalIgnoreCase)) + { + section = null; + continue; + } + if (section != "triangles") + continue; // nodes, skeleton, vertex animation: not part of the static mesh + + // A material line, then its triangle's three vertices. + cancellationToken.ThrowIfCancellationRequested(); + if (++triangles > MaxTriangles) + throw new InvalidDataException($"The SMD has more than {MaxTriangles:N0} triangles."); + var material = line; + if (!parts.TryGetValue(material, out var part)) + { + model.Materials.Add(new Material3D { Name = Path.GetFileNameWithoutExtension(material) }); + part = new MeshPart { Name = Path.GetFileNameWithoutExtension(material), Material = model.Materials.Count - 1, Normals = [], TexCoords = [] }; + parts[material] = part; + model.Parts.Add(part); + } + for (var k = 0; k < 3; k++) + { + var vertex = Next() ?? throw new InvalidDataException($"The SMD ends inside a triangle (line {lineNumber})."); + var fields = vertex.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + if (fields.Length < 9) + throw new InvalidDataException($"Line {lineNumber} of the SMD is not a triangle's vertex."); + var position = new Vector3(Number(fields[1], lineNumber), Number(fields[2], lineNumber), Number(fields[3], lineNumber)); + var normal = new Vector3(Number(fields[4], lineNumber), Number(fields[5], lineNumber), Number(fields[6], lineNumber)); + var uv = new Vector2(Number(fields[7], lineNumber), Number(fields[8], lineNumber)); + corners[k] = part.AddVertex(MeshMath.ToYUp(position) * SourceModel.MillimetresPerUnit, MeshMath.ToYUp(normal), uv); + } + part.AddTriangle(corners[0], corners[1], corners[2]); + progress?.Report(Math.Min(1, (double)stream.Position / length)); + } + return model; + } + + private static float Number(string text, int line) => + float.TryParse(text, NumberStyles.Float, Invariant, out var value) && float.IsFinite(value) + ? value + : throw new InvalidDataException($"Line {line} of the SMD has \"{text}\" where a number belongs."); + + /// A static prop: one root bone at the origin, every vertex weighted to it. + public static void Write(Model3D model, Stream stream, CancellationToken cancellationToken) + { + using var writer = new StreamWriter(stream, new UTF8Encoding(false), 1 << 16, leaveOpen: true) { NewLine = "\n" }; + writer.WriteLine("version 1"); + writer.WriteLine("nodes"); + writer.WriteLine("0 \"root\" -1"); + writer.WriteLine("end"); + writer.WriteLine("skeleton"); + writer.WriteLine("time 0"); + writer.WriteLine("0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000"); + writer.WriteLine("end"); + writer.WriteLine("triangles"); + foreach (var part in model.Parts) + { + if (part.Normals is null) + part.ComputeNormals(); + var material = MaterialName(model.MaterialOf(part), part.Name); + var indices = part.Indices; + for (var t = 0; t + 2 < indices.Count; t += 3) + { + if (t % 3000 == 0) + cancellationToken.ThrowIfCancellationRequested(); + writer.WriteLine(material); + for (var k = 0; k < 3; k++) + { + var v = indices[t + k]; + var position = MeshMath.ToZUp(part.Position(v)) / SourceModel.MillimetresPerUnit; + WriteVertex(writer, 0, position, MeshMath.ToZUp(part.Normal(v)), part.TexCoord(v)); + writer.WriteLine(" 1 0 1.000000"); + } + } + } + writer.WriteLine("end"); + } + + /// A Source model as a reference SMD: its bones in their bind pose, its triangles with their weights. + public static void WriteSkeletal(SourceModel model, Stream stream, CancellationToken cancellationToken) + { + using var writer = new StreamWriter(stream, new UTF8Encoding(false), 1 << 16, leaveOpen: true) { NewLine = "\n" }; + writer.WriteLine("version 1"); + writer.WriteLine("nodes"); + var bones = model.Bones.Count > 0 ? model.Bones : [new SourceModel.Bone("root", -1, Vector3.Zero, Vector3.Zero)]; + for (var i = 0; i < bones.Count; i++) + writer.WriteLine($"{i} \"{bones[i].Name.Replace('"', '\'')}\" {bones[i].Parent}"); + writer.WriteLine("end"); + writer.WriteLine("skeleton"); + writer.WriteLine("time 0"); + for (var i = 0; i < bones.Count; i++) + { + var (p, r) = (bones[i].Position, bones[i].Rotation); + writer.WriteLine(string.Create(Invariant, $"{i} {p.X:F6} {p.Y:F6} {p.Z:F6} {r.X:F6} {r.Y:F6} {r.Z:F6}")); + } + writer.WriteLine("end"); + writer.WriteLine("triangles"); + foreach (var mesh in model.Meshes) + { + cancellationToken.ThrowIfCancellationRequested(); + var material = Clean(mesh.Material); + // Source's triangles are clockwise; an SMD's counter-clockwise. + for (var t = 0; t + 2 < mesh.Triangles.Count; t += 3) + { + writer.WriteLine(material); + foreach (var index in new[] { mesh.Triangles[t + 2], mesh.Triangles[t + 1], mesh.Triangles[t] }) + { + var v = model.Vertices[index]; + var bone = Math.Clamp(v.Bone0, 0, bones.Count - 1); + WriteVertex(writer, bone, v.Position, v.Normal, new Vector2(v.Uv.X, 1 - v.Uv.Y)); + var links = new StringBuilder(); + var count = 0; + foreach (var (b, w) in new[] { (v.Bone0, v.Weight0), (v.Bone1, v.Weight1), (v.Bone2, v.Weight2) }.Take(v.BoneCount)) + { + if (b < 0 || b >= bones.Count) + continue; + links.Append(Invariant, $" {b} {w:F6}"); + count++; + } + writer.WriteLine(count > 0 ? $" {count}{links}" : $" 1 {bone} 1.000000"); + } + } + } + writer.WriteLine("end"); + } + + private static void WriteVertex(StreamWriter writer, int bone, Vector3 p, Vector3 n, Vector2 uv) => + writer.Write(string.Create(Invariant, $"{bone} {p.X:F6} {p.Y:F6} {p.Z:F6} {n.X:F6} {n.Y:F6} {n.Z:F6} {uv.X:F6} {uv.Y:F6}")); + + /// The material line: studiomdl looks the name up as a .vmt, so it is the material's or texture's name. + private static string MaterialName(Material3D? material, string partName) + { + var name = material?.Name is { Length: > 0 } named ? named + : material?.BaseColorTexture?.Name is { Length: > 0 } texture ? texture + : partName; + return Clean(name); + } + + /// One word without spaces or quotes; "default" when nothing is left. + private static string Clean(string name) + { + var clean = new string([.. name.Trim().Select(c => char.IsWhiteSpace(c) || c == '"' ? '_' : c)]); + return clean.Length == 0 ? "default" : clean; + } +} diff --git a/tests/Filee.Core.Tests/FormatFieldsTests.cs b/tests/Filee.Core.Tests/FormatFieldsTests.cs index 7c61bd9bd..465d8ed3a 100644 --- a/tests/Filee.Core.Tests/FormatFieldsTests.cs +++ b/tests/Filee.Core.Tests/FormatFieldsTests.cs @@ -33,7 +33,7 @@ private static string[] Hidden(IReadOnlySet turnedOff) => "lzma", "m4s", "man", "mcaddon", "mcpack", "mctemplate", "mcw", "mcworld", "mdl", "mediawiki", "miff", "mng", "mobi", "mod", "mp2", "mpc", "mrpack", "msix", "mxf", "nov", "nupkg", "obj", "oeb", "org", "osm", "otf", "pam", "pcd", "pcx", "pdb", "pdn", "pfm", "plist", "plt", "ply", "pml", "ppm", "prc", "ps", "psb", "psd", - "psw", "qoi", "ras", "raw", "rb", "rm", "rpm", "rst", "s3m", "scc", "sda", "sdc", "sdw", "sgi", "shp", "snb", "spx", + "psw", "qoi", "ras", "raw", "rb", "rm", "rpm", "rst", "s3m", "scc", "sda", "sdc", "sdw", "sgi", "shp", "smd", "snb", "spx", "ssa", "stl", "sub", "swf", "tak", "tar", "tar7z", "tbz2", "tcr", "tex", "textile", "tgz", "tlz", "toml", "tta", "ttf", "ttml", "txtz", "txz", "typ", "tz", "usm", "vob", "voc", "vpk", "vtf", "w64", "wbmp", "wmf", "woff", "woff2", "wpd", "wtv", "wv", "xbm", "xcf", "xlt", "xm", "xpm", "xz", "yaml", "z", "zxp" @@ -45,7 +45,7 @@ private static string[] Hidden(IReadOnlySet turnedOff) => { "model3d", ["fbx", "glb", "gltf", "ply"] }, { "geo", ["fit", "geojson", "gpx", "kml", "kmz", "osm", "shp"] }, { "font", ["otf", "ttf", "woff", "woff2"] }, - { "game", ["bik", "bsp", "dds", "gma", "mcaddon", "mcpack", "mctemplate", "mcworld", "mdl", "mrpack", "usm", "vpk", "vtf"] }, + { "game", ["bik", "bsp", "dds", "gma", "mcaddon", "mcpack", "mctemplate", "mcworld", "mdl", "mrpack", "smd", "usm", "vpk", "vtf"] }, { "design", ["ai", "cdr", "dst", "eps", "exr", "hdr", "kra", "pdn", "plt", "ps", "psb", "psd", "raw", "xcf"] }, { "science", ["dcm", "fits"] }, { "developer", [ @@ -72,7 +72,7 @@ private static string[] Hidden(IReadOnlySet turnedOff) => { "model3d", ["3mf", "fbx", "gcode", "glb", "gltf", "obj", "ply", "stl"] }, { "geo", ["fit", "geojson", "gpx", "kml", "kmz", "osm", "shp"] }, { "font", ["eot", "fon", "otf", "ttf", "woff", "woff2"] }, - { "game", ["bik", "bsp", "dds", "flc", "gma", "it", "mcaddon", "mcpack", "mctemplate", "mcworld", "mdl", "mrpack", "tga", "usm", "vpk", "vtf", "xm"] }, + { "game", ["bik", "bsp", "dds", "flc", "gma", "it", "mcaddon", "mcpack", "mctemplate", "mcworld", "mdl", "mrpack", "smd", "tga", "usm", "vpk", "vtf", "xm"] }, { "design", ["ai", "cdr", "dst", "eps", "exr", "hdr", "jp2", "kra", "pcd", "pdn", "plt", "ps", "psb", "psd", "raw", "xcf"] }, { "science", ["dcm", "fits", "ipynb", "jp2"] }, { "developer", [ @@ -145,7 +145,7 @@ public void Grade_counts_match_the_published_table() { [FormatGrade.Everyday] = 19, [FormatGrade.General] = 99, - [FormatGrade.Specialist] = 132, + [FormatGrade.Specialist] = 133, [FormatGrade.Legacy] = 52, }, counts); diff --git a/tests/Filee.Core.Tests/SourceFormatTests.cs b/tests/Filee.Core.Tests/SourceFormatTests.cs index c005fd27f..0ec083c63 100644 --- a/tests/Filee.Core.Tests/SourceFormatTests.cs +++ b/tests/Filee.Core.Tests/SourceFormatTests.cs @@ -1,6 +1,6 @@ // Source engine files (#1526): Garry's Mod addons, Valve paks and maps (for their packed files) open the archives donut, -// which existing libraries learn without taking a donut of the user's own, and Source models the 3D models donut; each -// is a target only where the Games field is on. +// which existing libraries learn without taking a donut of the user's own, and Source models and SMD the 3D models +// donut; each is a target only where the Games field is on. using Filee.Core.Formats; using Filee.Core.Presets; @@ -17,6 +17,7 @@ public class SourceFormatTests [InlineData("pak01_dir.vpk", "vpk", "archives")] [InlineData("gm_construct.bsp", "bsp", "archives")] [InlineData("gate_e2.mdl", "mdl", "models")] + [InlineData("gate_e2_reference.smd", "smd", "models")] public void Source_files_use_their_donut(string file, string format, string donut) { Assert.Same(FormatRegistry.Get(format), FormatRegistry.Detect(file)); @@ -28,6 +29,7 @@ public void Source_files_use_their_donut(string file, string format, string donu [InlineData("vpk", FormatCategory.Archive, false)] [InlineData("bsp", FormatCategory.Archive, false)] [InlineData("mdl", FormatCategory.Model3D, false)] + [InlineData("smd", FormatCategory.Model3D, true)] public void Source_files_are_game_formats(string id, FormatCategory category, bool writable) { var format = FormatRegistry.Get(id)!; @@ -37,6 +39,15 @@ public void Source_files_are_game_formats(string id, FormatCategory category, bo Assert.True(FormatFields.IsShown(id, new HashSet())); } + [Fact] + public void Smd_joins_the_3d_models_in_the_palette_after_gltf() + { + var presets = BuiltInData.CreatePresets(); + var ids = presets.Select(p => p.Id).ToList(); + Assert.Equal(ids.IndexOf("to-gltf") + 1, ids.IndexOf("to-smd")); + Assert.Equal("smd", presets.Single(p => p.Id == "to-smd").TargetFormat); + } + [Fact] public void Version_95_libraries_learn_the_source_files_without_taking_custom_claims() { @@ -75,16 +86,18 @@ public void Version_96_libraries_get_the_gma_preset_after_the_game_packs() } [Fact] - public void Version_99_libraries_learn_source_models_without_taking_custom_claims() + public void Version_99_libraries_learn_source_models_and_get_the_smd_preset() { var presets = BuiltInData.CreatePresets(); var profiles = BuiltInData.CreateProfiles(); var models = profiles.Single(p => p.Id == "models"); - models.Extensions.RemoveAll(e => e is "mdl"); + models.Extensions.RemoveAll(e => e is "mdl" or "smd"); + presets.RemoveAll(p => p.Id == "to-smd"); Assert.True(SettingsMigrations.ApplyToLibrary(99, presets, profiles)); Assert.Contains("mdl", models.Extensions); - Assert.Equal(BuiltInData.CreatePresets().Select(p => p.Id), presets.Select(p => p.Id)); // no preset added + Assert.Contains("smd", models.Extensions); + Assert.Equal(BuiltInData.CreatePresets().Select(p => p.Id), presets.Select(p => p.Id)); // "SMD" back after glTF Assert.False(SettingsMigrations.ApplyToLibrary(99, presets, profiles)); models.Extensions.Remove("mdl"); diff --git a/tests/Filee.Core.Tests/TranslatePresetMigrationTests.cs b/tests/Filee.Core.Tests/TranslatePresetMigrationTests.cs index c2b51c93e..8503ed71a 100644 --- a/tests/Filee.Core.Tests/TranslatePresetMigrationTests.cs +++ b/tests/Filee.Core.Tests/TranslatePresetMigrationTests.cs @@ -127,7 +127,7 @@ public void Translate_goes_after_the_built_in_preset_before_it() var defaults = BuiltInData.CreatePresets(); List Only(params string[] ids) => ids.Select(id => defaults.Single(p => p.Id == id)).ToList(); // Later steps add their own presets; what is checked here is where Translate lands. - static IEnumerable Ids(List presets) => presets.Select(p => p.Id).Where(id => id is not (BuiltInData.MetadataId or BuiltInData.UpscaleId or BuiltInData.RingtoneId or BuiltInData.PdfAId or "to-iso" or "to-gma" or BuiltInData.VtfId)); + static IEnumerable Ids(List presets) => presets.Select(p => p.Id).Where(id => id is not (BuiltInData.MetadataId or BuiltInData.UpscaleId or BuiltInData.RingtoneId or BuiltInData.PdfAId or "to-iso" or "to-gma" or BuiltInData.VtfId or "to-smd")); List Profiles() { var profiles = BuiltInData.CreateProfiles(); diff --git a/tests/Filee.Core.Tests/UserDataStoreTests.cs b/tests/Filee.Core.Tests/UserDataStoreTests.cs index f2081cf41..39a51415a 100644 --- a/tests/Filee.Core.Tests/UserDataStoreTests.cs +++ b/tests/Filee.Core.Tests/UserDataStoreTests.cs @@ -1060,7 +1060,7 @@ public void Version_14_libraries_get_3d_model_presets_and_their_donut_after_data Assert.Equal(ids.IndexOf("models") + 1, ids.IndexOf("geo")); // and, from schema 25, the geospatial donut Assert.Equal(ids.IndexOf("mixed") - 1, ids.IndexOf("geo")); var models = profiles.Single(p => p.Id == "models"); - Assert.Equal(["glb", "gltf", "stl", "obj", "3mf", "ply", "fbx", "mdl"], models.Extensions); + Assert.Equal(["glb", "gltf", "stl", "obj", "3mf", "ply", "fbx", "mdl", "smd"], models.Extensions); Assert.Equal(BuiltInData.ModelProfile().PresetIds, models.PresetIds); Assert.False(SettingsMigrations.ApplyToLibrary(14, presets, profiles)); // once only } @@ -1074,7 +1074,7 @@ public void The_3d_donut_leaves_out_extensions_a_profile_of_the_user_takes() Assert.True(SettingsMigrations.ApplyToLibrary(14, presets, profiles)); - Assert.Equal(["glb", "gltf", "obj", "ply", "fbx", "mdl"], profiles.Single(p => p.Id == "models").Extensions); + Assert.Equal(["glb", "gltf", "obj", "ply", "fbx", "mdl", "smd"], profiles.Single(p => p.Id == "models").Extensions); } [Fact] @@ -1088,7 +1088,7 @@ public void Version_21_libraries_learn_fbx_on_the_3d_donut_and_nothing_else() Assert.True(SettingsMigrations.ApplyToLibrary(21, presets, profiles)); - Assert.Equal(["glb", "gltf", "stl", "obj", "3mf", "mdl", "fbx"], models.Extensions); + Assert.Equal(["glb", "gltf", "stl", "obj", "3mf", "mdl", "smd", "fbx"], models.Extensions); Assert.False(SettingsMigrations.ApplyToLibrary(21, presets, profiles)); // once only // A donut of the user that already opens FBX files keeps them. diff --git a/tests/Filee.Engines.Tests/SmdTests.cs b/tests/Filee.Engines.Tests/SmdTests.cs new file mode 100644 index 000000000..ffb0b3612 --- /dev/null +++ b/tests/Filee.Engines.Tests/SmdTests.cs @@ -0,0 +1,149 @@ +// Studiomdl data (.smd, #1526): an MDL becomes a reference SMD with its bones, bind pose and weights (as studiomdl +// reads it: counter-clockwise, UVs from the bottom left, inches, Z up); other models become a static prop with one root +// bone; an SMD's triangles become a model again. + +using System.Globalization; +using System.Numerics; +using Filee.Core.Conversion; +using Filee.Core.Presets; +using Filee.Engines.Models3D; +using static Filee.Engines.Tests.SourceModelBuilder; + +namespace Filee.Engines.Tests; + +public class SmdTests(EngineFixture fx) : IClassFixture +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private async Task ConvertOkAsync(string source, string target) + { + var job = await fx.ConvertAsync([source], new Preset { TargetFormat = target }); + Assert.True(job.State == JobState.Completed, string.Join("; ", job.Files.Select(f => $"{f.ErrorKey} {f.ErrorDetail}"))); + return Assert.Single(job.Outputs); + } + + /// The lines of a section ("nodes", "skeleton", "triangles") up to its "end". + private static List Section(string[] lines, string name) + { + var start = Array.IndexOf(lines, name); + Assert.True(start >= 0, $"no {name} section"); + return [.. lines.Skip(start + 1).TakeWhile(l => l != "end")]; + } + + private static float[] Numbers(string line) => + [.. line.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(f => float.Parse(f, CultureInfo.InvariantCulture))]; + + [Fact] + public void Smd_converts_to_and_from_the_3d_formats_in_one_step() + { + var planner = fx.Catalog.CreatePlanner(["model3d"]); + foreach (var (from, to) in new[] { ("mdl", "smd"), ("obj", "smd"), ("glb", "smd"), ("fbx", "smd"), ("smd", "glb"), ("smd", "stl"), ("smd", "obj") }) + Assert.Single(planner.Plan(from, to)!.Steps); + } + + [Fact] + public async Task A_model_becomes_a_reference_smd_with_its_bones_and_weights() + { + Bone[] bones = + [ + new("root", -1, Vector3.Zero, Vector3.Zero), + new("lid", 0, new Vector3(0, 0, 5), new Vector3(0, 0, MathF.PI / 2)), + ]; + var mdl = Write(fx.NewFolder(), "box", "", ["brick", "glass"], TileAndTriangle(weighted: true), bones); + + var smd = await ConvertOkAsync(mdl, "smd"); + + var lines = File.ReadAllLines(smd); + Assert.Equal("version 1", lines[0]); + Assert.Equal(["0 \"root\" -1", "1 \"lid\" 0"], Section(lines, "nodes")); + var skeleton = Section(lines, "skeleton"); + Assert.Equal("time 0", skeleton[0]); + Assert.Equal([1, 0, 0, 5, 0, 0, MathF.PI / 2], Numbers(skeleton[2]), new FloatComparer()); + + var triangles = Section(lines, "triangles"); + Assert.Equal(3 * 4, triangles.Count); // three triangles, each a material line and three vertices + Assert.Equal(["brick", "brick", "glass"], new[] { triangles[0], triangles[4], triangles[8] }); + // The tile's first triangle (0, 1, 2) in Source's order comes reversed: (10, 10, 0), (0, 10, 0), (0, 0, 0). + Assert.Equal([0, 10, 10, 0, 0, 0, 1, 1, 1, 1, 0, 1], Numbers(triangles[1]), new FloatComparer()); + Assert.Equal([0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1], Numbers(triangles[3]), new FloatComparer()); + // Its corner at (10, 0, 0) is weighted 0.75 / 0.25 to both bones; UV (1, 1) from the top is (1, 0) from the bottom. + Assert.Equal([0, 10, 0, 0, 0, 0, 1, 1, 0, 2, 0, 0.75f, 1, 0.25f], Numbers(triangles[5]), new FloatComparer()); + // The triangle above it hangs on the lid. + Assert.Equal(1, Numbers(triangles[9])[0]); + Assert.Equal([1, 1, 1], Numbers(triangles[9])[^3..], new FloatComparer()); + } + + [Fact] + public async Task An_smd_becomes_a_model_facing_the_same_way_as_its_mdl() + { + var mdl = Write(fx.NewFolder(), "box", "", ["brick", "glass"], TileAndTriangle()); + var fromMdl = ModelConverter.Read(mdl, "mdl", null, Ct); + + var smd = await ConvertOkAsync(mdl, "smd"); + var fromSmd = ModelConverter.Read(smd, "smd", null, Ct); + + Assert.Equal(fromMdl.TriangleCount, fromSmd.TriangleCount); + Assert.Equal(fromMdl.Bounds(), fromSmd.Bounds()); + Assert.Equal(["brick", "glass"], fromSmd.Materials.Select(m => m.Name)); + foreach (var part in fromSmd.Parts) + { + for (var t = 0; t < part.TriangleCount; t++) + Assert.True(Vector3.Dot(part.FaceNormal(t), Vector3.UnitY) > 0.99f); + } + + var glb = await ConvertOkAsync(smd, "glb"); + Assert.Equal(3, ModelConverter.Read(glb, "glb", null, Ct).TriangleCount); + } + + [Fact] + public void Another_model_becomes_a_static_prop_and_reads_back() + { + var model = new Model3D(); + model.Materials.Add(new Material3D { Name = "Painted Metal" }); + var part = new MeshPart { Name = "plate", Material = 0, TexCoords = [] }; + part.AddVertex(new Vector3(0, 0, 0), uv: new Vector2(0, 0)); + part.AddVertex(new Vector3(254, 0, 0), uv: new Vector2(1, 0)); + part.AddVertex(new Vector3(0, 0, -254), uv: new Vector2(0, 1)); + part.AddTriangle(0, 1, 2); + model.Parts.Add(part); + + var path = Path.Combine(fx.NewFolder(), "plate.smd"); + using (var stream = File.Create(path)) + SmdFormat.Write(model, stream, Ct); + + var lines = File.ReadAllLines(path); + Assert.Equal(["0 \"root\" -1"], Section(lines, "nodes")); + var triangles = Section(lines, "triangles"); + Assert.Equal("Painted_Metal", triangles[0]); + // 254 mm along X is 10 inches; −254 mm along Z (Y up) is +10 inches along Y (Z up). + Assert.Equal([0, 10, 0, 0], Numbers(triangles[2])[..4], new FloatComparer()); + Assert.Equal([0, 0, 10, 0], Numbers(triangles[3])[..4], new FloatComparer()); + Assert.EndsWith(" 1 0 1.000000", triangles[1]); + + var back = SmdFormat.Read(path, null, Ct); + back.Validate(); + Assert.Equal(1, back.TriangleCount); + Assert.True(Vector3.Distance(model.Bounds().Max, back.Bounds().Max) < 0.01f); + Assert.True(Vector3.Distance(model.Bounds().Min, back.Bounds().Min) < 0.01f); + Assert.Equal(new Vector2(1, 0), back.Parts[0].TexCoord(1)); + } + + [Fact] + public void A_broken_smd_is_refused_with_its_line() + { + var path = Path.Combine(fx.NewFolder(), "broken.smd"); + File.WriteAllText(path, "version 1\ntriangles\nmat\n0 1 2 3 0 0 1 0 0\n0 1 2 x 0 0 1 0 0\n"); + var error = Assert.Throws(() => SmdFormat.Read(path, null, Ct)); + Assert.Contains("Line 5", error.Message); + + File.WriteAllText(path, "version 1\nnodes\n0 \"root\" -1\nend\nskeleton\ntime 0\n0 0 0 0 0 0 0\nend\n"); + var animation = Assert.Throws(() => ModelConverter.Read(path, "smd", null, Ct)); + Assert.Contains("no triangles", animation.Message); + } + + private sealed class FloatComparer : IEqualityComparer + { + public bool Equals(float a, float b) => MathF.Abs(a - b) < 0.0005f; + public int GetHashCode(float value) => 0; + } +}