From 49fae65f8cb95ef9f9c51997cc98f5be81397f98 Mon Sep 17 00:00:00 2001 From: KnifeLemon Date: Thu, 19 Nov 2026 13:40:18 +0900 Subject: [PATCH 1/5] Read Zstandard files: .zst, .tar.zst and .tzst (#1646) ZST and TAR.ZST become archive formats like XZ and TAR.XZ: recognised by extension, unpacked in-process (the TAR in a second stage), converted to every archive target, and ZST to the other single-file formats. ZstdReader puts SharpCompress's decoder behind Filee's checks: the first frame's magic (other data is "not an archive", not an empty file), frames one after another and skippable frames, damage and early ends as "damaged", windows over 128 MiB left to 7-Zip. RPM payloads in zstd use the same reader. Read only: writing would add a developer-only output, which FORMAT-RULES rule 5 keeps out while the developer field is on by default. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/ENGINES.md | 7 +- src/Filee.Core/Formats/FormatFields.cs | 8 +- src/Filee.Core/Formats/FormatRegistry.cs | 4 + .../Archives/ArchiveConverter.cs | 2 +- .../Archives/ArchiveExtractor.cs | 10 +-- src/Filee.Engines/Archives/CompressedFile.cs | 14 +-- src/Filee.Engines/Archives/ZstdReader.cs | 89 +++++++++++++++++++ tests/Filee.Core.Tests/FormatFieldsTests.cs | 11 +-- .../ArchiveFormatBuilders.cs | 13 +++ tests/Filee.Engines.Tests/ArchiveFuzzTests.cs | 3 +- .../ArchiveInProcessTests.cs | 40 ++++++++- tests/Filee.Engines.Tests/ArchiveTests.cs | 4 + 13 files changed, 178 insertions(+), 29 deletions(-) create mode 100644 src/Filee.Engines/Archives/ZstdReader.cs diff --git a/README.md b/README.md index 8507f7781..8ffff6341 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ To undo it, remove the entries in Windows Security → App & browser control → | **Video**⁵: MP4, MOV, MKV, WEBM, AVI, WMV, FLV, MPEG, Video CD (DAT), TS, M2TS, 3GP, OGV, VOB, …;
Wisenet WAVE / Nx Witness exports (NOV) read | MP4 (H.264, HEVC, AV1), WEBM, MOV, MKV, AVI, GIF, APNG, WEBP, MP3, M4A, 720p, split into parts;
subtitle tracks kept (MKV, MP4, MOV, WebM);
subtitles (SRT, ASS, WebVTT) | | **Audio**⁵: MP3, M4A, AAC, WAV, FLAC, OGG, OPUS, WMA, AIFF, AMR, APE, WavPack, TTA, DSD (DSF, DFF), DTS, AC3, E-AC3, …;
tracker modules (XM, IT, S3M) read | MP3, M4A (AAC, ALAC), M4R (iPhone ringtone), WAV, FLAC, OGG, OPUS, AAC, WavPack, TTA, W64, AC3, E-AC3 | | **Subtitles**: SRT, ASS/SSA, WebVTT, SAMI (SMI), LRC, KRC, TTML, MicroDVD, SCC | each other as UTF-8 (Korean CP949 files read correctly) | -| **Archives**: ZIP, ANSYS Workbench (WBPZ), 7Z, RAR, TAR, TAR.GZ, TAR.BZ2, TAR.XZ, GZ, XZ, ISO, UDF, IMG, DMG, CAB, LHA, ARJ, ALZ, EGG, Garry's Mod addon (GMA), Valve pak (VPK), files packed into a Source map (BSP), …;
JAR, NuGet (NUPKG), Android packages (APKS, XAPK, APKM, AAB), Adobe extensions (ZXP) read | extract, ZIP, 7Z, TAR, TAR.GZ, TAR.XZ, XZ, GMA;
any files → one ZIP; folders → ZIP, ISO (Joliet or UDF 2.01), GMA | +| **Archives**: ZIP, ANSYS Workbench (WBPZ), 7Z, RAR, TAR, TAR.GZ, TAR.BZ2, TAR.XZ, TAR.ZST, GZ, XZ, ZST, ISO, UDF, IMG, DMG, CAB, LHA, ARJ, ALZ, EGG, Garry's Mod addon (GMA), Valve pak (VPK), files packed into a Source map (BSP), …;
JAR, NuGet (NUPKG), Android packages (APKS, XAPK, APKM, AAB), Adobe extensions (ZXP) read | extract, ZIP, 7Z, TAR, TAR.GZ, TAR.XZ, XZ, GMA;
any files → one ZIP; folders → ZIP, ISO (Joliet or UDF 2.01), GMA | | **CAD**: DWG, DXF
Gerber PCB layers (GBR, GTL, GBL, GTO, …) read;
G-code of 3D printers and CNC machines (GCODE, GCO, NGC) read | PDF, SVG, PNG, DWG ↔ DXF;
G-code → toolpath (from above or isometric) or the slicer's preview | | **Fonts**: TTF, OTF, WOFF, WOFF2, EOT;
FON (Windows bitmap fonts) read | each other;
FON → one font per size | | **Data**: JSON, JSON Lines, YAML, TOML, XML
vCard, iCalendar | each other, CSV and XLSX (tables);
vCard, iCalendar ↔ CSV | diff --git a/docs/ENGINES.md b/docs/ENGINES.md index 2ec22312c..173b1aec9 100644 --- a/docs/ENGINES.md +++ b/docs/ENGINES.md @@ -41,7 +41,7 @@ Filee chooses engines automatically. Settings → *Engines* shows their status, | **Geospatial vector data** | built in | **GeoJSON, KML, KMZ, GPX, Shapefile ↔ each other and CSV** (longitude / latitude or WKT); Garmin FIT (activities) and OpenStreetMap XML (`.osm`), both read only, → each of them and CSV; projected Shapefiles (UTM, Web Mercator, Lambert, Albers) are converted to WGS 84; through CSV to XLSX, JSON, … (see below) | | **Filee renderer** | built in, on by default (Settings → Engines → *Use the Filee renderer*) | **HWPX, HWP, Word documents (DOCX and DOC), ODT, RTF, TXT, Markdown, HTML, e-books, CSV / TSV, OFD, the markup languages Filee reads itself (and Pandoc's formats) → PDF** with Filee's own layout engine; **PPTX, PPTM, POTX, POTM, PPSX, PPT ([PPT reader](#ppt-reader)), ODP, OTP, FODP ([ODP / ODG reader](#odp--odg-reader)) → PDF, PNG, JPG** with its slide renderer; files with what it doesn't draw yet take the HWPX writer and rhwp (see [Filee renderer: PDF output](#filee-renderer-pdf-output) and [slides](#filee-renderer-slides)) | | rhwp | bundled with the installer (`engines/rhwp`, with the Microsoft C++ runtime, see below) | HWP/HWPX → PDF when the Filee renderer is off or can't draw a file yet, **HWPX → HWP** (HWP → HWPX too, ranked below the built-in HWP reader) | -| Archives | built in (SharpCompress, DiscUtils, .NET and Filee's own readers and writers); 7-Zip optional (Settings → *Engines*, ~2 MB download) | **in-process**: ZIP (including ANSYS Workbench WBPZ and Adobe ZXP, and Windows app packages MSIX / APPX), Minecraft and Modrinth packs, Garry's Mod addons (GMA), Valve paks (VPK), the files packed into Source maps (BSP), 7Z (solid ones too), RAR, TAR (+ GZ/BZ2/XZ/Z/7Z/LZ), GZ, BZ2, XZ, LZMA, Z, lzip, CPIO, DEB, RPM, ISO (UDF discs too), IMG and DMG disk images, split (.001), ALZ, EGG, CAB, LHA, ARJ; **7-Zip, when installed**: the rare variants the in-process readers leave to it (see [Archives](#archives)) **→ folder, ZIP, 7Z, TAR, TAR.GZ, TAR.BZ2, TAR.XZ, BZ2, XZ, the packs, GMA** (in-process); "Compress into one archive" for any files | +| Archives | built in (SharpCompress, DiscUtils, .NET and Filee's own readers and writers); 7-Zip optional (Settings → *Engines*, ~2 MB download) | **in-process**: ZIP (including ANSYS Workbench WBPZ and Adobe ZXP, and Windows app packages MSIX / APPX), Minecraft and Modrinth packs, Garry's Mod addons (GMA), Valve paks (VPK), the files packed into Source maps (BSP), 7Z (solid ones too), RAR, TAR (+ GZ/BZ2/XZ/Z/7Z/LZ/ZST), GZ, BZ2, XZ, LZMA, Z, ZST, lzip, CPIO, DEB, RPM, ISO (UDF discs too), IMG and DMG disk images, split (.001), ALZ, EGG, CAB, LHA, ARJ; **7-Zip, when installed**: the rare variants the in-process readers leave to it (see [Archives](#archives)) **→ folder, ZIP, 7Z, TAR, TAR.GZ, TAR.BZ2, TAR.XZ, BZ2, XZ, the packs, GMA** (in-process); "Compress into one archive" for any files | | LibreOffice | **optional download** (~370 MB, 1.1 GB on disk) | older and rare formats only: Word 6.0 / 95 and Works documents, PowerPoint 95 and older PPT, binary Visio (`.vsd`), and ODP / ODG with what Filee doesn't draw yet (charts, OLE objects, 3D) → PDF; XLS, PPT, RTF and OpenDocument to each other, output as DOCX/ODT/ODS, and ODP for what the [ODP writer](#odp-writer) doesn't take (OpenDocument presentations themselves, Keynote, decks with charts, effects or 3D); read-only formats (see below) → PDF and their kind's editable formats; FODT, OpenDocument templates (OTT, OTS, OTP) and XLT out; SVG → EMF/WMF/ODG, a PDF page → SVG | | Pandoc | **optional download** (~42 MB, 240 MB on disk) | Markdown ↔ DOCX/ODT/RTF, DOCX/ODT/HTML/RTF → Markdown, HTML ↔ DOCX/ODT, markup languages (LaTeX, AsciiDoc, Typst, man; and RTF / EPUB from the built-in ones) ↔ Markdown/HTML/DOCX/ODT and each other (and → RTF, EPUB, TXT); their AST feeds the built-in PPTX writer (Pandoc's own slides only where Filee assumes a Pandoc that isn't installed) | | Ghostscript | **optional download** (~20 MB, 31 MB on disk) | EPS/PS → PDF, PostScript-based AI → PDF (images through PDF and PDFium), EPS/PS → PNG; PDF → EPS/PS only when moved above the built-in PostScript writer; smaller PDFs when a preset asks for it; PDF/A-2b for PDF/A presets; renders CMYK PDF/AI pages for CMYK JPG/TIFF | @@ -3436,11 +3436,12 @@ paths leaving the folder, bombs, entry floods) are final and never retried. Sett | 7Z | Filee (`SevenZipReader`: Copy, LZMA, LZMA2, BCJ, BCJ2, ARM64, ARM, ARMT, PPC, SPARC, IA64, Delta, Deflate, BZip2; solid blocks decoded once, see below); other methods (PPMd, RISC-V …) by 7-Zip when it is installed, else SharpCompress, else the method is named in the "install 7-Zip" error | in-process (Filee's `SevenZipWriter`, LZMA2): UTF-16 names, times, attributes, empty folders | | RAR 4 / 5 | SharpCompress (only unpacks, under the unRAR terms, see THIRD-PARTY-NOTICES), with the other volumes (`.part2.rar`, `.r00`) | — | | TAR | .NET's System.Formats.Tar (ustar, PAX, GNU, V7): tells symbolic from hard links | in-process (System.Formats.Tar, PAX: UTF-8 names, times) | -| TAR.GZ, TAR.BZ2, TAR.XZ, TAR.Z, TAR.LZ, TAR.7Z | in two stages: the outer layer as below, then the TAR | TAR.GZ, TAR.BZ2 and TAR.XZ in-process | +| TAR.GZ, TAR.BZ2, TAR.XZ, TAR.Z, TAR.LZ, TAR.ZST, TAR.7Z | in two stages: the outer layer as below, then the TAR | TAR.GZ, TAR.BZ2 and TAR.XZ in-process | | GZ | .NET's gzip decoder (CRC-32 checked) | in-process (GZipStream); one file only | | BZ2 | SharpCompress | in-process (Filee's `Bzip2Writer`); one file only | | XZ | Filee (`XzReader`) | in-process (Filee's `XzWriter`); one file only | | LZMA (`.lzma`) | Filee (`LzmaRawReader`) | — | +| ZST (Zstandard, #1646) | SharpCompress through `ZstdReader`: frames one after another, skippable frames passed over; not starting with a frame is "not an archive"; windows over 128 MiB (`zstd --long=28` and up) are left to 7-Zip, as the zstd tool needs `--memory` for them | — (rule 5 of [FORMAT-RULES](FORMAT-RULES.md): a new output only developers make waits while that field is on by default) | | Z (Unix compress) | Filee (`UnixCompressStream`: LZW, 9–16-bit codes, block mode with table clears) | — | | LZ (lzip) | Filee: members found from the end, each LZMA stream decoded by SharpCompress, CRC-32 and sizes checked (data longer than the trailer states is refused) | — | | CPIO | Filee (`CpioReader`: newc, crc with its byte sum checked, odc, old binary in either byte order; newc hard link groups become copies) | — | @@ -3541,7 +3542,7 @@ paths leaving the folder, bombs, entry floods) are final and never retried. Sett levels and chunk buffers over 64 MB are refused before DiscUtils reads them, and DiscUtils itself only reads through a window with a budget per call (256 MB, plus the file being copied) and a stack check, so structures that lead in circles end as damage instead of running on. -- **Single compressed streams** (GZ, BZ2, XZ, LZMA, Z, and the first stage of TAR.GZ, TAR.BZ2, TAR.XZ, TAR.Z) state +- **Single compressed streams** (GZ, BZ2, XZ, LZMA, Z, ZST, and the first stage of TAR.GZ, TAR.BZ2, TAR.XZ, TAR.Z, TAR.ZST) state no size (bzip2) or one their author chose (gzip, modulo 4 GiB), so the listing can't be trusted: the in-process decoders stop once one passes the room on the drive less a tenth of the free space (`ArchiveSafety.StreamLimit`, "unpacks to more than …, but only … is free") and delete what was written; when 7-Zip decodes one (the fallback), diff --git a/src/Filee.Core/Formats/FormatFields.cs b/src/Filee.Core/Formats/FormatFields.cs index 94e6aade5..e03d88b1e 100644 --- a/src/Filee.Core/Formats/FormatFields.cs +++ b/src/Filee.Core/Formats/FormatFields.cs @@ -80,8 +80,8 @@ .. Enum.GetValues().Select(category => category.ToString().ToLow "fodt", "fods", "fodp", "rst", "tex", "org", "adoc", "typ", "mediawiki", "dbk", "ipynb", "textile", "man", "jsonl", "yaml", "toml", "har", "plist", "htmlz", "txtz", "chm", "dv", "mxf", "cavs", "usm", "m4s", "dav", "nov", "bik", "h265", "h264", "ac3", "caf", "mp2", "dss", "wv", "tta", "w64", "eac3", "ape", "tak", "dsf", "dff", - "dts", "tar", "tgz", "tbz2", "txz", "tar7z", "tlz", "gz", "bz2", "xz", "lz", "lzma", "jar", "zxp", "nupkg", "apks", - "aab", "msix", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", "cab", "gerber", + "dts", "tar", "tgz", "tbz2", "txz", "tar7z", "tlz", "tzst", "gz", "bz2", "xz", "lz", "lzma", "zst", "jar", "zxp", + "nupkg", "apks", "aab", "msix", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", "cab", "gerber", "cpio", "deb", "rpm", "dwg", "ttf", "otf", "woff", "woff2", "ass", "ssa", "ttml", "scc", "glb", "gltf", "stl", "obj", "3mf", "ply", "fbx", "gcode", "geojson", "kml", "kmz", "gpx", "shp", "fit", "osm", "xm", "it", "ani", "vtf", "gma", "vpk", "bsp" ]), @@ -107,8 +107,8 @@ .. Enum.GetValues().Select(category => category.ToString().ToLow // ODF for version control, man pages, streaming segments, packages and compressed archives. ([ "ppm", "icns", "qoi", "cur", "ani", "xpm", "xbm", "pam", "pfm", "miff", "fodg", "fodt", "fods", "fodp", "man", - "jsonl", "yaml", "toml", "har", "plist", "m4s", "tar", "tgz", "tbz2", "txz", "tar7z", "tlz", "gz", "bz2", "xz", - "lz", "lzma", "jar", "zxp", "nupkg", "apks", "aab", "msix", "cab", "cpio", "deb", "rpm" + "jsonl", "yaml", "toml", "har", "plist", "m4s", "tar", "tgz", "tbz2", "txz", "tar7z", "tlz", "tzst", "gz", "bz2", + "xz", "lz", "lzma", "zst", "jar", "zxp", "nupkg", "apks", "aab", "msix", "cab", "cpio", "deb", "rpm" ], ["developer"]), // Markup sources, papers and outlines, maintenance-doc drawings (CGM), Calibre's ebook formats. (["cgm", "rst", "tex", "org", "adoc", "typ", "mediawiki", "dbk", "textile", "htmlz", "txtz"], ["writing"]), diff --git a/src/Filee.Core/Formats/FormatRegistry.cs b/src/Filee.Core/Formats/FormatRegistry.cs index 1f3097de2..afce02771 100644 --- a/src/Filee.Core/Formats/FormatRegistry.cs +++ b/src/Filee.Core/Formats/FormatRegistry.cs @@ -332,12 +332,16 @@ public static class FormatRegistry new("tar7z", "TAR.7Z", FormatCategory.Archive, ["tar.7z"], Writable: false), new("tlz", "TAR.LZ", FormatCategory.Archive, ["tar.lz", "tlz"], Writable: false), new("tz", "TAR.Z", FormatCategory.Archive, ["tar.z", "tz"], Writable: false), + // Zstandard (#1646): read only. Writing would add a developer-only output, which docs/FORMAT-RULES.md rule 5 + // keeps out while the developer field is on by default. + new("tzst", "TAR.ZST", FormatCategory.Archive, ["tar.zst", "tzst"], Writable: false), new("gz", "GZ", FormatCategory.Archive, ["gz"]), new("bz2", "BZ2", FormatCategory.Archive, ["bz2", "bz"]), new("xz", "XZ", FormatCategory.Archive, ["xz"]), new("lz", "LZ", FormatCategory.Archive, ["lz"], Writable: false), new("lzma", "LZMA", FormatCategory.Archive, ["lzma"], Writable: false), new("z", "Z", FormatCategory.Archive, ["z"], Writable: false), + new("zst", "ZST", FormatCategory.Archive, ["zst"], Writable: false), new("jar", "JAR", FormatCategory.Archive, ["jar"], Writable: false), // Adobe extension packages for After Effects, Premiere Pro, Photoshop … (#1382): signed ZIPs. Never a target, // since installing one needs the signature its developer makes with Adobe's ZXPSignCmd. diff --git a/src/Filee.Engines/Archives/ArchiveConverter.cs b/src/Filee.Engines/Archives/ArchiveConverter.cs index f69eb3d1a..51e4fa96e 100644 --- a/src/Filee.Engines/Archives/ArchiveConverter.cs +++ b/src/Filee.Engines/Archives/ArchiveConverter.cs @@ -24,7 +24,7 @@ public sealed class ArchiveConverter : IConverter, IFileCombiner private static readonly string[] ArchiveTargets = ["zip", "7z", "tar", "tgz", "tbz2", "txz", "gma", .. GamePacks.Formats]; /// Compressed single files (no names or folders inside); they convert among each other. - private static readonly string[] SingleFileSources = ["gz", "bz2", "xz", "lzma", "z", "lz"]; + private static readonly string[] SingleFileSources = ["gz", "bz2", "xz", "lzma", "z", "lz", "zst"]; private static readonly IReadOnlyList AllEdges = CreateEdges(); diff --git a/src/Filee.Engines/Archives/ArchiveExtractor.cs b/src/Filee.Engines/Archives/ArchiveExtractor.cs index f4de44910..40a0ea258 100644 --- a/src/Filee.Engines/Archives/ArchiveExtractor.cs +++ b/src/Filee.Engines/Archives/ArchiveExtractor.cs @@ -38,7 +38,7 @@ internal sealed class ArchiveExtractor(SevenZip? sevenZip, Func? s public static readonly IReadOnlyList BuiltInFormats = [ "zip", "jar", "zxp", "nupkg", "apks", "aab", "msix", "7z", "rar", "tar", "tgz", "tbz2", "txz", "tar7z", "tz", "tlz", - "gz", "bz2", "xz", "lzma", "z", "lz", "cpio", "deb", "rpm", "iso", "alz", "egg", "split", "cab", "lha", "arj", "dmg", + "tzst", "gz", "bz2", "xz", "lzma", "z", "lz", "zst", "cpio", "deb", "rpm", "iso", "alz", "egg", "split", "cab", "lha", "arj", "dmg", "img", "gma", "vpk", "bsp", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", ]; @@ -46,19 +46,19 @@ internal sealed class ArchiveExtractor(SevenZip? sevenZip, Func? s /// Formats 7-Zip can read too, so it may try again after an in-process reader gave up. private static readonly HashSet SevenZipAlsoReads = [ - "zip", "jar", "zxp", "nupkg", "apks", "aab", "msix", "7z", "rar", "tar", "tgz", "tbz2", "txz", "tar7z", "tz", "gz", - "bz2", "xz", "lzma", "z", "cpio", "deb", "rpm", "iso", "split", "cab", "lha", "arj", "dmg", "img", "mcpack", + "zip", "jar", "zxp", "nupkg", "apks", "aab", "msix", "7z", "rar", "tar", "tgz", "tbz2", "txz", "tar7z", "tz", "tzst", + "gz", "bz2", "xz", "lzma", "z", "zst", "cpio", "deb", "rpm", "iso", "split", "cab", "lha", "arj", "dmg", "img", "mcpack", "mcaddon", "mcworld", "mctemplate", "mrpack", ]; /// A TAR inside a compressed stream (or a 7z): unpacking the outer layer yields one .tar file. - private static readonly HashSet TarWrapped = ["tgz", "tbz2", "txz", "tar7z", "tz", "tlz"]; + private static readonly HashSet TarWrapped = ["tgz", "tbz2", "txz", "tar7z", "tz", "tlz", "tzst"]; /// /// One compressed stream, no names or sizes to check beforehand (bzip2 states no size, gzip one modulo 4 GiB that /// its author chose). /// - private static readonly HashSet SingleStream = ["gz", "bz2", "xz", "lzma", "z", "zst", "tgz", "tbz2", "txz", "tz"]; + private static readonly HashSet SingleStream = ["gz", "bz2", "xz", "lzma", "z", "zst", "tgz", "tbz2", "txz", "tz", "tzst"]; /// Formats an RPM payload comes in (7-Zip shows it as one compressed CPIO file). private static readonly HashSet RpmPayload = ["cpio", "gz", "bz2", "xz", "lzma", "z", "zst"]; diff --git a/src/Filee.Engines/Archives/CompressedFile.cs b/src/Filee.Engines/Archives/CompressedFile.cs index e11345e2f..ce8f724a0 100644 --- a/src/Filee.Engines/Archives/CompressedFile.cs +++ b/src/Filee.Engines/Archives/CompressedFile.cs @@ -1,14 +1,13 @@ -// Unpacks a compressed single file (.gz, .bz2, .xz, .lzma, .Z, .zst and the outer layer of .tar.gz, .tar.bz2, .tar.xz -// and .tar.Z) in-process: gzip with .NET's own decoder, zstd with SharpCompress (MIT), bzip2, xz, LZMA and Unix -// compress with Filee's own decoders (Bzip2Decoder, XzReader, LzmaRawReader, UnixCompressStream; bzip2 with -// randomised blocks from old encoders goes back to SharpCompress). None of these +// Unpacks a compressed single file (.gz, .bz2, .xz, .lzma, .Z, .zst and the outer layer of .tar.gz, .tar.bz2, .tar.xz, +// .tar.Z and .tar.zst) in-process: gzip with .NET's own decoder, zstd with SharpCompress (MIT, see ZstdReader), bzip2, +// xz, LZMA and Unix compress with Filee's own decoders (Bzip2Decoder, XzReader, LzmaRawReader, UnixCompressStream; +// bzip2 with randomised blocks from old encoders goes back to SharpCompress). None of these // formats declares a size it must keep (gzip's is the size modulo 4 GiB), so what comes out is counted by // UnpackBudget against bombs and a full drive. using System.IO.Compression; using Filee.Engines.Archives.Lzma; using SharpCompress.Compressors.BZip2; -using SharpCompress.Compressors.ZStandard; using SharpCompressMode = SharpCompress.Compressors.CompressionMode; namespace Filee.Engines.Archives; @@ -18,7 +17,7 @@ internal static class CompressedFile { /// Formats (and TAR wrappers) reads. public static readonly IReadOnlySet Formats = - new HashSet { "gz", "tgz", "bz2", "tbz2", "xz", "txz", "lzma", "z", "tz", "zst" }; + new HashSet { "gz", "tgz", "bz2", "tbz2", "xz", "txz", "lzma", "z", "tz", "zst", "tzst" }; /// /// The largest dictionary a .lzma (or EGG LZMA) header may name: real files use at most a few hundred MiB. (Filee's @@ -90,7 +89,8 @@ internal static Stream Open(string format, Stream input, string name, ref long d case "tz": return new UnixCompressStream(new NonClosingStream(input)); case "zst": - return new DecompressionStream(new NonClosingStream(input)); + case "tzst": + return new ZstdReader(input, name); case "lzma": { // "LZMA alone": properties (lc/lp/pb, dictionary size), unpacked size u64 (all ones: not known, diff --git a/src/Filee.Engines/Archives/ZstdReader.cs b/src/Filee.Engines/Archives/ZstdReader.cs new file mode 100644 index 000000000..6d7ce53fa --- /dev/null +++ b/src/Filee.Engines/Archives/ZstdReader.cs @@ -0,0 +1,89 @@ +// Reads Zstandard data (.zst, the outer layer of .tar.zst, zstd RPM payloads) with SharpCompress's decoder (MIT). +// Frames follow one another (what "zstd -c a b > ab.zst" makes), and skippable frames (magic 0x184D2A5?) are passed +// over. Filee checks the first frame's magic itself, so a file that isn't zstd is "not an archive" instead of an empty +// result, and turns the decoder's errors into Filee's messages. A frame whose window is larger than 128 MiB (made with +// "zstd --long=28" or more) is left to 7-Zip, as the zstd tool refuses it without --memory: the decoder would set the +// window aside before a single byte shows the file is real. + +using System.Buffers.Binary; +using SharpCompress.Compressors.ZStandard; +using SharpCompress.Compressors.ZStandard.Unsafe; + +namespace Filee.Engines.Archives; + +/// A read-only stream of the data inside a zstd file. +internal sealed class ZstdReader : Stream +{ + private const uint FrameMagic = 0xFD2FB528; + private const uint SkippableMagic = 0x184D2A50; + + private readonly DecompressionStream _decoder; + private readonly string _name; + + /// Reads zstd frames from the current position of (which stays open). + public ZstdReader(Stream input, string name) + { + _name = name; + Span magic = stackalloc byte[4]; + var start = input.Position; + if (input.ReadAtLeast(magic, magic.Length, throwOnEndOfStream: false) < magic.Length || !IsFrameStart(magic)) + throw ArchiveSafety.NotAnArchive(name); + input.Position = start; + _decoder = new DecompressionStream(new NonClosingStream(input), leaveOpen: false); + } + + /// True for the magic of a zstd frame or a skippable frame. + internal static bool IsFrameStart(ReadOnlySpan magic) + { + var value = BinaryPrimitives.ReadUInt32LittleEndian(magic); + return value == FrameMagic || (value & 0xFFFFFFF0) == SkippableMagic; + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + try + { + return _decoder.Read(buffer); + } + catch (ZstdException ex) when (ex.Code == ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge) + { + throw new NotSupportedException("The zstd window is larger than 128 MiB (made with zstd --long).", ex); + } + catch (ZstdException ex) + { + throw ArchiveSafety.Damaged(_name, "zstd: " + ex.Message, ex); + } + catch (EndOfStreamException ex) + { + throw ArchiveSafety.Damaged(_name, "the zstd data ends early", ex); + } + } + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + _decoder.Dispose(); + base.Dispose(disposing); + } +} diff --git a/tests/Filee.Core.Tests/FormatFieldsTests.cs b/tests/Filee.Core.Tests/FormatFieldsTests.cs index 3cc6e8466..af0b2edc3 100644 --- a/tests/Filee.Core.Tests/FormatFieldsTests.cs +++ b/tests/Filee.Core.Tests/FormatFieldsTests.cs @@ -35,8 +35,8 @@ private static string[] Hidden(IReadOnlySet turnedOff) => "org", "osm", "otf", "pam", "pcd", "pcx", "pdb", "pdn", "pfm", "plist", "plt", "ply", "pml", "ppm", "prc", "ps", "psb", "psd", "psw", "qoi", "ras", "raw", "rb", "rm", "rpm", "rst", "s3m", "scc", "sda", "sdc", "sdw", "sgi", "shp", "snb", "spx", "ssa", "stl", "sub", "swf", "tak", "tar", "tar7z", "tbz2", "tcr", "tex", "textile", "tgz", "tlz", "toml", - "tta", "ttf", "ttml", "txtz", "txz", "typ", "tz", "usm", "vob", "voc", "vpk", "vtf", "w64", "wbmp", "wmf", "woff", "woff2", - "wpd", "wtv", "wv", "xbm", "xcf", "xlt", "xm", "xpm", "xz", "yaml", "z", "zxp" + "tta", "ttf", "ttml", "txtz", "txz", "typ", "tz", "tzst", "usm", "vob", "voc", "vpk", "vtf", "w64", "wbmp", "wmf", "woff", + "woff2", "wpd", "wtv", "wv", "xbm", "xcf", "xlt", "xm", "xpm", "xz", "yaml", "z", "zst", "zxp" ]; public static TheoryData HiddenWithOneFieldOff => new() @@ -51,7 +51,7 @@ private static string[] Hidden(IReadOnlySet turnedOff) => { "developer", [ "aab", "ani", "apks", "bz2", "cab", "cpio", "cur", "deb", "fodg", "fodp", "fods", "fodt", "gz", "har", "icns", "jar", "jsonl", "lz", "lzma", "m4s", "man", "miff", "msix", "nupkg", "pam", "pfm", "plist", "ppm", "qoi", - "rpm", "tar", "tar7z", "tbz2", "tgz", "tlz", "toml", "txz", "xbm", "xpm", "xz", "yaml", "zxp" + "rpm", "tar", "tar7z", "tbz2", "tgz", "tlz", "toml", "txz", "tzst", "xbm", "xpm", "xz", "yaml", "zst", "zxp" ] }, { "writing", ["adoc", "cgm", "dbk", "htmlz", "mediawiki", "org", "rst", "tex", "textile", "txtz", "typ"] }, { "video-pro", [ @@ -78,7 +78,8 @@ private static string[] Hidden(IReadOnlySet turnedOff) => { "developer", [ "aab", "ani", "apks", "bz2", "cab", "caf", "chm", "cpio", "cur", "deb", "fodg", "fodp", "fods", "fodt", "gz", "har", "icns", "ipynb", "jar", "jsonl", "lz", "lzma", "m4s", "man", "miff", "msix", "nupkg", "pam", "pfm", - "plist", "ppm", "qoi", "rpm", "tar", "tar7z", "tbz2", "tgz", "tlz", "toml", "txz", "xbm", "xpm", "xz", "yaml", "zxp" + "plist", "ppm", "qoi", "rpm", "tar", "tar7z", "tbz2", "tgz", "tlz", "toml", "txz", "tzst", "xbm", "xpm", "xz", "yaml", + "zst", "zxp" ] }, { "writing", ["adoc", "cgm", "chm", "dbk", "htmlz", "mediawiki", "org", "rst", "tex", "textile", "txtz", "typ"] }, { "video-pro", [ @@ -145,7 +146,7 @@ public void Grade_counts_match_the_published_table() { [FormatGrade.Everyday] = 19, [FormatGrade.General] = 99, - [FormatGrade.Specialist] = 131, + [FormatGrade.Specialist] = 133, [FormatGrade.Legacy] = 52, }, counts); diff --git a/tests/Filee.Engines.Tests/ArchiveFormatBuilders.cs b/tests/Filee.Engines.Tests/ArchiveFormatBuilders.cs index b51141026..610a33198 100644 --- a/tests/Filee.Engines.Tests/ArchiveFormatBuilders.cs +++ b/tests/Filee.Engines.Tests/ArchiveFormatBuilders.cs @@ -765,6 +765,19 @@ public static byte[] Gzip(byte[] data) return ms.ToArray(); } + /// One zstd frame of (SharpCompress's encoder), with a window of 2^windowLog bytes. + public static byte[] Zstd(byte[] data, int level = 3, int? windowLog = null) + { + using var ms = new MemoryStream(); + using (var zstd = new SharpCompress.Compressors.ZStandard.CompressionStream(ms, level, leaveOpen: true)) + { + if (windowLog is { } log) + zstd.SetParameter(SharpCompress.Compressors.ZStandard.Unsafe.ZSTD_cParameter.ZSTD_c_windowLog, log); + zstd.Write(data); + } + return ms.ToArray(); + } + /// A TAR (PAX) of . public static byte[] Tar(IReadOnlyList entries) { diff --git a/tests/Filee.Engines.Tests/ArchiveFuzzTests.cs b/tests/Filee.Engines.Tests/ArchiveFuzzTests.cs index a6a88a9d8..cb17339ef 100644 --- a/tests/Filee.Engines.Tests/ArchiveFuzzTests.cs +++ b/tests/Filee.Engines.Tests/ArchiveFuzzTests.cs @@ -83,9 +83,10 @@ public void Unix_compress_reader_unpacks_or_refuses_every_mutated_file() => Fuzz("z", [UnixCompress(Words(20_000)), UnixCompress(Noise(5_000, 1), maxBits: 9)]); [Fact] - public void Lzip_alz_and_egg_readers_unpack_or_refuse_every_mutated_file() + public void Lzip_zstd_alz_and_egg_readers_unpack_or_refuse_every_mutated_file() { Fuzz("lz", [Lzip(Words(20_000))]); + Fuzz("zst", [Zstd(Words(20_000)), [.. Zstd(Words(3_000), level: 1), .. Zstd(Noise(2_000, 2))]]); var folder = fx.NewFolder(); var alz = Path.Combine(folder, "seed.alz"); WriteAlz(alz, [new("폴더\\", null), new("폴더\\a.txt", Words(2000), Method: 2), new("b.txt", Words(1000), Method: 1, Packed: ToAlzBzip2(Bzip2(Words(1000))))]); diff --git a/tests/Filee.Engines.Tests/ArchiveInProcessTests.cs b/tests/Filee.Engines.Tests/ArchiveInProcessTests.cs index 11f9abfbd..49073268f 100644 --- a/tests/Filee.Engines.Tests/ArchiveInProcessTests.cs +++ b/tests/Filee.Engines.Tests/ArchiveInProcessTests.cs @@ -150,6 +150,7 @@ public async Task Tar_and_compressed_tars_unpack_in_process() await AssertUnpacksAsync(Write("backup.tar.xz", Xz(tar)), "txz", entries); await AssertUnpacksAsync(Write("backup.tar.Z", UnixCompress(tar)), "tz", entries); await AssertUnpacksAsync(Write("backup.tar.lz", Lzip(tar)), "tlz", entries); + await AssertUnpacksAsync(Write("backup.tar.zst", Zstd(tar)), "tzst", entries); await AssertUnpacksAsync(Write("backup.tar.7z", SevenZip([new("backup.tar", tar)])), "tar7z", entries); } @@ -160,6 +161,7 @@ public async Task Tar_and_compressed_tars_unpack_in_process() [InlineData("lzma")] [InlineData("z")] [InlineData("lz")] + [InlineData("zst")] public async Task Compressed_single_files_unpack_in_process(string format) { var data = Words(1_500_000); @@ -170,12 +172,46 @@ public async Task Compressed_single_files_unpack_in_process(string format) "xz" => Xz(data), "lzma" => [0x5D, 0x00, 0x00, 0x10, 0x00, .. BitConverter.GetBytes((long)data.Length), .. RawLzma(data, endMarker: false)], "z" => UnixCompress(data), + "zst" => Zstd(data), _ => Lzip(data), }; var folder = await UnpackAsync(Write("log.txt." + format, packed), format); Assert.Equal(data, await File.ReadAllBytesAsync(Path.Combine(folder, "log.txt"), Ct)); } + [Fact] + public async Task Zstd_frames_follow_one_another_and_other_data_damage_or_wide_windows_are_refused() + { + // Two frames (as "zstd -c a b" writes them) with skippable frames around them, as seekable-format tools add. + byte[] skippable = [0x5E, 0x2A, 0x4D, 0x18, 3, 0, 0, 0, 1, 2, 3]; + var first = Words(400_000); + var second = Noise(50_000, 7); + var folder = await UnpackAsync(Write("two.txt.zst", [.. skippable, .. Zstd(first), .. Zstd(second, level: 19), .. skippable]), "zst"); + byte[] expected = [.. first, .. second]; + Assert.Equal(expected, await File.ReadAllBytesAsync(Path.Combine(folder, "two.txt"), Ct)); + + Assert.Contains("not a valid archive", await RefusedAsync(Write("fake.zst", "not zstd at all"u8.ToArray()), "zst")); + Assert.Contains("not a valid archive", await RefusedAsync(Write("empty.tar.zst", []), "tzst")); + var zst = Zstd(Words(200_000)); + Assert.Contains("damaged", await RefusedAsync(Write("cut.zst", zst[..(zst.Length / 2)]), "zst")); + Assert.Contains("damaged", await RefusedAsync(Write("trailing.zst", [.. zst, 1, 2, 3, 4, 5, 6, 7, 8]), "zst")); + // A 256 MiB window (zstd --long=28) is left to 7-Zip, as the zstd tool needs --memory for it. + var wide = await Assert.ThrowsAsync( + () => UnpackAsync(Write("wide.tar.zst", Zstd(Words(300_000), windowLog: 28)), "tzst")); + Assert.Equal(ArchiveExtractor.NeedsSevenZip, wide.ErrorKey); + } + + [Theory] + [InlineData("backup.tar.zst", "tzst")] + [InlineData("backup.tzst", "tzst")] + [InlineData("BACKUP.TAR.ZST", "tzst")] + [InlineData("log.txt.zst", "zst")] + public void Zstd_names_are_recognised(string name, string format) + { + Assert.Equal(format, FormatRegistry.Detect(name)?.Id); + Assert.False(FormatRegistry.Get(format).Writable); + } + [Theory] [InlineData(9)] [InlineData(12)] @@ -222,11 +258,11 @@ public async Task Cpio_hard_link_groups_become_copies() } [Fact] - public async Task Rpm_payloads_in_gzip_and_xz_unpack_to_their_files() + public async Task Rpm_payloads_in_gzip_xz_and_zstd_unpack_to_their_files() { var entries = Entries(); var cpio = Cpio(entries); - foreach (var payload in new[] { Gzip(cpio), Xz(cpio), cpio }) + foreach (var payload in new[] { Gzip(cpio), Xz(cpio), Zstd(cpio), cpio }) await AssertUnpacksAsync(Write("filee-test.rpm", Rpm(payload)), "rpm", entries); } diff --git a/tests/Filee.Engines.Tests/ArchiveTests.cs b/tests/Filee.Engines.Tests/ArchiveTests.cs index 65ed45033..003f66911 100644 --- a/tests/Filee.Engines.Tests/ArchiveTests.cs +++ b/tests/Filee.Engines.Tests/ArchiveTests.cs @@ -57,6 +57,8 @@ private async Task ConvertOkAsync(string source, Preset preset) [InlineData("egg")] [InlineData("lz")] [InlineData("tlz")] + [InlineData("tzst")] + [InlineData("zst")] [InlineData("split")] public void Every_readable_archive_converts_in_one_step(string from) { @@ -75,6 +77,8 @@ public void Single_file_compressors_convert_among_each_other_but_archives_do_not { var planner = fx.Catalog.CreatePlanner(["archive"]); Assert.Single(planner.Plan("lz", "gz")!.Steps); + Assert.Single(planner.Plan("zst", "xz")!.Steps); + Assert.Null(planner.Plan("gz", "zst")); // zstd is read, not written Assert.Single(planner.Plan("xz", "gz")!.Steps); Assert.Single(planner.Plan("gz", "xz")!.Steps); // Filee's own XZ writer Assert.Single(planner.Plan("lzma", "bz2")!.Steps); // and its own bzip2 writer From 737015b0cd3a886ee69342f1e417c18083a300dc Mon Sep 17 00:00:00 2001 From: KnifeLemon Date: Thu, 19 Nov 2026 15:38:22 +0900 Subject: [PATCH 2/5] Faster LZMA decoding, fast-level search and bzip2 move-to-front - LZMA decoder: a match's source is asked into the cache (prefetch hint, x86) when the match is decoded, so it arrives while the next symbols are decoded and before the deferred copy; the three older repeat distances live in fields, which leaves registers to the coder. Same bytes. - Hash-chain search (Fast): a search of its own with its values in registers and one two-byte check per candidate (same matches), a 10-bit hash for the two-byte table (4 KB instead of 256 KB per thread), and the next position's table slots prefetched. Output stays deterministic for any number of threads; Normal and Maximum are unchanged. - bzip2: the move-to-front shift moves 16 bytes at a time (decoder and encoder; the decoder's symbol thread sets the pace), and the transform writes single rows without a span fill. Same bytes. Co-Authored-By: Claude Opus 5.5 --- .../Archives/Bzip2BlockEncoder.cs | 2 +- src/Filee.Engines/Archives/Bzip2Decoder.cs | 32 +++- src/Filee.Engines/Archives/Bzip2Sort.cs | 6 +- .../Archives/Lzma/LzmaDecoder.cs | 46 ++++-- .../Archives/Lzma/MatchFinder.cs | 154 ++++++++++++++---- src/Filee.Engines/Filee.Engines.csproj | 2 + tests/Filee.Engines.Tests/Bzip2WriterTests.cs | 17 ++ 7 files changed, 203 insertions(+), 56 deletions(-) diff --git a/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs b/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs index 443d6516d..55a52488f 100644 --- a/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs +++ b/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs @@ -192,7 +192,7 @@ private int MoveToFront(ReadOnlySpan bwt, ReadOnlySpan number, int i run = 0; } var j = list[1..inUse].IndexOf(value) + 1; - list[..j].CopyTo(list[1..]); + Bzip2Decoder.MoveUp(list, j); list[0] = value; symbols[count++] = (ushort)(j + 1); frequencies[j + 1]++; diff --git a/src/Filee.Engines/Archives/Bzip2Decoder.cs b/src/Filee.Engines/Archives/Bzip2Decoder.cs index fb973b8f4..0f501a14c 100644 --- a/src/Filee.Engines/Archives/Bzip2Decoder.cs +++ b/src/Filee.Engines/Archives/Bzip2Decoder.cs @@ -15,7 +15,10 @@ using System.Buffers; using System.Collections.Concurrent; +using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace Filee.Engines.Archives; @@ -299,15 +302,7 @@ private Block ReadBlock(int[] tt, int blockMax, uint? expectedCrc) if (index >= inUseCount) throw new InvalidDataException("invalid bzip2 symbol"); var front = mtf[index]; - if (index < 16) - { - for (var k = index; k > 0; k--) - mtf[k] = mtf[k - 1]; - } - else - { - mtf[..index].CopyTo(mtf[1..]); - } + MoveUp(mtf, index); mtf[0] = front; var b = seqToUnseq[front]; counts[b]++; @@ -340,6 +335,25 @@ int NextSymbol() } } + /// + /// Moves the first bytes of the 256-byte move-to-front list one place up (count below + /// 256): 16 bytes at a time from the top while there are that many, each load taken before the store above it, + /// then byte by byte. One thread decodes the symbols of every block (the inverse transforms run on others), so + /// this step sets the pace; Bzip2BlockEncoder uses it too. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void MoveUp(Span mtf, int count) + { + if ((uint)count >= (uint)mtf.Length || mtf.Length < 256) + throw new ArgumentOutOfRangeException(nameof(count)); + ref var list = ref MemoryMarshal.GetReference(mtf); + var i = count; + for (; i >= 16; i -= 16) + Vector128.LoadUnsafe(ref list, (nuint)(i - 16)).StoreUnsafe(ref list, (nuint)(i - 15)); + for (; i > 0; i--) + Unsafe.Add(ref list, i) = Unsafe.Add(ref list, i - 1); + } + /// /// Canonical Huffman decode tables as in the reference bzip2 (limit / base / perm per code length), plus a table /// of the codes of up to bits found the same way. diff --git a/src/Filee.Engines/Archives/Bzip2Sort.cs b/src/Filee.Engines/Archives/Bzip2Sort.cs index d62eb7333..796d5f9fb 100644 --- a/src/Filee.Engines/Archives/Bzip2Sort.cs +++ b/src/Filee.Engines/Archives/Bzip2Sort.cs @@ -44,7 +44,11 @@ public static int Transform(ReadOnlySpan block, Span output, Cancell var at = sa[i]; if (at == start) origin = row; - output.Slice(row, repeats).Fill(word[at == 0 ? root - 1 : at - 1]); + var last = word[at == 0 ? root - 1 : at - 1]; + if (repeats == 1) + output[row] = last; // nearly every block: no repeating root + else + output.Slice(row, repeats).Fill(last); row += repeats; } return origin; diff --git a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs index 7066a4e0f..b993a0375 100644 --- a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs +++ b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs @@ -9,6 +9,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace Filee.Engines.Archives.Lzma; @@ -511,7 +512,9 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput var pbMask = _pbMask; ref var probs = ref MemoryMarshal.GetArrayDataReference(_probabilities); var state = _state; - uint rep0 = _rep0, rep1 = _rep1, rep2 = _rep2, rep3 = _rep3; + // The last distance stays in a register; the three before it, used only by repeats, stay in the fields, so the + // coder and the loop's other values have the registers. + var rep0 = _rep0; ref var input = ref MemoryMarshal.GetArrayDataReference(rc.Buffer); var inEnd = rc.End; rc.Take(out var range, out var code, out var inPos); @@ -539,30 +542,30 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput uint distance; if (Bit(ref Unsafe.Add(ref probs, IsRepG1At + state), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) { - distance = rep1; + distance = _rep1; } else { if (Bit(ref Unsafe.Add(ref probs, IsRepG2At + state), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) { - distance = rep2; + distance = _rep2; } else { - distance = rep3; - rep3 = rep2; + distance = _rep3; + _rep3 = _rep2; } - rep2 = rep1; + _rep2 = _rep1; } - rep1 = rep0; + _rep1 = rep0; rep0 = distance; } } else { - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; + _rep3 = _rep2; + _rep2 = _rep1; + _rep1 = rep0; lengthModel = MatchLengthAt; } @@ -623,7 +626,9 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput // The bytes are copied later, several matches at a time: the copies' sources are far apart in the window // and mostly not in the cache, and done back to back their loads overlap. Nothing before the copies reads - // the window (only literals do), so the bytes come out the same. + // the window (only literals do), so the bytes come out the same. The source is asked into the cache now, + // so it arrives while the next symbols are decoded. + Prefetch(window, pos - (int)rep0 - 1, copyLength); Unsafe.Add(ref deferred, copies * 3) = pos; Unsafe.Add(ref deferred, copies * 3 + 1) = (int)rep0 + 1; Unsafe.Add(ref deferred, copies * 3 + 2) = copyLength; @@ -650,13 +655,26 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput _pos = pos; _state = state; _rep0 = rep0; - _rep1 = rep1; - _rep2 = rep2; - _rep3 = rep3; rc.Put(range, code, inPos); return next; } + /// + /// A hint to load the source of a copy into the cache (x86; elsewhere nothing). may be + /// below zero (the source wrapped to the window's end); only the address is used, nothing is read through it. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe void Prefetch(byte[] window, int source, int count) + { + if (!Sse.IsSupported) + return; + if (source < 0) + source += window.Length; + ref var start = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(window), source); + Sse.Prefetch0(Unsafe.AsPointer(ref start)); + Sse.Prefetch0(Unsafe.AsPointer(ref Unsafe.Add(ref start, count - 1))); + } + /// Does the first copies MatchRun put off, in order. private void CopyDeferred(byte[] window, int count) { diff --git a/src/Filee.Engines/Archives/Lzma/MatchFinder.cs b/src/Filee.Engines/Archives/Lzma/MatchFinder.cs index f3d5eb3d5..fbb2ad515 100644 --- a/src/Filee.Engines/Archives/Lzma/MatchFinder.cs +++ b/src/Filee.Engines/Archives/Lzma/MatchFinder.cs @@ -6,7 +6,8 @@ // rebuilds the tree on the way with the cursor as the new root (everything smaller to its left, larger to its // right). Bytes already known to match on both sides are not compared again, so the longest matches are found in // far fewer steps than a chain needs; inserting a position costs the same walk. -// Two chainless tables keyed by the next two and three bytes catch close short matches either way. +// Two chainless tables keyed by the next two and three bytes catch close short matches either way (the two-byte one +// a small hash of them with the chain, so the fast level's tables stay in the cache). // // The window is one array: the dictionary behind the cursor plus room ahead for new input, slid back when the room // is used up. Positions are stored as buffer index + an offset that grows with each slide, so the tables never need @@ -27,6 +28,7 @@ using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; namespace Filee.Engines.Archives.Lzma; @@ -48,7 +50,10 @@ internal sealed class MatchFinder private readonly int[] _links; // chain: one previous position per slot; tree: left and right child per slot private readonly int[] _head4; private readonly int[] _head3 = new int[1 << Hash3Bits]; - private readonly int[] _head2 = new int[1 << 16]; + // Two-byte table: the bytes themselves with the tree (every pair its slot); a 10-bit hash of them with the chain, + // whose small dictionary and many threads want tables that stay in the cache. + private const int ChainHash2Bits = 10; + private readonly int[] _head2; private readonly int _hash4Shift; private readonly int _depth; private readonly int _nice; @@ -73,6 +78,7 @@ public MatchFinder(int dictionary, int ahead, int depth, int nice, bool tree) _mask = dictionary - 1; _tree = tree; _links = new int[tree ? 2L * dictionary : dictionary]; + _head2 = new int[tree ? 1 << 16 : 1 << ChainHash2Bits]; var hashBits = Math.Clamp(BitOperations.Log2((uint)dictionary) - 1, 16, 24); _head4 = new int[1 << hashBits]; _hash4Shift = 32 - hashBits; @@ -85,7 +91,7 @@ public MatchFinder(int dictionary, int ahead, int depth, int nice, bool tree) public static long TableBytes(int dictionary, bool tree) { var hashBits = Math.Clamp(BitOperations.Log2((uint)dictionary) - 1, 16, 24); - return 4L * ((tree ? 2L : 1L) * dictionary + (1 << hashBits) + (1 << Hash3Bits) + (1 << 16)); + return 4L * ((tree ? 2L : 1L) * dictionary + (1 << hashBits) + (1 << Hash3Bits) + (tree ? 1 << 16 : 1 << ChainHash2Bits)); } /// Forgets the input and every position entered, keeping the buffer and the tables' memory. @@ -172,7 +178,9 @@ static void Rebase(int[] table, int delta) /// bytes, each longer than the one before; returns how many. /// public int FindAll(int pos, int maxLength, int[] lengths, uint[] distances) => - _ahead is { } ahead ? ahead.Take(pos, maxLength, lengths, distances) : Search(pos, maxLength, lengths, distances, out _); + _ahead is { } ahead ? ahead.Take(pos, maxLength, lengths, distances) + : _tree ? Search(pos, maxLength, lengths, distances, out _) + : ChainSearch(pos, maxLength, lengths, distances); /// /// Starts searching every position from to on @@ -266,8 +274,10 @@ private void NearSearch(int pos, int maxLength, ref Found found) } } + /// A search of the binary tree (the hash chain has ). private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, out uint distance) { + System.Diagnostics.Debug.Assert(_tree, "the chain searches with ChainSearch"); distance = 0; System.Diagnostics.Debug.Assert(pos >= _nextInsert, "a position is searched twice"); SkipTo(pos); @@ -306,33 +316,15 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou } } - if (_tree) - { - TreeWalk(pos, stored, candidate, ref found); - // The tree compares at most the nice length; a match that long may go on. - if (found.Best >= _nice && found.Best < maxLength) - { - var at = pos - (int)found.Distance - 1; - var longer = Common(buffer, pos, at, maxLength); - found.Best = longer; - if (lengths is not null) - lengths[found.Count - 1] = longer; - } - } - else + TreeWalk(pos, stored, candidate, ref found); + // The tree compares at most the nice length; a match that long may go on. + if (found.Best >= _nice && found.Best < maxLength) { - _links[stored & _mask] = candidate; - for (var depth = _depth; candidate > 0 && depth > 0 && found.Best < maxLength && found.Best < _nice; depth--) - { - var gap = stored - candidate; - if (gap <= 0 || gap >= _dictionary) - break; - var at = candidate - _offset; - candidate = _links[candidate & _mask]; - if (buffer[at + found.Best] != buffer[pos + found.Best] || buffer[at] != buffer[pos]) - continue; - found.Offer(Common(buffer, pos, at, maxLength), gap); - } + var at = pos - (int)found.Distance - 1; + var longer = Common(buffer, pos, at, maxLength); + found.Best = longer; + if (lengths is not null) + lengths[found.Count - 1] = longer; } distance = found.Distance; @@ -341,6 +333,104 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou return lengths is null ? found.Best : found.Count; } + /// + /// for the hash chain (the fast level), on its own so its loop keeps its values in registers: + /// the same tables, candidates and lists. A candidate is only measured when its two bytes ending at the best length + /// so far agree with the cursor's (a match longer than the best has them), which takes one load per candidate. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private int ChainSearch(int pos, int maxLength, int[] lengths, uint[] distances) + { + System.Diagnostics.Debug.Assert(pos >= _nextInsert, "a position is searched twice"); + SkipTo(pos); + _nextInsert = pos + 1; + var buffer = Buffer; + if (pos + 4 > Filled || (uint)pos > (uint)(buffer.Length - Math.Max(4, maxLength))) + return 0; + + ref var data = ref MemoryMarshal.GetArrayDataReference(buffer); + ref var cursor = ref Unsafe.Add(ref data, pos); + var value = Unsafe.ReadUnaligned(ref cursor); + if (!BitConverter.IsLittleEndian) + value = BinaryPrimitives.ReverseEndianness(value); + var offset = _offset; + var dictionary = _dictionary; + var stored = pos + offset; + // The hashes are below the tables' sizes by construction: 16 bits, Hash3Bits and the shift for _head4. + ref var slot2 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), ChainHash2(value)); + ref var slot3 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits))); + ref var slot4 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), (int)((value * 2654435761u) >> _hash4Shift)); + var candidate2 = slot2; + slot2 = stored; + var candidate3 = slot3; + slot3 = stored; + var candidate = slot4; + slot4 = stored; + ref var links = ref MemoryMarshal.GetArrayDataReference(_links); + var mask = _mask; + Unsafe.Add(ref links, stored & mask) = candidate; + // The next position is searched or entered right after this one: its table slots (mostly not in the cache) + // are asked for now, so they arrive while this chain is walked. + if (pos + 5 <= Filled) + PrefetchSlots(Unsafe.ReadUnaligned(ref Unsafe.Add(ref cursor, 1))); + + // Candidates lie within the dictionary behind pos, so they and maxLength bytes after them fit the buffer. + var best = 1; + var count = 0; + if (maxLength >= 2) + { + if (candidate2 > 0 && stored - candidate2 < dictionary && Unsafe.Add(ref data, candidate2 - offset) == cursor) + Offer(Common(buffer, pos, candidate2 - offset, maxLength), stored - candidate2, ref best, ref count, lengths, distances); + if (candidate3 > 0 && stored - candidate3 < dictionary && best < maxLength && Unsafe.Add(ref data, candidate3 - offset) == cursor) + Offer(Common(buffer, pos, candidate3 - offset, maxLength), stored - candidate3, ref best, ref count, lengths, distances); + } + var stopAt = Math.Min(maxLength, _nice); + for (var depth = _depth; candidate > 0 && depth > 0 && best < stopAt; depth--) + { + var gap = stored - candidate; + if (gap <= 0 || gap >= dictionary) + break; + var at = candidate - offset; + candidate = Unsafe.Add(ref links, candidate & mask); + if (Unsafe.ReadUnaligned(ref Unsafe.Add(ref data, at + best - 1)) != Unsafe.ReadUnaligned(ref Unsafe.Add(ref cursor, best - 1))) + continue; + Offer(Common(buffer, pos, at, maxLength), gap, ref best, ref count, lengths, distances); + } + return best < 2 ? 0 : count; + } + + /// + /// A hint to load the three- and four-byte table slots of the position whose first bytes are + /// (little-endian) into the cache (x86; elsewhere nothing). Nothing is read through the + /// addresses. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe void PrefetchSlots(uint value) + { + if (!Sse.IsSupported) + return; + if (!BitConverter.IsLittleEndian) + value = BinaryPrimitives.ReverseEndianness(value); + Sse.Prefetch0(Unsafe.AsPointer(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits))))); + Sse.Prefetch0(Unsafe.AsPointer(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), (int)((value * 2654435761u) >> _hash4Shift)))); + } + + /// The chain's two-byte slot for the bytes in the low half of . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ChainHash2(uint value) => (int)(((value & 0xFFFF) * 2654435761u) >> (32 - ChainHash2Bits)); + + /// on locals. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Offer(int length, int gap, ref int best, ref int count, int[] lengths, uint[] distances) + { + if (length <= best) + return; + best = length; + lengths[count] = length; + distances[count] = (uint)(gap - 1); + count++; + } + /// /// Walks the tree of (the old root) for the position , offering /// what matches, and makes the root. @@ -448,11 +538,13 @@ private void Insert(int pos) var found = new Found(0, null, null); for (var p = _nextInsert; p < end; p++) { + if (!_tree && p + 8 < end) + PrefetchSlots(BinaryPrimitives.ReadUInt32LittleEndian(buffer.AsSpan(p + 4))); var value = BinaryPrimitives.ReadUInt32LittleEndian(buffer.AsSpan(p)); var h4 = (int)((value * 2654435761u) >> _hash4Shift); var h3 = (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits)); var stored = p + _offset; - Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), (int)(value & 0xFFFF)) = stored; + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), _tree ? (int)(value & 0xFFFF) : ChainHash2(value)) = stored; Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), h3) = stored; ref var slot4 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), h4); var candidate = slot4; diff --git a/src/Filee.Engines/Filee.Engines.csproj b/src/Filee.Engines/Filee.Engines.csproj index 867ac31fb..00856d175 100644 --- a/src/Filee.Engines/Filee.Engines.csproj +++ b/src/Filee.Engines/Filee.Engines.csproj @@ -5,6 +5,8 @@ Filee.Engines + + true diff --git a/tests/Filee.Engines.Tests/Bzip2WriterTests.cs b/tests/Filee.Engines.Tests/Bzip2WriterTests.cs index ede9905ed..f0841393c 100644 --- a/tests/Filee.Engines.Tests/Bzip2WriterTests.cs +++ b/tests/Filee.Engines.Tests/Bzip2WriterTests.cs @@ -169,6 +169,23 @@ public async Task Output_unpacks_byte_for_byte_with_three_decoders(Kind kind, in Assert.True(bz2.Length < data.Length / 2, $"{bz2.Length} bytes for {data.Length}"); } + /// The move-to-front shift (16 bytes at a time, then single bytes) against a plain byte loop, for every count. + [Fact] + public void The_move_to_front_shift_matches_a_byte_loop() + { + var random = new Random(9); + for (var count = 0; count < 256; count++) + { + var list = new byte[256]; + random.NextBytes(list); + var expected = (byte[])list.Clone(); + for (var k = count; k > 0; k--) + expected[k] = expected[k - 1]; + Bzip2Decoder.MoveUp(list, count); + Assert.Equal(expected, list); + } + } + /// /// The transform against sorting every rotation by comparing it whole, for short blocks over small alphabets, where /// periodic blocks, repeats and ties are common: the same last column, and the origin row is the unrotated block. From b2be755223fc7a944e30265b286a889de8478816 Mon Sep 17 00:00:00 2001 From: KnifeLemon Date: Thu, 19 Nov 2026 16:07:49 +0900 Subject: [PATCH 3/5] BCJ2: copy the main stream's bytes up to each possible jump in one go The converter took the main stream byte by byte through a buffered reader; it now scans its own buffer for the next E8 / E9 / 0F 8x opcode and copies the run before it at once. Same bytes, same checks. Co-Authored-By: Claude Opus 5.5 --- src/Filee.Engines/Archives/SevenZipCoders.cs | 47 +++++++++++++++----- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/src/Filee.Engines/Archives/SevenZipCoders.cs b/src/Filee.Engines/Archives/SevenZipCoders.cs index c8fc638b3..3af63398a 100644 --- a/src/Filee.Engines/Archives/SevenZipCoders.cs +++ b/src/Filee.Engines/Archives/SevenZipCoders.cs @@ -404,7 +404,10 @@ private int Convert(Span data) /// internal sealed class Bcj2Stream : ReadOnlyDecoderStream { - private readonly ByteInput _main; + private readonly Stream _main; + private readonly byte[] _mainBuffer = new byte[1 << 16]; + private int _mainPosition; + private int _mainEnd; private readonly ByteInput _call; private readonly ByteInput _jump; private readonly RangeDecoder _rc = new(); @@ -418,7 +421,7 @@ internal sealed class Bcj2Stream : ReadOnlyDecoderStream public Bcj2Stream(Stream main, Stream call, Stream jump, Stream rc, long size) { - _main = new ByteInput(main); + _main = main; _call = new ByteInput(call); _jump = new ByteInput(jump); _size = size; @@ -437,17 +440,41 @@ public override int Read(Span buffer) _done++; continue; } - var b = _main.ReadByte(); - if (b < 0) - throw new InvalidDataException("the BCJ2 main stream ends too early"); - buffer[n++] = (byte)b; - _done++; - if (!IsJump(_previous, (byte)b) || _done == _size) + if (_mainPosition == _mainEnd) { - _previous = (byte)b; + _mainPosition = 0; + _mainEnd = Math.Max(0, _main.Read(_mainBuffer, 0, _mainBuffer.Length)); + if (_mainEnd == 0) + throw new InvalidDataException("the BCJ2 main stream ends too early"); + } + // The main stream's bytes up to and with the next opcode that may have a target taken out, in one copy. + var run = _mainBuffer.AsSpan(_mainPosition, (int)Math.Min(Math.Min(_mainEnd - _mainPosition, buffer.Length - n), _size - _done)); + var previous = _previous; + var length = 0; + var jump = false; + while (length < run.Length) + { + var c = run[length++]; + if (IsJump(previous, c)) + { + jump = true; + previous = c; + break; + } + previous = c; + } + run[..length].CopyTo(buffer[n..]); + n += length; + _done += length; + _mainPosition += length; + var b = run[length - 1]; + var before = length >= 2 ? run[length - 2] : _previous; + if (!jump || _done == _size) + { + _previous = previous; continue; } - var index = b == 0xE8 ? _previous : b == 0xE9 ? 256 : 257; + var index = b == 0xE8 ? before : b == 0xE9 ? 256 : 257; if (_rc.DecodeBit(ref _probabilities[index]) == 0) { _previous = (byte)b; From 76a45e0868de04d76d6ecc8ed9d84bdce9b4b9e4 Mon Sep 17 00:00:00 2001 From: KnifeLemon Date: Thu, 19 Nov 2026 17:20:20 +0900 Subject: [PATCH 4/5] LZMA decoder: keep the prefetch hint's addresses inside the window A source that wraps around the window's end is hinted only up to it, so no reference past the array is ever formed. Co-Authored-By: Claude Opus 5.5 --- src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs index b993a0375..a8bfdc33f 100644 --- a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs +++ b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs @@ -670,9 +670,10 @@ private static unsafe void Prefetch(byte[] window, int source, int count) return; if (source < 0) source += window.Length; - ref var start = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(window), source); - Sse.Prefetch0(Unsafe.AsPointer(ref start)); - Sse.Prefetch0(Unsafe.AsPointer(ref Unsafe.Add(ref start, count - 1))); + // Both addresses stay inside the window (a source that wraps around its end is only hinted up to there). + ref var data = ref MemoryMarshal.GetArrayDataReference(window); + Sse.Prefetch0(Unsafe.AsPointer(ref Unsafe.Add(ref data, source))); + Sse.Prefetch0(Unsafe.AsPointer(ref Unsafe.Add(ref data, Math.Min(source + count - 1, window.Length - 1)))); } /// Does the first copies MatchRun put off, in order. From 65987f2cf652d41e620825e0f16121475954f32a Mon Sep 17 00:00:00 2001 From: KnifeLemon Date: Thu, 19 Nov 2026 18:49:11 +0900 Subject: [PATCH 5/5] settings: v104 teaches existing archive donuts Zstandard files (#1646) Co-Authored-By: Claude Opus 5.5 --- src/Filee.Core/Settings/AppSettings.cs | 2 +- src/Filee.Core/Settings/UserDataStore.cs | 3 ++ tests/Filee.Core.Tests/ZstSettingsTests.cs | 34 ++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/Filee.Core.Tests/ZstSettingsTests.cs diff --git a/src/Filee.Core/Settings/AppSettings.cs b/src/Filee.Core/Settings/AppSettings.cs index b591d04c8..9255f0040 100644 --- a/src/Filee.Core/Settings/AppSettings.cs +++ b/src/Filee.Core/Settings/AppSettings.cs @@ -170,7 +170,7 @@ public static bool IsFlipKeyAllowed(string key, IEnumerable gest public sealed class AppSettings { /// Current schema version. Increase when the format changes and add a migration. - public const int CurrentSchemaVersion = 103; + public const int CurrentSchemaVersion = 104; public int SchemaVersion { get; set; } = CurrentSchemaVersion; diff --git a/src/Filee.Core/Settings/UserDataStore.cs b/src/Filee.Core/Settings/UserDataStore.cs index 1a16a004e..52003ffab 100644 --- a/src/Filee.Core/Settings/UserDataStore.cs +++ b/src/Filee.Core/Settings/UserDataStore.cs @@ -1221,6 +1221,9 @@ public static bool ApplyToLibrary(int from, List presets, List + Assert.Equal("archives", ProfileSelector.Select(BuiltInData.CreateProfiles(), [name])?.Id); + + [Fact] + public void Version_104_archives_learn_zstandard_without_taking_custom_claims() + { + var presets = BuiltInData.CreatePresets(); + var profiles = BuiltInData.CreateProfiles(); + var archives = profiles.Single(p => p.Id == "archives"); + archives.Extensions.RemoveAll(e => Extensions.Contains(e)); + profiles.Insert(0, new ToolbarProfile { Id = "mine", Name = "Mine", Extensions = ["ZST"] }); + + Assert.True(SettingsMigrations.ApplyToLibrary(103, presets, profiles)); + Assert.DoesNotContain("zst", archives.Extensions, StringComparer.OrdinalIgnoreCase); + Assert.Contains("tar.zst", archives.Extensions); + Assert.Contains("tzst", archives.Extensions); + Assert.Equal("mine", ProfileSelector.Select(profiles, ["logs.zst"])?.Id); + Assert.False(SettingsMigrations.ApplyToLibrary(103, presets, profiles)); + } +}