From 1a2be857df8994f1c279619c7159ca6626eb613e Mon Sep 17 00:00:00 2001 From: KnifeLemon Date: Thu, 19 Nov 2026 10:11:22 +0900 Subject: [PATCH 1/2] Decode archives ahead on worker threads; faster LZMA match and tree decoding - ReadAheadStream: 7z coders that feed another coder, the folder output and single compressed files decode on a thread of their own while the next stage works - 7z packed streams are read under the archive stream's lock (coders on threads) - LZMA2 segments and xz blocks decoded on workers stream out as they decode - LZMA decoder: match copies deferred and done back to back, 16-byte chunk copies with a guard past the dictionary, tree bits read both children ahead, shorter bit masks Co-Authored-By: Claude Opus 5.5 --- src/Filee.Engines/Archives/CompressedFile.cs | 3 +- .../Archives/Lzma/Lzma2Parallel.cs | 135 +++++++++--- .../Archives/Lzma/LzmaDecoder.cs | 165 +++++++++++---- src/Filee.Engines/Archives/Lzma/XzReader.cs | 61 ++++-- src/Filee.Engines/Archives/ReadAheadStream.cs | 197 ++++++++++++++++++ src/Filee.Engines/Archives/SevenZipCoders.cs | 13 +- src/Filee.Engines/Archives/SevenZipReader.cs | 51 ++++- 7 files changed, 526 insertions(+), 99 deletions(-) create mode 100644 src/Filee.Engines/Archives/ReadAheadStream.cs diff --git a/src/Filee.Engines/Archives/CompressedFile.cs b/src/Filee.Engines/Archives/CompressedFile.cs index 95319fe87..e11345e2f 100644 --- a/src/Filee.Engines/Archives/CompressedFile.cs +++ b/src/Filee.Engines/Archives/CompressedFile.cs @@ -40,7 +40,8 @@ public static void Decompress(string archive, string format, string outputFile, try { long declared = -1; - using (var decoder = Open(format, input, name, ref declared)) + // Decoded on a thread of its own while this one writes the file. + using (var decoder = ReadAheadStream.Wrap(Open(format, input, name, ref declared), declared, cancellationToken)) using (var output = new FileStream(outputFile, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16)) { // Past the room on the drive is "too large"; past a header's size (within that room) is damage. diff --git a/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs b/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs index 63c1477f8..51bc4ce7f 100644 --- a/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs +++ b/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs @@ -12,7 +12,9 @@ // What a segment decodes to is held in memory until it is read, so the threads and the size of a segment are // bounded: at most one segment per processor, all of them in an eighth of the memory, none past 512 MiB. A segment // that fails on its thread is decoded again, from its start, on the reader's thread as it is read: so damaged data -// gives the very bytes and the very error, at the same place, as the one-thread decoder. +// gives the very bytes and the very error, at the same place, as the one-thread decoder. Workers hand out what they +// decoded as they go (Decoded.Publish), so the segment being read streams out while the next ones decode; after a +// failure the reader's own decoder skips what was handed out already. using System.Runtime.ExceptionServices; @@ -158,21 +160,70 @@ public static int Workers(long unpacked, long packed, uint dictionary) return segments; } - /// What a worker decoded: the output, how much of it is good, and the error that stopped it, if any. + /// + /// What a worker decodes: the output, how much of it is good, and the error that stopped it, if any. The worker + /// publishes the output as it goes, so the reader hands out the start of a segment while the rest is decoded. + /// internal sealed class Decoded(byte[] output) { + /// The worker publishes after every this many bytes. + public const int Step = 1 << 20; + + private readonly object _gate = new(); + private int _ready; + private bool _done; + public byte[] Output { get; } = output; + + /// Bytes the worker decoded so far (its own count; readers go by ). public int Count { get; set; } + public ExceptionDispatchInfo? Error { get; set; } + + /// Called by the worker: the first bytes of the output are final. + public void Publish() + { + lock (_gate) + { + _ready = Count; + Monitor.PulseAll(_gate); + } + } + + /// Called by the worker when it stops, with set if it failed. + public void Finish() + { + lock (_gate) + { + if (Error is null) + _ready = Count; + _done = true; + Monitor.PulseAll(_gate); + } + } + + /// + /// Waits until more than bytes are published or the worker stopped; returns the + /// bytes published (all that will be when ). + /// + public int WaitBeyond(int position, out bool done) + { + lock (_gate) + { + while (_ready <= position && !_done) + Monitor.Wait(_gate); + done = _done; + return _ready; + } + } } /// - /// Decodes one segment: bytes of chunks followed by the end byte (the caller puts it at - /// packed[length]), expected to give bytes. + /// Decodes one segment into : bytes of chunks followed by the + /// end byte (the caller puts it at packed[length]), expected to give bytes. /// - public static Decoded DecodeSegment(byte[] packed, int length, uint dictionary, int unpacked, CancellationToken cancellationToken) + public static void DecodeSegment(Decoded result, byte[] packed, int length, uint dictionary, int unpacked, CancellationToken cancellationToken) { - var result = new Decoded(GC.AllocateUninitializedArray(unpacked)); try { var reader = new Lzma2Reader(new ByteInput(new MemoryStream(packed, 0, length + 1, writable: false)), dictionary, unpacked, @@ -180,17 +231,40 @@ public static Decoded DecodeSegment(byte[] packed, int length, uint dictionary, var output = result.Output; while (result.Count < unpacked) { - var read = reader.Read(output.AsSpan(result.Count)); + var read = reader.Read(output.AsSpan(result.Count, Math.Min(Decoded.Step, unpacked - result.Count))); if (read == 0) throw new InvalidDataException("the LZMA2 chunk is damaged"); result.Count += read; + result.Publish(); } } catch (Exception ex) { result.Error = ExceptionDispatchInfo.Capture(ex); } - return result; + finally + { + result.Finish(); + } + } + + /// + /// Reads and drops bytes of : data decoded again after a worker + /// failed, up to where the reader had got. gets them on the way. + /// + public static void Skip(Lzma2Reader reader, long count, Action>? seen = null) + { + if (count <= 0) + return; + var scratch = new byte[Math.Min(count, 1 << 16)]; + while (count > 0) + { + var read = reader.Read(scratch.AsSpan(0, (int)Math.Min(count, scratch.Length))); + if (read == 0) + return; + seen?.Invoke(scratch.AsSpan(0, read)); + count -= read; + } } /// The segments of a 7z LZMA2 coder, decoded ahead on worker threads and read in order. @@ -201,10 +275,11 @@ private sealed class SegmentStream : Stream private readonly uint _dictionary; private readonly int _workers; private readonly CancellationTokenSource _cancel; - private readonly Queue<(Segment Segment, Task Work)> _ahead = new(); + private readonly Queue<(Segment Segment, Decoded Work)> _ahead = new(); private int _next; - private Decoded? _current; + private (Segment Segment, Decoded Work)? _current; private int _position; + private int _ready; private Lzma2Reader? _replay; // a failed segment, decoded again here public SegmentStream(PackedWindow packed, List segments, uint dictionary, int workers, CancellationToken cancellationToken) @@ -234,29 +309,34 @@ public override int Read(Span buffer) } if (_current is { } current) { - if (_position < current.Count) + if (_position < _ready) { - var count = Math.Min(buffer.Length, current.Count - _position); - current.Output.AsSpan(_position, count).CopyTo(buffer); + var count = Math.Min(buffer.Length, _ready - _position); + current.Work.Output.AsSpan(_position, count).CopyTo(buffer); _position += count; return count; } + _ready = current.Work.WaitBeyond(_position, out var done); + if (_position < _ready || !done) + continue; _current = null; + if (current.Work.Error is { } error) + { + // Decoded again here, as the one-thread reader reads it, from where the worker's output ran out. + if (error.SourceException is OperationCanceledException) + error.Throw(); + _replay = Open(current.Segment, _cancel.Token); + Skip(_replay, _position); + } + continue; } _cancel.Token.ThrowIfCancellationRequested(); Start(); if (!_ahead.TryDequeue(out var next)) return 0; - var decoded = next.Work.GetAwaiter().GetResult(); - if (decoded.Error is { } error) - { - if (error.SourceException is OperationCanceledException) - error.Throw(); - _replay = Open(next.Segment, _cancel.Token); - continue; - } - _current = decoded; + _current = next; _position = 0; + _ready = 0; } } @@ -269,7 +349,9 @@ private void Start() var packed = ReadPacked(segment); var dictionary = _dictionary; var token = _cancel.Token; - _ahead.Enqueue((segment, Task.Run(() => DecodeSegment(packed, segment.Length, dictionary, segment.Unpacked, token), CancellationToken.None))); + var decoded = new Decoded(GC.AllocateUninitializedArray(segment.Unpacked)); + _ = Task.Run(() => DecodeSegment(decoded, packed, segment.Length, dictionary, segment.Unpacked, token), CancellationToken.None); + _ahead.Enqueue((segment, decoded)); } } @@ -321,7 +403,10 @@ public int ReadAt(long at, Span target) return 0; if (target.Length > length - at) target = target[..(int)(length - at)]; - shared.Position = offset + at; - return shared.ReadAtLeast(target, target.Length, throwOnEndOfStream: false); + lock (shared) // as SharedWindowStream: other coders of the folder may read on other threads + { + shared.Position = offset + at; + return shared.ReadAtLeast(target, target.Length, throwOnEndOfStream: false); + } } } diff --git a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs index 8937503c5..7066a4e0f 100644 --- a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs +++ b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs @@ -8,6 +8,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace Filee.Engines.Archives.Lzma; @@ -190,6 +191,16 @@ internal sealed class LzmaDecoder private const int FirstWindow = 1 << 16; + /// + /// Bytes the full window has beyond the dictionary. A match copy may write up to − 1 + /// bytes past its end (whole chunks are quicker than the exact tail); with this much room, the bytes it overwrites + /// lie farther back than any distance can reach before they are written again. + /// + private const int Guard = 32; + + /// Match copies move this many bytes at a time where they may (see ). + private const int CopyChunk = 16; + // The probabilities, in one array: the single-bit decisions, the slot and distance trees, the two length models // and, last, the literal coders (0x300 per context, 2^(lc + lp) contexts). private const int States = LzmaModel.States; @@ -225,6 +236,10 @@ internal sealed class LzmaDecoder private int _pending; // bytes of the current match not yet copied out private ushort[] _probabilities = []; + + // Copies MatchRun decoded but has not done yet: target position, distance and length, three ints each. + private const int MaxDeferred = 32; + private readonly int[] _deferred = new int[MaxDeferred * 3]; private int _runStop; // where the current Run stops private int _runTotalBase; // pos + this has the low bits of the total private int _lc; @@ -426,9 +441,9 @@ private int LiteralRun(RangeDecoder rc) where TInput : struct, IInputMod { uint p0 = Unsafe.Add(ref literal, symbol * 2); uint p1 = Unsafe.Add(ref literal, symbol * 2 + 1); - var one = BitMask(p, ref Unsafe.Add(ref literal, symbol), ref range, ref code, ref input, ref inPos, ref inEnd, rc); - p = p0 ^ ((p0 ^ p1) & one); - symbol = (symbol << 1) - one; + var zero = ZeroMask(p, ref Unsafe.Add(ref literal, symbol), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + p = p1 ^ ((p0 ^ p1) & zero); + symbol = (symbol << 1) + 1 + zero; } while (symbol < 0x100); } @@ -451,11 +466,11 @@ private int LiteralRun(RangeDecoder rc) where TInput : struct, IInputMod var index1 = offset1 + (after & offset1) + symbol * 2 + 1; uint p0 = Unsafe.Add(ref literal, index0); uint p1 = Unsafe.Add(ref literal, index1); - var one = BitMask(p, ref Unsafe.Add(ref literal, index), ref range, ref code, ref input, ref inPos, ref inEnd, rc); - p = p0 ^ ((p0 ^ p1) & one); - index = index0 ^ ((index0 ^ index1) & one); - offset = offset0 ^ ((offset0 ^ offset1) & one); - symbol = (symbol << 1) - one; + var zero = ZeroMask(p, ref Unsafe.Add(ref literal, index), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + p = p1 ^ ((p0 ^ p1) & zero); + index = index1 ^ ((index0 ^ index1) & zero); + offset = offset1 ^ ((offset0 ^ offset1) & zero); + symbol = (symbol << 1) + 1 + zero; match = after; } while (symbol < 0x100); @@ -500,6 +515,8 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput ref var input = ref MemoryMarshal.GetArrayDataReference(rc.Buffer); var inEnd = rc.End; rc.Take(out var range, out var code, out var inPos); + ref var deferred = ref MemoryMarshal.GetArrayDataReference(_deferred); + var copies = 0; int next; while (true) { @@ -549,28 +566,37 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput lengthModel = MatchLengthAt; } + int copyLength; if (shortRep) { state = state < LzmaModel.LiteralStates ? 9 : 11; - window[pos] = window[ByteAt(pos, rep0, window.Length)]; - pos++; + copyLength = 1; } else { - // The length, from the match or the repeat length model. + // The length, from the match or the repeat length model: one of three trees (one loop for all three + // keeps the method small enough for the coder to stay in registers). ref var model = ref Unsafe.Add(ref probs, lengthModel); + int tree, bits; if (Bit(ref model, ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) { - length = Tree(ref Unsafe.Add(ref model, LowAt + (posState << LengthModel.LowBits)), LengthModel.LowBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + tree = LowAt + (posState << LengthModel.LowBits); + bits = LengthModel.LowBits; + length = 0; } else if (Bit(ref Unsafe.Add(ref model, Choice2At), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) { - length = LengthModel.LowSymbols + Tree(ref Unsafe.Add(ref model, MidAt + (posState << LengthModel.MidBits)), LengthModel.MidBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + tree = MidAt + (posState << LengthModel.MidBits); + bits = LengthModel.MidBits; + length = LengthModel.LowSymbols; } else { - length = LengthModel.LowSymbols + LengthModel.MidSymbols + Tree(ref Unsafe.Add(ref model, HighAt), LengthModel.HighBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + tree = HighAt; + bits = LengthModel.HighBits; + length = LengthModel.LowSymbols + LengthModel.MidSymbols; } + length += Tree(ref Unsafe.Add(ref model, tree), bits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); if (lengthModel == RepLengthAt) { @@ -588,12 +614,24 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput break; } } - if (rep0 >= (_full ? (uint)window.Length : (uint)pos)) + if (rep0 >= (_full ? _dictionary : Math.Min((uint)pos, _dictionary))) throw new InvalidDataException("the LZMA data refers to bytes before the start of the window"); var matchLength = (int)length + LzmaModel.MatchMinLength; - var fits = Math.Min(matchLength, stop - pos); - _pending = matchLength - fits; - pos = Copy(window, pos, (int)rep0 + 1, fits); + copyLength = Math.Min(matchLength, stop - pos); + _pending = matchLength - copyLength; + } + + // The bytes are copied later, several matches at a time: the copies' sources are far apart in the window + // and mostly not in the cache, and done back to back their loads overlap. Nothing before the copies reads + // the window (only literals do), so the bytes come out the same. + Unsafe.Add(ref deferred, copies * 3) = pos; + Unsafe.Add(ref deferred, copies * 3 + 1) = (int)rep0 + 1; + Unsafe.Add(ref deferred, copies * 3 + 2) = copyLength; + pos += copyLength; + if (++copies == MaxDeferred) + { + CopyDeferred(window, copies); + copies = 0; } if (pos >= stop || (!TInput.Checked && inPos > inEnd - InputMargin)) @@ -608,6 +646,7 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput break; } } + CopyDeferred(window, copies); _pos = pos; _state = state; _rep0 = rep0; @@ -618,6 +657,14 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput return next; } + /// Does the first copies MatchRun put off, in order. + private void CopyDeferred(byte[] window, int count) + { + ref var deferred = ref MemoryMarshal.GetArrayDataReference(_deferred); + for (var i = 0; i < count; i++) + Copy(window, Unsafe.Add(ref deferred, i * 3), Unsafe.Add(ref deferred, i * 3 + 1), Unsafe.Add(ref deferred, i * 3 + 2)); + } + /// The index of the byte + 1 before . [MethodImpl(MethodImplOptions.AggressiveInlining)] private static long ByteAt(int pos, uint distance, int windowLength) @@ -639,6 +686,20 @@ private static int Copy(byte[] window, int pos, int distance, int count) if (source < 0) source += windowLength; ref var win = ref MemoryMarshal.GetArrayDataReference(window); + // Most matches: the source at least a chunk behind (or, wrapped, far ahead) and room for whole chunks on both + // sides. Chunks read only bytes written before them, and the few bytes written past the end are written + // again before anything reads them (Guard). + if (distance >= CopyChunk && source + count <= windowLength - CopyChunk && pos + count <= windowLength - CopyChunk) + { + var i = 0; + do + { + Vector128.LoadUnsafe(ref win, (nuint)(source + i)).StoreUnsafe(ref win, (nuint)(pos + i)); + i += CopyChunk; + } + while (i < count); + return pos + count; + } if (source + count <= windowLength) { // A forward copy is what LZMA means. It equals a block move when source and target don't overlap or the @@ -680,13 +741,13 @@ private static int Copy(byte[] window, int pos, int distance, int count) return pos + count; } - /// The write position reached the end of the window: grow it while it is below the dictionary, else wrap. + /// The write position reached the end of the window: grow it while it is below the dictionary (and the guard), else wrap. private void WindowEnd() { - if ((uint)_window.Length < _dictionary) + if ((uint)_window.Length < _dictionary + Guard) { - // Still filling: grow (twice the size, up to the dictionary) instead of wrapping. - Array.Resize(ref _window, (int)Math.Min(_dictionary, (long)_window.Length * 2)); + // Still filling: grow (twice the size, up to the dictionary and the guard) instead of wrapping. + Array.Resize(ref _window, (int)Math.Min(_dictionary + Guard, (long)_window.Length * 2)); } else { @@ -761,41 +822,49 @@ private static uint Bit(ref ushort probability, ref uint range, ref uint } /// - /// One adaptive bit without a branch on its value: returns all ones for a 1 and 0 for a 0, so a tree index moves - /// on with (m << 1) − mask. The probability update is the same as in : - /// p − ((p − 2017) >> 5) with an arithmetic shift is p + ((2048 − p) >> 5) for every p in 0 … 2048. + /// One adaptive bit, with the probability already read into , without a branch on its value: + /// returns all ones for a 0 and 0 for a 1 (the borrow of code − bound), so a tree index moves on with + /// (m << 1) + 1 + mask and a choice between two values is b ^ ((a ^ b) & mask). The + /// probability update is the same as in : p − ((p − 2017) >> 5) with an arithmetic + /// shift is p + ((2048 − p) >> 5) for every p in 0 … 2048. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static uint BitMask(ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) - where TInput : struct, IInputMode => - BitMask(probability, ref probability, ref range, ref code, ref input, ref inPos, ref inEnd, rc); - - /// with the probability already read into . - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static uint BitMask(uint p, ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) + private static uint ZeroMask(uint p, ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) where TInput : struct, IInputMode { var bound = (range >> RangeCoder.ProbabilityBits) * p; - var one = (uint)(((long)bound - code - 1) >> 63); // code >= bound - range = bound + ((range - 2 * bound) & one); // bound for a 0, range − bound for a 1 - code -= bound & one; - probability = (ushort)(p - (uint)((int)(p - (~one & (RangeCoder.ProbabilityOne - (1 << RangeCoder.MoveBits) + 1))) >> RangeCoder.MoveBits)); + var zero = (uint)(((ulong)code - bound) >> 32); // code < bound: the 64-bit difference wraps + range = range - bound + ((2 * bound - range) & zero); // bound for a 0, range − bound for a 1 + code -= bound & ~zero; + probability = (ushort)(p - (uint)((int)(p - (zero & (RangeCoder.ProbabilityOne - (1 << RangeCoder.MoveBits) + 1))) >> RangeCoder.MoveBits)); if (range < Top) { range <<= 8; code = code << 8 | Next(ref input, ref inPos, ref inEnd, rc); } - return one; + return zero; } - /// A -bit symbol from a binary tree of probabilities (index 1 is the root). + /// + /// A -bit symbol from a binary tree of probabilities (index 1 is the root). Like the + /// literals, each step reads both children's probabilities before its bit is known (all but the last, whose + /// children lie past the tree), so the next step doesn't wait for a load that depends on this bit. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint Tree(ref ushort probs, int bits, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) where TInput : struct, IInputMode { uint m = 1; - for (var i = 0; i < bits; i++) - m = (m << 1) - BitMask(ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + uint p = Unsafe.Add(ref probs, 1); + for (var i = 1; i < bits; i++) + { + uint p0 = Unsafe.Add(ref probs, m * 2); + uint p1 = Unsafe.Add(ref probs, m * 2 + 1); + var zero = ZeroMask(p, ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + p = p1 ^ ((p0 ^ p1) & zero); + m = (m << 1) + 1 + zero; + } + m = (m << 1) + 1 + ZeroMask(p, ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc); return m - (1u << bits); } @@ -806,13 +875,19 @@ private static uint ReverseTree(ref ushort probs, int bits, ref uint ran { uint m = 1; uint symbol = 0; - for (var i = 0; i < bits; i++) + uint p = Unsafe.Add(ref probs, 1); + for (var i = 0; i < bits - 1; i++) { - var bit = 0u - BitMask(ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + uint p0 = Unsafe.Add(ref probs, m * 2); + uint p1 = Unsafe.Add(ref probs, m * 2 + 1); + var zero = ZeroMask(p, ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + p = p1 ^ ((p0 ^ p1) & zero); + var bit = zero + 1; m = (m << 1) + bit; symbol |= bit << i; } - return symbol; + var last = 1 + ZeroMask(p, ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + return symbol | (last << (bits - 1)); } /// A match distance minus 1 (0xFFFFFFFF is the end marker): slot, then reverse-coded or direct bits. @@ -834,7 +909,9 @@ private static uint DecodeDistance(ref ushort probs, uint length, ref ui { range >>= 1; code -= range; - var t = 0u - (code >> 31); // all ones when the subtraction went below zero (bit 0) + // All ones when the subtraction went below zero (bit 0). The range is now below 2^31 and the code was below + // twice the range plus one, so a difference that didn't go below zero is below 2^31 too: the sign tells. + var t = (uint)((int)code >> 31); code += range & t; if (code == range) throw new InvalidDataException("the LZMA data is damaged"); diff --git a/src/Filee.Engines/Archives/Lzma/XzReader.cs b/src/Filee.Engines/Archives/Lzma/XzReader.cs index 5b411e341..e4b391418 100644 --- a/src/Filee.Engines/Archives/Lzma/XzReader.cs +++ b/src/Filee.Engines/Archives/Lzma/XzReader.cs @@ -12,7 +12,8 @@ // whole, then decoded and checked on a worker thread while the next block headers are read (Lzma2Parallel bounds how // many and how large). Blocks are handed out in order; a block that fails on its thread is decoded again on the // reader's thread as it is read, so damaged data gives the same bytes and the same error, at the same place, as the -// one-thread reader. An error in a header read ahead is thrown after the blocks before it. +// one-thread reader. An error in a header read ahead is thrown after the blocks before it. A worker hands out its +// block as it decodes it (the check is compared at the block's end, as the one-thread reader does). using System.Buffers.Binary; using System.Runtime.ExceptionServices; @@ -50,7 +51,8 @@ private enum Step { StreamHeader, BlockOrIndex, BlockData, StreamPadding, Done } // Blocks decoding on worker threads, in order; the one being handed out; an error met while reading ahead. private readonly Queue _ahead = new(); - private Lzma2Parallel.Decoded? _current; + private AheadBlock? _current; + private int _currentReady; private Replay? _replay; private int _currentPosition; private int _aheadLimit = Environment.ProcessorCount; @@ -98,32 +100,39 @@ public override int Read(Span buffer) } if (_current is { } current) { - if (_currentPosition < current.Count) + if (_currentPosition < _currentReady) { - var count = Math.Min(buffer.Length, current.Count - _currentPosition); - current.Output.AsSpan(_currentPosition, count).CopyTo(buffer); + var count = Math.Min(buffer.Length, _currentReady - _currentPosition); + current.Work.Output.AsSpan(_currentPosition, count).CopyTo(buffer); _currentPosition += count; _position += count; return count; } + // The worker hands its block out as it decodes it. + _currentReady = current.Work.WaitBeyond(_currentPosition, out var done); + if (_currentPosition < _currentReady || !done) + continue; _current = null; - } - ReadAhead(); - if (_ahead.TryDequeue(out var next)) - { - var decoded = next.Work.GetAwaiter().GetResult(); - if (decoded.Error is { } error) + if (current.Work.Error is { } error) { - // Decoded again here, as the one-thread reader reads it. + // Decoded again here, as the one-thread reader reads it, past what the worker handed out. if (error.SourceException is OperationCanceledException) error.Throw(); - var input = new ByteInput(new MemoryStream(next.Data, 0, next.Length, writable: false)); - _replay = new Replay(next, input, new Lzma2Reader(input, next.Dictionary, next.DeclaredUnpacked, _cancellationToken), + var input = new ByteInput(new MemoryStream(current.Data, 0, current.Length, writable: false)); + var again = new Replay(current, input, new Lzma2Reader(input, current.Dictionary, current.DeclaredUnpacked, _cancellationToken), Check.Create(_checkType)); - continue; + Lzma2Parallel.Skip(again.Reader, _currentPosition, again.Check.Append); + again.Unpacked = _currentPosition; + _replay = again; } - _current = decoded; + continue; + } + ReadAhead(); + if (_ahead.TryDequeue(out var next)) + { + _current = next; _currentPosition = 0; + _currentReady = 0; continue; } if (_deferred is { } deferred) @@ -286,14 +295,15 @@ private bool TryDecodeAhead(uint dictionary) _blocks.Add((_blockHeaderSize + _declaredPacked + checkSize, _declaredUnpacked)); _cancel ??= CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken); var (token, packed, unpacked, checkType, name) = (_cancel.Token, _declaredPacked, (int)_declaredUnpacked, _checkType, _name); - var work = Task.Run(() => DecodeBlock(data, read, dictionary, packed, unpacked, checkType, name, token), CancellationToken.None); + var work = new Lzma2Parallel.Decoded(GC.AllocateUninitializedArray(unpacked)); + _ = Task.Run(() => DecodeBlock(work, data, read, dictionary, packed, unpacked, checkType, name, token), CancellationToken.None); _ahead.Enqueue(new AheadBlock(work, data, read, dictionary, packed, unpacked)); _step = Step.BlockOrIndex; return true; } /// A block read whole and decoding on a worker thread, with what it takes to decode it again here. - private sealed record AheadBlock(Task Work, byte[] Data, int Length, uint Dictionary, long DeclaredPacked, + private sealed record AheadBlock(Lzma2Parallel.Decoded Work, byte[] Data, int Length, uint Dictionary, long DeclaredPacked, int DeclaredUnpacked); /// A block that failed on its thread, decoded again on this one as it is read. @@ -307,10 +317,9 @@ private sealed class Replay(AheadBlock block, ByteInput input, Lzma2Reader reade } /// A block read whole (data, padding and check), decoded and checked as does. - private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint dictionary, long declaredPacked, int declaredUnpacked, - int checkType, string name, CancellationToken cancellationToken) + private static void DecodeBlock(Lzma2Parallel.Decoded result, byte[] data, int length, uint dictionary, long declaredPacked, + int declaredUnpacked, int checkType, string name, CancellationToken cancellationToken) { - var result = new Lzma2Parallel.Decoded(GC.AllocateUninitializedArray(declaredUnpacked)); try { var input = new ByteInput(new MemoryStream(data, 0, length, writable: false)); @@ -319,7 +328,9 @@ private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint d var output = result.Output; while (true) { - var read = result.Count < declaredUnpacked ? reader.Read(output.AsSpan(result.Count)) : 0; + var read = result.Count < declaredUnpacked + ? reader.Read(output.AsSpan(result.Count, Math.Min(Lzma2Parallel.Decoded.Step, declaredUnpacked - result.Count))) + : 0; if (read == 0) { // All of it, or the data ended; one more byte would be past the header's size. @@ -330,6 +341,7 @@ private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint d } check.Append(output.AsSpan(result.Count, read)); result.Count += read; + result.Publish(); } CheckBlockEnd(input, input.Consumed, declaredPacked, declaredUnpacked, result.Count, checkType, check, name); } @@ -337,7 +349,10 @@ private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint d { result.Error = ExceptionDispatchInfo.Capture(ex); } - return result; + finally + { + result.Finish(); + } } private void FinishBlock() diff --git a/src/Filee.Engines/Archives/ReadAheadStream.cs b/src/Filee.Engines/Archives/ReadAheadStream.cs new file mode 100644 index 000000000..e8d1d93b8 --- /dev/null +++ b/src/Filee.Engines/Archives/ReadAheadStream.cs @@ -0,0 +1,197 @@ +// Runs a decoder on a thread of its own, a few blocks ahead of whoever reads it. A decoder that has to go through its +// data in order (one LZMA stream, a BCJ filter after it, the sub-streams of BCJ2) then works while the reader checks +// CRCs and writes files, or while the next stage of the chain works on what came before; the slowest stage sets the +// pace instead of the sum of them all. +// +// What the reader gets is exactly what the decoder gives, in the same order. The worker asks the decoder for +// PieceSize bytes at a time (what ArchiveFiles.CopyLimited asks for), so an error comes after the same bytes, give or +// take one piece: the decoder's exception itself is thrown to the reader once the bytes before it are read. Disposing +// stops the worker and waits for it, so the decoder (and the archive stream under it) is never used after that. + +using System.Runtime.ExceptionServices; + +namespace Filee.Engines.Archives; + +/// A read-only stream that reads its inner stream ahead on a worker thread. +internal sealed class ReadAheadStream : Stream +{ + /// Streams shorter than this are read directly: a thread would cost more than it gives. + public const long MinSize = 1 << 20; + + /// Bytes handed over at a time. + private const int BlockSize = 1 << 20; + + /// Bytes the worker asks the decoder for at a time. + private const int PieceSize = 1 << 16; + + /// Blocks read ahead at most (memory: this many blocks, plus the one being read). + private const int Depth = 4; + + /// Tests turn read-ahead off (false) or on for any size (true). + internal static readonly AsyncLocal TestMode = new(); + + /// Streams opened with a worker so far (for tests). + internal static int Opened; + + private readonly Stream _inner; + private readonly CancellationTokenSource _cancel; + private readonly object _gate = new(); + private readonly Queue _filled = new(); + private readonly Stack _free = new(); + private readonly Thread _worker; + private Block? _current; + private int _at; + private bool _stopping; + private bool _workerDone; + + private sealed record Block(byte[] Data, int Count, bool End, ExceptionDispatchInfo? Error); + + private ReadAheadStream(Stream inner, CancellationToken cancellationToken) + { + _inner = inner; + _cancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + for (var i = 0; i < Depth + 1; i++) + _free.Push(GC.AllocateUninitializedArray(BlockSize)); + Interlocked.Increment(ref Opened); + _worker = new Thread(Work) { IsBackground = true, Name = "Filee read-ahead" }; + _worker.Start(); + } + + /// + /// read ahead on a worker thread when that pays off (more than one processor, at least + /// bytes, or -1 for not known); else itself. + /// The result owns . + /// + public static Stream Wrap(Stream inner, long size, CancellationToken cancellationToken) + { + var mode = TestMode.Value; + if (mode == false || (mode is null && (Environment.ProcessorCount < 2 || (size >= 0 && size < MinSize)))) + return inner; + return new ReadAheadStream(inner, cancellationToken); + } + + private void Work() + { + var token = _cancel.Token; + try + { + while (true) + { + byte[] data; + lock (_gate) + { + while (_free.Count == 0 && !_stopping) + Monitor.Wait(_gate); + if (_stopping) + return; + data = _free.Pop(); + } + var count = 0; + var end = false; + ExceptionDispatchInfo? error = null; + try + { + while (count < data.Length) + { + token.ThrowIfCancellationRequested(); + var read = _inner.Read(data, count, Math.Min(PieceSize, data.Length - count)); + if (read <= 0) + { + end = true; + break; + } + count += read; + } + } + catch (Exception ex) + { + error = ExceptionDispatchInfo.Capture(ex); + } + lock (_gate) + { + _filled.Enqueue(new Block(data, count, end, error)); + Monitor.PulseAll(_gate); + } + if (end || error is not null) + return; + } + } + finally + { + lock (_gate) + { + _workerDone = true; + Monitor.PulseAll(_gate); + } + } + } + + public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + if (buffer.IsEmpty) + return 0; + while (true) + { + if (_current is { } current) + { + if (_at < current.Count) + { + var count = Math.Min(buffer.Length, current.Count - _at); + current.Data.AsSpan(_at, count).CopyTo(buffer); + _at += count; + return count; + } + current.Error?.Throw(); + if (current.End) + return 0; + lock (_gate) + { + _free.Push(current.Data); + Monitor.PulseAll(_gate); + } + _current = null; + } + lock (_gate) + { + while (_filled.Count == 0) + { + if (_workerDone) + throw new ObjectDisposedException(nameof(ReadAheadStream)); + Monitor.Wait(_gate); + } + _current = _filled.Dequeue(); + _at = 0; + } + } + } + + protected override void Dispose(bool disposing) + { + if (disposing && !_stopping) + { + lock (_gate) + { + _stopping = true; + Monitor.PulseAll(_gate); + } + _cancel.Cancel(); + // The decoder and the archive under it are left alone once this returns. + _worker.Join(); + _cancel.Dispose(); + _inner.Dispose(); + } + base.Dispose(disposing); + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); +} diff --git a/src/Filee.Engines/Archives/SevenZipCoders.cs b/src/Filee.Engines/Archives/SevenZipCoders.cs index 4b80c5b05..c8fc638b3 100644 --- a/src/Filee.Engines/Archives/SevenZipCoders.cs +++ b/src/Filee.Engines/Archives/SevenZipCoders.cs @@ -195,7 +195,7 @@ protected override void Dispose(bool disposing) /// /// A window of a stream others read too (the packed streams of one 7z folder side by side): every read seeks to -/// where this window stopped. Disposing it leaves the stream open. +/// where this window stopped, holding the stream's lock. Disposing it leaves the stream open. /// internal sealed class SharedWindowStream(Stream shared, long offset, long length) : ReadOnlyDecoderStream { @@ -215,9 +215,14 @@ public override int Read(Span buffer) return 0; if (buffer.Length > _end - _position) buffer = buffer[..(int)(_end - _position)]; - if (shared.Position != _position) - shared.Position = _position; - var read = shared.Read(buffer); + int read; + // The coders of one folder may read their packed streams on threads of their own (ReadAheadStream). + lock (shared) + { + if (shared.Position != _position) + shared.Position = _position; + read = shared.Read(buffer); + } if (read <= 0) return 0; _position += read; diff --git a/src/Filee.Engines/Archives/SevenZipReader.cs b/src/Filee.Engines/Archives/SevenZipReader.cs index 7468a5252..b7191a07d 100644 --- a/src/Filee.Engines/Archives/SevenZipReader.cs +++ b/src/Filee.Engines/Archives/SevenZipReader.cs @@ -725,7 +725,28 @@ public static Stream Open(Stream stream, SevenZipListing listing, int index, str firstOut[i] = firstOut[i - 1] + folder.Coders[i - 1].OutStreams; } var opened = new bool[folder.Coders.Length]; - return OpenCoder(Array.IndexOf(firstOut, folder.MainOut)); + // Each coder that feeds another decodes ahead on a thread of its own, and so does the last one: the chain's + // stages and the writing of the files run side by side. The threads are stopped with the folder's stream, + // or here when the folder doesn't open. + var ahead = new List(); + try + { + var output = Ahead(OpenCoder(Array.IndexOf(firstOut, folder.MainOut)), folder.UnpackSize); + return ahead.Count == 0 ? output : new FolderOutput(output, ahead); + } + catch + { + FolderOutput.Stop(ahead); + throw; + } + + Stream Ahead(Stream decoder, long size) + { + var stream = ReadAheadStream.Wrap(decoder, size, cancellationToken); + if (stream != decoder) + ahead.Add(stream); + return stream; + } Stream OpenCoder(int coder) { @@ -740,7 +761,8 @@ Stream OpenCoder(int coder) var pair = Array.FindIndex(folder.BindPairs, p => p.In == input); if (pair >= 0) { - inputs[j] = OpenCoder(Array.IndexOf(firstOut, folder.BindPairs[pair].Out)); + var bound = folder.BindPairs[pair].Out; + inputs[j] = Ahead(OpenCoder(Array.IndexOf(firstOut, bound)), folder.UnpackSizes[bound]); continue; } var pack = folder.FirstPackStream + Array.IndexOf(folder.PackedStreams, input); @@ -752,6 +774,31 @@ Stream OpenCoder(int coder) } } + /// + /// A folder's output with coders read through s. Disposing it stops them all, the + /// later (which read the earlier) first. + /// + private sealed class FolderOutput(Stream output, List ahead) : ReadOnlyDecoderStream + { + public override int Read(Span buffer) => output.Read(buffer); + + public static void Stop(List ahead) + { + for (var i = ahead.Count - 1; i >= 0; i--) + ahead[i].Dispose(); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + output.Dispose(); + Stop(ahead); + } + base.Dispose(disposing); + } + } + /// Reads the numbers, bit vectors and digests of a 7z header; running out of bytes is damage. private sealed class HeaderReader(byte[] data, string name) { From 72a5679685cd4673e7163d574c29e13f97b4ee83 Mon Sep 17 00:00:00 2001 From: KnifeLemon Date: Thu, 19 Nov 2026 11:21:04 +0900 Subject: [PATCH 2/2] Faster LZMA and bzip2 encoding: split tree search, lazy fast parse, vector table costs - Binary-tree match finder searches on up to four threads: the trees are split by hash, so the matches (and bytes) are the same whatever the number of threads; walks stop a little short of the dictionary so no thread reads a slot another took over - Fast level: lazy parse with a look at the next position, 256 KiB dictionary (tables stay in cache), 1 MiB blocks; 7z solid blocks stay 64 MiB - Range encoder keeps low and range in locals for trees, literals and direct bits, coded without a branch on the bit - bzip2: blocks on up to eight threads, group costs of all tables in one vector - Tests: search threads give the same bytes, a distance past the dictionary is damage within the window's guard, read-ahead errors, end, stop and cancel, BCJ/BCJ2 folders with coders on threads Co-Authored-By: Claude Opus 5.5 --- .../Archives/Bzip2BlockEncoder.cs | 37 +- src/Filee.Engines/Archives/Bzip2Writer.cs | 6 +- .../Archives/Lzma/Lzma2Writer.cs | 36 +- .../Archives/Lzma/LzmaEncoder.cs | 168 +++++--- .../Archives/Lzma/MatchFinder.cs | 388 ++++++++++++++---- .../Archives/Lzma/RangeEncoder.cs | 118 +++++- src/Filee.Engines/Archives/SevenZipWriter.cs | 15 +- .../Filee.Engines.Tests/ArchiveThreadTests.cs | 215 +++++++++- 8 files changed, 805 insertions(+), 178 deletions(-) diff --git a/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs b/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs index 6f6a3a89f..443d6516d 100644 --- a/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs +++ b/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs @@ -6,6 +6,10 @@ // bits or less, which every bzip2 decoder takes. The result is a bit string (Bzip2Bits) that starts with the block // magic; blocks are not byte aligned, so the writer joins them bit by bit. +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + namespace Filee.Engines.Archives; /// Encodes bzip2 blocks; reusable, one block at a time. @@ -28,6 +32,10 @@ internal sealed class Bzip2BlockEncoder private readonly uint[][] _codes = CreateTables(); private readonly int[][] _tableFrequencies = CreateTables(); + // Per symbol, its code length in each table side by side (eight lanes, the unused ones too dear to be chosen), so + // a group's cost in every table is one vector sum. + private readonly Vector128[] _packedLengths = new Vector128[MaxAlphabet]; + /// The last block's bits, from its magic to its last Huffman code. public Bzip2Bits Output { get; } = new(); @@ -231,28 +239,41 @@ private void ChooseTables(int count, int alphabet, int tables, int selectorCount remaining -= sum; } - Span cost = stackalloc int[MaxTables]; + Span lanes = stackalloc ushort[Vector128.Count]; var symbols = _symbols; + if (count > symbols.Length) + throw new ArgumentOutOfRangeException(nameof(count)); + ref var symbol0 = ref MemoryMarshal.GetArrayDataReference(symbols); + ref var packed = ref MemoryMarshal.GetArrayDataReference(_packedLengths); for (var pass = 0; pass < passes; pass++) { for (var t = 0; t < tables; t++) Array.Clear(_tableFrequencies[t], 0, alphabet); + for (var v = 0; v < alphabet; v++) + { + // A group holds at most 50 symbols of at most 17 bits: 255 a lane never wins and never overflows. + lanes.Fill(255); + for (var t = 0; t < tables; t++) + lanes[t] = _lengths[t][v]; + _packedLengths[v] = Vector128.Create(lanes); + } for (var g = 0; g < selectorCount; g++) { var start = g * GroupSize; var end = Math.Min(count, start + GroupSize); - cost.Clear(); + var cost = Vector128.Zero; for (var i = start; i < end; i++) - { - var symbol = symbols[i]; - for (var t = 0; t < tables; t++) - cost[t] += _lengths[t][symbol]; - } + cost += Unsafe.Add(ref packed, Unsafe.Add(ref symbol0, i)); // symbols are below the alphabet var best = 0; + var bestCost = cost.GetElement(0); for (var t = 1; t < tables; t++) { - if (cost[t] < cost[best]) + var c = cost.GetElement(t); + if (c < bestCost) + { best = t; + bestCost = c; + } } _selectors[g] = (byte)best; var chosen = _tableFrequencies[best]; diff --git a/src/Filee.Engines/Archives/Bzip2Writer.cs b/src/Filee.Engines/Archives/Bzip2Writer.cs index 3da25d791..baf75e8af 100644 --- a/src/Filee.Engines/Archives/Bzip2Writer.cs +++ b/src/Filee.Engines/Archives/Bzip2Writer.cs @@ -1,7 +1,7 @@ // Writes .bz2 files with Filee's own encoder, from the bzip2 format: the "BZh9" stream header, blocks of at most // 900,000 bytes, and the end marker with the CRC of all block CRCs. The input goes through bzip2's first run-length // step here (runs of 4 to 255 equal bytes become 4 bytes and a count) while the block CRC is taken over the bytes as -// they came; full blocks are encoded (Bzip2BlockEncoder) on up to four threads at once, and written in order. Memory +// they came; full blocks are encoded (Bzip2BlockEncoder) on up to eight threads at once, and written in order. Memory // stays bounded by the number of blocks in flight (about 20 MB each while sorting). It is a write-only stream, so a TAR // can be compressed while it is written (.tar.bz2) without a temporary file. @@ -45,8 +45,8 @@ private sealed class Slot public Task? Work { get; set; } } - /// Blocks encoded at once by default: a few cores, leaving the rest to the machine. - internal static int DefaultThreads => Math.Clamp(Environment.ProcessorCount / 2, 1, 4); + /// Blocks encoded at once by default: one per processor, at most eight (about 20 MB each while sorting). + internal static int DefaultThreads => Math.Clamp(Environment.ProcessorCount, 1, 8); /// Blocks encoded at once (1 encodes on the writing thread). public Bzip2Writer(Stream output, ArchiveLevel level, CancellationToken cancellationToken = default, bool leaveOpen = false, diff --git a/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs b/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs index d919df87e..7cb81387f 100644 --- a/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs +++ b/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs @@ -8,7 +8,7 @@ // A block's first chunk resets the dictionary and sets the properties; after a first stored chunk (0x01) the decoder // needs the properties again, so the next LZMA chunk carries them (0xC0). // With processors to spare (fewer blocks than half of them), each block's binary-tree match finder searches ahead on -// a thread of its own (MatchFinder.StartAhead), which gives the same bytes too. +// threads of its own (MatchFinder.StartAhead, the trees split between them by hash), which gives the same bytes too. using System.Numerics; using Filee.Core.Presets; @@ -18,9 +18,11 @@ namespace Filee.Engines.Archives.Lzma; /// Dictionary and search effort for one . internal readonly record struct Lzma2Settings(int Dictionary, LzmaEncoderSettings Encoder, bool Store) { + /// /// The settings for , with the dictionary cut to the input when its size is known. Memory - /// for one block's encoder is about 8 × the dictionary with the hash chain (fast: 4 MiB → ~32 MB) and 10–11 × with + /// for one block's encoder is about 8 × the dictionary with the hash chain (fast: 256 KiB → ~2 MB; small, so the + /// tables mostly stay in the processor's cache, and the lazy parse makes up the ratio) and 10–11 × with /// the binary tree (normal: 16 MiB → ~180 MB, maximum: 64 MiB → ~660 MB, as xz -9), plus the block itself while it /// is coded (); unpacking needs only the dictionary. /// @@ -29,7 +31,7 @@ public static Lzma2Settings For(ArchiveLevel level, long expectedSize = -1) var settings = level switch { ArchiveLevel.Store => new Lzma2Settings(1 << 12, default, Store: true), - ArchiveLevel.Fast => new Lzma2Settings(1 << 22, new LzmaEncoderSettings(Depth: 8, Nice: 32, Optimal: false, Tree: false), false), + ArchiveLevel.Fast => new Lzma2Settings(1 << 18, new LzmaEncoderSettings(Depth: 8, Nice: 32, Optimal: false, Tree: false), false), ArchiveLevel.Maximum => new Lzma2Settings(1 << 26, new LzmaEncoderSettings(Depth: 64, Nice: 64, Optimal: true, Tree: true), false), _ => new Lzma2Settings(1 << 24, new LzmaEncoderSettings(Depth: 32, Nice: 32, Optimal: true, Tree: true), false), }; @@ -43,8 +45,8 @@ public static Lzma2Settings For(ArchiveLevel level, long expectedSize = -1) /// /// The input coded as one block, from one dictionary reset to the next, at least 1 MiB and at most 256 MiB: one - /// dictionary at the fast level (many blocks for many threads; about 0.2 % larger output), two at normal (about - /// 0.15 %), four at maximum. Matches never reach into the block before, which costs that little ratio. + /// dictionary at the fast level (that is 1 MiB: many blocks for many threads), two at normal (about 0.15 % larger + /// output), four at maximum. Matches never reach into the block before, which costs that little ratio. /// public int BlockSize => (int)Math.Clamp((long)Dictionary * (!Encoder.Optimal ? 1 : Encoder.Depth >= 64 ? 4 : 2), 1 << 20, 1 << 28); } @@ -60,6 +62,9 @@ internal sealed class Lzma2Writer /// Tests turn the match finder's own thread on or off (normally on with processors to spare). internal static readonly AsyncLocal TestSearchAhead = new(); + /// Tests set how many threads the match finder of each block searches on (normally from the processors). + internal static readonly AsyncLocal TestSearchThreads = new(); + private readonly Stream _output; private readonly Lzma2Settings _settings; private readonly Action? _onBlock; @@ -67,7 +72,7 @@ internal sealed class Lzma2Writer private readonly int _blockSize; private readonly int _firstCapacity; private readonly int _threads; - private readonly bool _searchAhead; + private readonly int _searchThreads; // per block: 0 for none, else the match finder's tree threads private readonly bool _check; private readonly Stack _free = new(); private readonly Queue<(Lzma2BlockEncoder Encoder, Task Work)> _pending = new(); @@ -96,8 +101,11 @@ public Lzma2Writer(Stream output, Lzma2Settings settings, long expectedSize, Can ? (int)Math.Min(_blockSize, Math.Max(expectedSize, 1) + LzmaModel.MatchMaxLength + 4) : _blockSize; _threads = TestThreads.Value ?? Threads(settings, _blockSize, expectedSize); - _searchAhead = settings.Encoder is { Tree: true, Optimal: true } - && (TestSearchAhead.Value ?? 2 * _threads <= Environment.ProcessorCount); + // Processors left over by the blocks go to their match finders: the trees are split between threads by hash. + var searchAhead = settings.Encoder is { Tree: true, Optimal: true } + && (TestSearchAhead.Value ?? 2 * _threads <= Environment.ProcessorCount); + _searchThreads = !searchAhead ? 0 + : TestSearchThreads.Value ?? Math.Clamp(Environment.ProcessorCount / _threads - 1, 1, MatchFinder.MaxSearchThreads); } /// The dictionary size the decoder needs (for the filter properties). @@ -187,12 +195,12 @@ private void Dispatch(Lzma2BlockEncoder encoder) _cancellationToken.ThrowIfCancellationRequested(); if (_threads == 1) { - encoder.Encode(_cancellationToken, _searchAhead); + encoder.Encode(_cancellationToken, _searchThreads); Emit(encoder); _free.Push(encoder); return; } - var (token, ahead) = (_cancellationToken, _searchAhead); + var (token, ahead) = (_cancellationToken, _searchThreads); _pending.Enqueue((encoder, Task.Run(() => encoder.Encode(token, ahead), CancellationToken.None))); } @@ -311,8 +319,8 @@ public void Reset() } /// Codes the whole block. - /// Lets the match finder search on a thread of its own (binary tree, large blocks). - public void Encode(CancellationToken cancellationToken, bool searchAhead = false) + /// Lets the match finder search ahead on this many threads of its own (binary tree, large blocks; 0 for none). + public void Encode(CancellationToken cancellationToken, int searchThreads = 0) { _first = true; _needProperties = false; @@ -325,8 +333,8 @@ public void Encode(CancellationToken cancellationToken, bool searchAhead = false WriteStored(_stored.AsSpan(0, _storedCount)); return; } - if (searchAhead && _settings.Encoder.Tree && _mf.Filled >= 1 << 18) - _mf.StartAhead(cancellationToken); + if (searchThreads > 0 && _settings.Encoder.Tree && _mf.Filled >= 1 << 18) + _mf.StartAhead(cancellationToken, searchThreads); try { while (_mf.Position < _mf.Filled) diff --git a/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs b/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs index 2859f385d..3bec65072 100644 --- a/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs +++ b/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs @@ -3,8 +3,10 @@ // and a new match from the MatchFinder. // // Two parsers: -// - Greedy (the fast level): take the longest match. A repeat is preferred over a slightly longer new match, since it -// needs no distance; literal against short repeat is decided by their bit prices. +// - Lazy (the fast level): take the longest match, unless the next position starts a better one (longer, or as long +// and nearer), in which case a literal goes first. A repeat is preferred over a slightly longer new match, since +// it needs no distance, and a match one shorter is preferred when it is far nearer; literal against short repeat +// is decided by their bit prices. // - Price-based (normal and maximum): a shortest-path search over the next positions. Every position in a window // gets all its matches (one per length, the nearest distance), its four repeats, a literal and a short repeat; // each choice is priced in 1/16 bits with the current probabilities and the cheapest way to reach every later @@ -13,6 +15,7 @@ // its end is coded. Prices are refreshed from the probabilities every few KB. using System.Numerics; +using System.Runtime.CompilerServices; namespace Filee.Engines.Archives.Lzma; @@ -34,6 +37,12 @@ internal sealed class LzmaEncoder : LzmaModel private int _cachedLength; private uint _cachedDistance; + // The lazy parse's look at the next position: its matches, kept for the next call when a literal went first. + private int _nextAt = -1; + private int _nextCount; + private readonly int[] _nextLengths = new int[MatchMaxLength + 1]; + private readonly uint[] _nextDistances = new uint[MatchMaxLength + 1]; + // The price-based parser: matches of the current position, the path nodes, and price tables. private const int OptimalWindow = 2048; private const int LengthSymbols = MatchMaxLength - MatchMinLength + 1; @@ -91,6 +100,7 @@ public void Restart() ResetState(); _total = 0; _cachedAt = -1; + _nextAt = -1; } /// The window moved back by bytes (a slide); the cached search moves with it. @@ -98,6 +108,8 @@ public void Shifted(int shift) { if (_cachedAt >= 0) _cachedAt -= shift; + if (_nextAt >= 0) + _nextAt -= shift; } private void EncodeOne(RangeEncoder rc, int end) @@ -106,30 +118,26 @@ private void EncodeOne(RangeEncoder rc, int end) var pos = _mf.Position; var available = Math.Min(MatchMaxLength, end - pos); - var mainLength = _mf.Find(available, out var mainDistance); - - // The longest repeat of one of the last four distances. - var repLength = 0; - var repIndex = 0; - if (available >= 2) + // This position's matches (searched by the last call when it looked ahead), each longer than the one before. + int count; + if (_nextAt == pos) { - for (var i = 0; i < 4; i++) + count = _nextCount; + for (var k = 0; k < count; k++) { - var distance = Rep(i); - if (distance >= _total) - continue; - var from = pos - (int)distance - 1; - if (buffer[from] != buffer[pos] || buffer[from + 1] != buffer[pos + 1]) - continue; - var length = MatchFinder.Common(buffer, pos, from, available); - if (length > repLength) - { - repLength = length; - repIndex = i; - } + _lengths[k] = Math.Min(_nextLengths[k], available); + _distances[k] = _nextDistances[k]; } } + else + { + count = _mf.FindAll(pos, available, _lengths, _distances); + } + _nextAt = -1; + var mainLength = count > 0 ? _lengths[count - 1] : 0; + var mainDistance = count > 0 ? _distances[count - 1] : 0; + var repLength = LongestRep(buffer, pos, available, out var repIndex); if (repLength >= _settings.Nice) { EmitRep(rc, repIndex, repLength); @@ -141,6 +149,14 @@ private void EncodeOne(RangeEncoder rc, int end) return; } + // One byte shorter but far nearer is cheaper. + while (count > 1 && _lengths[count - 2] + 1 == mainLength && FarNearer(_distances[count - 2], mainDistance)) + { + count--; + mainLength = _lengths[count - 1]; + mainDistance = _distances[count - 1]; + } + // Short matches far back cost more than the literals they replace. if (mainLength == 2 && mainDistance >= 128 || mainLength == 3 && mainDistance >= 1 << 14) mainLength = 0; @@ -153,10 +169,84 @@ private void EncodeOne(RangeEncoder rc, int end) return; } - if (mainLength >= 2) - EmitMatch(rc, mainLength, mainDistance); - else + if (mainLength < 2) + { EmitLiteralOrShortRep(rc); + return; + } + + // A look at the next position: a better match there (or a repeat nearly as long) makes this one a literal. + if (available > 2) + { + var nextAvailable = Math.Min(MatchMaxLength, end - pos - 1); + _nextCount = _mf.FindAll(pos + 1, nextAvailable, _nextLengths, _nextDistances); + _nextAt = pos + 1; + if (_nextCount > 0) + { + var nextLength = _nextLengths[_nextCount - 1]; + var nextDistance = _nextDistances[_nextCount - 1]; + if (nextLength >= mainLength && nextDistance < mainDistance + || nextLength == mainLength + 1 && !FarNearer(mainDistance, nextDistance) + || nextLength > mainLength + 1 + || nextLength + 1 >= mainLength && mainLength >= 3 && FarNearer(nextDistance, mainDistance)) + { + EmitLiteral(rc); + return; + } + } + if (RepReaches(buffer, pos + 1, nextAvailable, Math.Max(mainLength - 1, 2))) + { + EmitLiteral(rc); + return; + } + } + EmitMatch(rc, mainLength, mainDistance); + } + + /// True when the distance is less than 1/128 of . + private static bool FarNearer(uint near, uint far) => far >> 7 > near; + + /// The longest repeat of one of the last four distances at (0 when none of 2 or more). + private int LongestRep(byte[] buffer, int pos, int available, out int repIndex) + { + repIndex = 0; + var repLength = 0; + if (available < 2) + return 0; + for (var i = 0; i < 4; i++) + { + var distance = Rep(i); + if (distance >= _total) + continue; + var from = pos - (int)distance - 1; + if (buffer[from] != buffer[pos] || buffer[from + 1] != buffer[pos + 1]) + continue; + var length = MatchFinder.Common(buffer, pos, from, available); + if (length > repLength) + { + repLength = length; + repIndex = i; + } + } + return repLength; + } + + /// True when one of the last four distances repeats at least bytes at (one byte past the cursor). + private bool RepReaches(byte[] buffer, int pos, int available, int length) + { + if (available < length) + return false; + for (var i = 0; i < 4; i++) + { + var distance = Rep(i); + if (distance > _total) + continue; + var from = pos - (int)distance - 1; + if (buffer[from] == buffer[pos] && buffer[from + 1] == buffer[pos + 1] + && MatchFinder.Common(buffer, pos, from, length) == length) + return true; + } + return false; } // ───────── Price-based parse ───────── @@ -430,6 +520,7 @@ private void RefreshPrices() _alignPrices[i] = BitPrice.ReverseTree(Align, AlignBits, i); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private uint Rep(int index) => index switch { 0 => Rep0, 1 => Rep1, 2 => Rep2, _ => Rep3 }; private void EmitLiteralOrShortRep(RangeEncoder rc) @@ -469,35 +560,12 @@ private void EmitLiteral(RangeEncoder rc) var pos = _mf.Position; var posState = (int)_total & PbMask; rc.EncodeBit(ref IsMatch[(State << 4) + posState], 0); - var probs = Literal.AsSpan(LiteralBase(_total, _total > 0 ? buffer[pos - 1] : 0), 0x300); + ref var probs = ref Literal.AsSpan(LiteralBase(_total, _total > 0 ? buffer[pos - 1] : 0), 0x300)[0]; uint symbol = buffer[pos]; if (State >= LiteralStates) - { - // Matched literal: while the bits agree with the byte at the last distance, a second set of - // probabilities (chosen by that byte's bit) is used. - uint matchByte = buffer[pos - (int)Rep0 - 1]; - uint m = 1; - var same = true; - for (var i = 7; i >= 0; i--) - { - var bit = (int)((symbol >> i) & 1); - if (same) - { - var matchBit = (matchByte >> i) & 1; - rc.EncodeBit(ref probs[(int)(((1 + matchBit) << 8) + m)], bit); - same = matchBit == (uint)bit; - } - else - { - rc.EncodeBit(ref probs[(int)m], bit); - } - m = (m << 1) | (uint)bit; - } - } + rc.EncodeMatchedLiteral(ref probs, symbol, buffer[pos - (int)Rep0 - 1]); else - { - rc.EncodeTree(probs, 8, symbol); - } + rc.EncodeLiteral(ref probs, symbol); State = AfterLiteral(State); Advance(1); } diff --git a/src/Filee.Engines/Archives/Lzma/MatchFinder.cs b/src/Filee.Engines/Archives/Lzma/MatchFinder.cs index 37522d4ca..f3d5eb3d5 100644 --- a/src/Filee.Engines/Archives/Lzma/MatchFinder.cs +++ b/src/Filee.Engines/Archives/Lzma/MatchFinder.cs @@ -12,10 +12,14 @@ // is used up. Positions are stored as buffer index + an offset that grows with each slide, so the tables never need // rewriting when the window moves; only every ~1 GiB of input are they rebased (Normalize). 0 marks an empty slot. // -// Searching ahead (binary tree, for the price-based parse, with the whole input in the buffer): a thread of its own -// searches every position in order, as the parse would search it or step over it (both walk the tree the same way), -// and hands the matches over in batches; the parse takes each position's list cut to the length it allows there. -// The lists are exactly those a search in the parse would find, so the output is the same bytes either way. +// Searching ahead (binary tree, for the price-based parse, with the whole input in the buffer): threads of their own +// search every position in order, as the parse would search it or step over it (both walk the tree the same way), +// and hand the matches over in batches; the parse takes each position's list cut to the length it allows there. +// A position's tree holds only positions with the same four-byte hash, so the trees are split between the threads +// by hash: each thread walks the positions of its own hashes, in order, and touches no other tree. The last thread +// to finish a batch adds the close matches of the two- and three-byte tables (kept in order, by one thread at a +// time) and joins them with the tree's as one search would. The lists are exactly those a search in the parse would +// find, so the output is the same bytes whatever the number of threads, or with none. using System.Buffers.Binary; using System.Collections.Concurrent; @@ -31,10 +35,14 @@ internal sealed class MatchFinder private const int Hash3Bits = 16; private const int MinAhead = 1 << 16; + /// The most threads searches on. + public const int MaxSearchThreads = Ahead.MaxThreads; + /// Tests lower the offset at which positions are rebased (normally 1 GiB). internal static readonly AsyncLocal TestNormalizeAt = new(); private readonly int _dictionary; + private readonly int _reach; // how far back the tree walks: the dictionary less the search threads' leeway private readonly int _mask; private readonly bool _tree; private readonly int[] _links; // chain: one previous position per slot; tree: left and right child per slot @@ -58,6 +66,10 @@ public MatchFinder(int dictionary, int ahead, int depth, int nice, bool tree) if (!BitOperations.IsPow2(dictionary) || dictionary < 1 << 12 || dictionary > 1 << 28) throw new ArgumentOutOfRangeException(nameof(dictionary)); _dictionary = dictionary; + // A tree slot is reused a dictionary later, possibly by another search thread (they split the trees by hash + // and may be up to Ahead.MaxLag positions apart): walks stop that far short of the dictionary, so no thread + // reads a slot another has taken over. The same with one thread or none, so the bytes don't depend on it. + _reach = tree ? dictionary - Math.Min(Ahead.MaxLag, dictionary >> 2) : dictionary; _mask = dictionary - 1; _tree = tree; _links = new int[tree ? 2L * dictionary : dictionary]; @@ -155,12 +167,6 @@ static void Rebase(int[] table, int delta) _offset -= delta; } - /// - /// Enters in the tables and returns the longest match there of at most - /// bytes (0 when there is none of at least 2), with its distance − 1. - /// - public int Find(int maxLength, out uint distance) => Search(Position, maxLength, null, null, out distance); - /// /// Enters in the tables and lists its matches of at most /// bytes, each longer than the one before; returns how many. @@ -169,15 +175,17 @@ public int FindAll(int pos, int maxLength, int[] lengths, uint[] distances) => _ahead is { } ahead ? ahead.Take(pos, maxLength, lengths, distances) : Search(pos, maxLength, lengths, distances, out _); /// - /// Starts searching every position from to on a thread of its own - /// (binary tree only); then takes its results and does nothing. Call - /// when done. + /// Starts searching every position from to on + /// threads of their own (binary tree only); then takes their + /// results and does nothing. Call when done. /// - public void StartAhead(CancellationToken cancellationToken) + public void StartAhead(CancellationToken cancellationToken, int threads = 1) { if (!_tree) throw new InvalidOperationException("only the binary tree searches ahead"); - _ahead = new Ahead(this, Position, Filled, cancellationToken); + if (threads > 1 && _dictionary < 4 * Ahead.MaxLag) + threads = 1; // the walks' leeway (a quarter of the dictionary at most) would not cover the threads' lag + _ahead = new Ahead(this, Position, Filled, Math.Clamp(threads, 1, Ahead.MaxThreads), cancellationToken); } /// Stops the search thread (if any) and waits for it. @@ -194,6 +202,7 @@ private ref struct Found(int maxLength, int[]? lengths, uint[]? distances) public uint Distance; public int Count; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Offer(int length, int gap) { length = Math.Min(length, maxLength); @@ -210,6 +219,53 @@ public void Offer(int length, int gap) } } + /// The four-byte hash bucket of (which has four bytes after it). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int Hash4(int pos) => (int)((BinaryPrimitives.ReadUInt32LittleEndian(Buffer.AsSpan(pos)) * 2654435761u) >> _hash4Shift); + + /// + /// The tree part of a search at (which has four bytes after it): enters it in its tree + /// and offers what the walk finds. + /// + private void TreeSearch(int pos, int h4, ref Found found) + { + var stored = pos + _offset; + ref var slot4 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), h4); + var candidate = slot4; + slot4 = stored; + TreeWalk(pos, stored, candidate, ref found); + } + + /// + /// The two- and three-byte part of a search at (which has four bytes after it): enters it + /// in those tables and offers their candidates. + /// + private void NearSearch(int pos, int maxLength, ref Found found) + { + var buffer = Buffer; + var value = BinaryPrimitives.ReadUInt32LittleEndian(buffer.AsSpan(pos)); + var stored = pos + _offset; + var h2 = (int)(value & 0xFFFF); + var h3 = (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits)); + ref var slot2 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), h2); + ref var slot3 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), h3); + var candidate2 = slot2; + slot2 = stored; + var candidate3 = slot3; + slot3 = stored; + if (maxLength < 2) + return; + foreach (var near in (ReadOnlySpan)[candidate2, candidate3]) + { + if (near > 0 && stored - near < _dictionary && found.Best < maxLength) + { + var at = near - _offset; + if (buffer[at] == buffer[pos]) + found.Offer(Common(buffer, pos, at, maxLength), stored - near); + } + } + } + private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, out uint distance) { distance = 0; @@ -225,12 +281,16 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou var h2 = (int)(value & 0xFFFF); var h3 = (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits)); var h4 = (int)((value * 2654435761u) >> _hash4Shift); - var candidate2 = _head2[h2]; - _head2[h2] = stored; - var candidate3 = _head3[h3]; - _head3[h3] = stored; - var candidate = _head4[h4]; - _head4[h4] = stored; + // The hashes are below the tables' sizes by construction: 16 bits, Hash3Bits and the shift for _head4. + ref var slot2 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), h2); + ref var slot3 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), h3); + ref var slot4 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), h4); + var candidate2 = slot2; + slot2 = stored; + var candidate3 = slot3; + slot3 = stored; + var candidate = slot4; + slot4 = stored; var found = new Found(maxLength, lengths, distances); if (maxLength >= 2) @@ -288,25 +348,39 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou private void TreeWalk(int pos, int stored, int candidate, ref Found found) { var buffer = Buffer; - var links = _links; + ref var data = ref MemoryMarshal.GetArrayDataReference(buffer); + ref var links = ref MemoryMarshal.GetArrayDataReference(_links); + var mask = _mask; + var reach = _reach; + var offset = _offset; var limit = Math.Min(_nice, Filled - pos); - var leftSlot = (stored & _mask) << 1; // where the next candidate smaller than pos goes - var rightSlot = leftSlot + 1; // where the next candidate larger than pos goes - int leftLength = 0, rightLength = 0; // bytes known to match the bounds on either side + // Every candidate checked below lies within the dictionary behind pos, so it and the limit fit the buffer; + // every slot is a node's (masked) or the cursor's. The checks stay, cheap next to the walk's cache misses. + if ((uint)pos > (uint)(buffer.Length - limit) || limit < 0) + throw new ArgumentOutOfRangeException(nameof(pos)); + ref var cursor = ref Unsafe.Add(ref data, pos); + var leftSlot = (stored & mask) << 1; // where the next candidate smaller than pos goes + var rightSlot = leftSlot + 1; // where the next candidate larger than pos goes + int leftLength = 0, rightLength = 0; // bytes known to match the bounds on either side for (var depth = _depth; ; depth--) { var gap = stored - candidate; - if (candidate <= 0 || gap <= 0 || gap >= _dictionary || depth == 0) + var at = candidate - offset; + if (candidate <= 0 || gap <= 0 || gap >= reach || depth == 0 || (uint)at > (uint)pos) { - links[leftSlot] = 0; - links[rightSlot] = 0; + Unsafe.Add(ref links, leftSlot) = 0; + Unsafe.Add(ref links, rightSlot) = 0; return; } - var at = candidate - _offset; - var node = (candidate & _mask) << 1; + ref var match = ref Unsafe.Add(ref data, at); + var node = (candidate & mask) << 1; + // Both children are read before the bytes are compared: the two loads (the node's and the candidate's + // bytes, both mostly cache misses) then overlap instead of one waiting for the other. + var smaller = Unsafe.Add(ref links, node); + var larger = Unsafe.Add(ref links, node + 1); var length = Math.Min(leftLength, rightLength); // Most candidates differ at the first byte not known to match: only then is the rest compared. - if (length < limit && buffer[at + length] == buffer[pos + length]) + if (length < limit && Unsafe.Add(ref match, length) == Unsafe.Add(ref cursor, length)) { length++; if (length < limit) @@ -316,22 +390,22 @@ private void TreeWalk(int pos, int stored, int candidate, ref Found found) if (length == limit) { // Equal as far as the tree looks: the candidate's subtrees become the new root's. - links[leftSlot] = links[node]; - links[rightSlot] = links[node + 1]; + Unsafe.Add(ref links, leftSlot) = smaller; + Unsafe.Add(ref links, rightSlot) = larger; return; } - if (buffer[at + length] < buffer[pos + length]) + if (Unsafe.Add(ref match, length) < Unsafe.Add(ref cursor, length)) { - links[leftSlot] = candidate; + Unsafe.Add(ref links, leftSlot) = candidate; leftSlot = node + 1; - candidate = links[node + 1]; + candidate = larger; leftLength = length; } else { - links[rightSlot] = candidate; + Unsafe.Add(ref links, rightSlot) = candidate; rightSlot = node; - candidate = links[node]; + candidate = smaller; rightLength = length; } } @@ -378,10 +452,11 @@ private void Insert(int pos) var h4 = (int)((value * 2654435761u) >> _hash4Shift); var h3 = (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits)); var stored = p + _offset; - _head2[(int)(value & 0xFFFF)] = stored; - _head3[h3] = stored; - var candidate = _head4[h4]; - _head4[h4] = stored; + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), (int)(value & 0xFFFF)) = stored; + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), h3) = stored; + ref var slot4 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), h4); + var candidate = slot4; + slot4 = stored; if (_tree) TreeWalk(p, stored, candidate, ref found); else @@ -390,72 +465,220 @@ private void Insert(int pos) _nextInsert = pos; } - /// The search thread and the batches of match lists it hands over. + /// The search threads and the batches of match lists they hand over. private sealed class Ahead { + /// The most tree threads one match finder starts. + public const int MaxThreads = 4; + private const int BatchSize = 4096; private const int Batches = 8; + /// + /// How far apart two search threads may be: a thread takes a batch only when the one eight before it was + /// handed to the parse, after every thread finished it, so they are at most nine batches (36 Ki positions) + /// apart. + /// + public const int MaxLag = 1 << 16; + + /// One thread's tree matches for the positions of a batch it owns: per position its offset in Data, or -1. + private sealed class Part + { + public readonly int[] Offsets = new int[BatchSize]; + public int[] Data = new int[BatchSize * 4]; + } + private sealed class Batch { - public int First; + public int First = -1; // -1: free + public int Parts; // tree threads done with it + public bool Ready; public readonly int[] Offsets = new int[BatchSize]; public int[] Data = new int[BatchSize * 4]; // per position: count, then length and distance per match + public Part[] Trees = []; } - private readonly BlockingCollection _free = new(); - private readonly BlockingCollection _full = new(); + private readonly MatchFinder _mf; + private readonly int _start; + private readonly int _end; + private readonly int _threads; + private readonly Batch[] _ring = new Batch[Batches]; + private readonly object _gate = new(); private readonly CancellationTokenSource _stop; - private readonly Task _producer; + private readonly Task[] _producers; + private readonly int[] _lengths = new int[LzmaModel.MatchMaxLength + 1]; + private readonly uint[] _distances = new uint[LzmaModel.MatchMaxLength + 1]; + private int _nextMerge; // the first batch (by number) not merged yet + private int _taken; // the first batch (by number) the parse has not handed back private ExceptionDispatchInfo? _error; private Batch? _current; - public Ahead(MatchFinder mf, int start, int end, CancellationToken cancellationToken) + public Ahead(MatchFinder mf, int start, int end, int threads, CancellationToken cancellationToken) { + _mf = mf; + _start = start; + _end = end; + _threads = threads; for (var i = 0; i < Batches; i++) - _free.Add(new Batch()); + { + _ring[i] = new Batch { Trees = new Part[threads] }; + for (var k = 0; k < threads; k++) + _ring[i].Trees[k] = new Part(); + } _stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var token = _stop.Token; - _producer = Task.Factory.StartNew(() => Produce(mf, start, end, token), CancellationToken.None, - TaskCreationOptions.LongRunning, TaskScheduler.Default); + _producers = new Task[threads]; + for (var k = 0; k < threads; k++) + { + var part = k; + _producers[k] = Task.Factory.StartNew(() => Produce(part, token), CancellationToken.None, + TaskCreationOptions.LongRunning, TaskScheduler.Default); + } } - private void Produce(MatchFinder mf, int start, int end, CancellationToken token) + private int BatchCount => (_end - _start + BatchSize - 1) / BatchSize; + + /// Thread : the tree searches of its hashes, batch by batch. + private void Produce(int part, CancellationToken token) { try { + var mf = _mf; var lengths = new int[LzmaModel.MatchMaxLength + 1]; var distances = new uint[LzmaModel.MatchMaxLength + 1]; - for (var first = start; first < end; first += BatchSize) + for (var number = 0; number < BatchCount; number++) { - var batch = _free.Take(token); - batch.First = first; + var batch = _ring[number % Batches]; + lock (_gate) + { + // The slot is free once the parse handed back the batch eight before. + while (number - _taken >= Batches && _error is null) + { + token.ThrowIfCancellationRequested(); + Monitor.Wait(_gate, 100); + } + if (_error is not null) + return; + } + var first = _start + number * BatchSize; + var last = Math.Min(_end, first + BatchSize); + var tree = batch.Trees[part]; var at = 0; - var last = Math.Min(end, first + BatchSize); for (var p = first; p < last; p++) { - batch.Offsets[p - first] = at; - var count = mf.Search(p, Math.Min(LzmaModel.MatchMaxLength, end - p), lengths, distances, out _); - if (at + 1 + 2 * count > batch.Data.Length) - Array.Resize(ref batch.Data, Math.Max(batch.Data.Length * 2, at + 1 + 2 * count)); - var data = batch.Data; - data[at++] = count; - for (var k = 0; k < count; k++) + var h4 = p + 4 <= mf.Filled ? mf.Hash4(p) : -1; + if (h4 < 0 || (_threads > 1 && h4 % _threads != part)) + { + tree.Offsets[p - first] = -1; + continue; + } + var found = new Found(Math.Min(LzmaModel.MatchMaxLength, _end - p), lengths, distances); + mf.TreeSearch(p, h4, ref found); + if (at + 1 + 2 * found.Count > tree.Data.Length) + Array.Resize(ref tree.Data, Math.Max(tree.Data.Length * 2, at + 1 + 2 * found.Count)); + tree.Offsets[p - first] = at; + var data = tree.Data; + data[at++] = found.Count; + for (var k = 0; k < found.Count; k++) { data[at++] = lengths[k]; data[at++] = (int)distances[k]; } } - _full.Add(batch, token); + token.ThrowIfCancellationRequested(); + bool merge; + lock (_gate) + { + if (batch.Parts == 0) + batch.First = first; + merge = ++batch.Parts == _threads; + } + if (merge) + MergeReady(); } } catch (Exception ex) { - _error = ExceptionDispatchInfo.Capture(ex); + lock (_gate) + { + _error ??= ExceptionDispatchInfo.Capture(ex); + Monitor.PulseAll(_gate); + } + } + } + + /// Joins the batches whose trees are all done, in order (one thread at a time). + private void MergeReady() + { + lock (_mergeGate) + { + while (true) + { + Batch batch; + lock (_gate) + { + if (_nextMerge >= BatchCount) + return; + batch = _ring[_nextMerge % Batches]; + if (batch.Parts != _threads || batch.Ready) + return; + } + Merge(batch); + lock (_gate) + { + batch.Ready = true; + _nextMerge++; + Monitor.PulseAll(_gate); + } + } } - finally + } + + private readonly object _mergeGate = new(); + + /// Adds the close matches to every position of the batch and joins them with its tree's. + private void Merge(Batch batch) + { + var mf = _mf; + var buffer = mf.Buffer; + var first = batch.First; + var last = Math.Min(_end, first + BatchSize); + var at = 0; + for (var p = first; p < last; p++) { - _full.CompleteAdding(); + batch.Offsets[p - first] = at; + var maxLength = Math.Min(LzmaModel.MatchMaxLength, _end - p); + var found = new Found(maxLength, _lengths, _distances); + var count = 0; + if (p + 4 <= mf.Filled) + { + mf.NearSearch(p, maxLength, ref found); + // The tree's matches longer than the close ones, as one search offers them after those. + var owner = _threads > 1 ? mf.Hash4(p) % _threads : 0; + var tree = batch.Trees[owner]; + var from = tree.Offsets[p - first]; + var treeData = tree.Data; + var treeCount = treeData[from++]; + for (var k = 0; k < treeCount; k++, from += 2) + found.Offer(treeData[from], treeData[from + 1] + 1); + // The tree compares at most the nice length; a match that long may go on. + if (found.Best >= mf._nice && found.Best < maxLength) + { + var longer = Common(buffer, p, p - (int)found.Distance - 1, maxLength); + found.Best = longer; + _lengths[found.Count - 1] = longer; + } + count = found.Best < 2 ? 0 : found.Count; + } + if (at + 1 + 2 * count > batch.Data.Length) + Array.Resize(ref batch.Data, Math.Max(batch.Data.Length * 2, at + 1 + 2 * count)); + var data = batch.Data; + data[at++] = count; + for (var k = 0; k < count; k++) + { + data[at++] = _lengths[k]; + data[at++] = (int)_distances[k]; + } } } @@ -464,15 +687,26 @@ public int Take(int pos, int maxLength, int[] lengths, uint[] distances) { while (_current is null || pos >= _current.First + BatchSize) { - if (_current is not null) - _free.Add(_current); - _current = null; - if (!_full.TryTake(out var next, Timeout.Infinite)) + lock (_gate) { + if (_current is not null) + { + // Handed back: its slot takes a later batch. + _current.First = -1; + _current.Parts = 0; + _current.Ready = false; + _current = null; + _taken++; + Monitor.PulseAll(_gate); + } + if (_taken >= BatchCount) + throw new InvalidOperationException("a position past the input was searched"); + var batch = _ring[_taken % Batches]; + while (!batch.Ready && _error is null) + Monitor.Wait(_gate); _error?.Throw(); - throw new InvalidOperationException("a position past the input was searched"); + _current = batch; } - _current = next; } if (maxLength < 2) return 0; @@ -496,17 +730,17 @@ public int Take(int pos, int maxLength, int[] lengths, uint[] distances) public void Stop() { _stop.Cancel(); + lock (_gate) + Monitor.PulseAll(_gate); try { - _producer.Wait(); + Task.WaitAll(_producers); } catch (AggregateException) { - // The producer catches everything. + // The producers catch everything. } _stop.Dispose(); - _free.Dispose(); - _full.Dispose(); } } } diff --git a/src/Filee.Engines/Archives/Lzma/RangeEncoder.cs b/src/Filee.Engines/Archives/Lzma/RangeEncoder.cs index d943d509e..2bc6612b5 100644 --- a/src/Filee.Engines/Archives/Lzma/RangeEncoder.cs +++ b/src/Filee.Engines/Archives/Lzma/RangeEncoder.cs @@ -1,8 +1,13 @@ // The LZMA range encoder, the mirror of RangeDecoder: a 33-bit low end (the extra bit is a carry into bytes already // produced), a 32-bit range, and a byte held back plus a count of 0xFF bytes after it, because a later carry can still // change them. Output goes to a byte array that holds one LZMA2 chunk. +// +// The multi-bit codings (trees, literals, direct bits) keep the low end and the range in locals for all their bits +// and code each bit without a branch on its value (the bits of a literal or a distance are close to random): a mask +// of the bit picks the new range and what the low end gains, as in the decoder. using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Filee.Engines.Archives.Lzma; @@ -39,21 +44,83 @@ public void Flush() ShiftLow(); } - private void ShiftLow() + private void ShiftLow() => _low = ShiftLow(_low); + + /// Moves the top byte of out (held back while a carry may still reach it); returns the new low end. + [MethodImpl(MethodImplOptions.NoInlining)] + private ulong ShiftLow(ulong low) { - if ((uint)_low < 0xFF000000u || (_low >> 32) != 0) + if ((uint)low < 0xFF000000u || (low >> 32) != 0) { - var carry = (byte)(_low >> 32); + var carry = (byte)(low >> 32); var value = _cache; do { _buffer[_count++] = (byte)(value + carry); value = 0xFF; } while (--_cacheSize != 0); - _cache = (byte)((uint)_low >> 24); + _cache = (byte)((uint)low >> 24); } _cacheSize++; - _low = (_low & 0x00FFFFFF) << 8; + return (low & 0x00FFFFFF) << 8; + } + + /// One bit with the adaptive probability , on the caller's locals, without a branch on the bit. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void Bit(ref ushort probability, uint bit, ref ulong low, ref uint range) + { + uint p = probability; + var bound = (range >> RangeCoder.ProbabilityBits) * p; + var one = 0 - bit; + low += bound & one; + range = bound + ((range - 2 * bound) & one); + // p + ((2048 − p) >> 5) for a 0 and p − (p >> 5) for a 1, as in the decoder. + probability = (ushort)(p - (uint)((int)(p - (~one & (RangeCoder.ProbabilityOne - (1 << RangeCoder.MoveBits) + 1))) >> RangeCoder.MoveBits)); + if (range < RangeCoder.TopValue) + { + range <<= 8; + low = ShiftLow(low); + } + } + + /// A literal (8 bits through the tree at , most significant first). + public void EncodeLiteral(ref ushort probabilities, uint symbol) + { + var low = _low; + var range = _range; + uint m = 1; + for (var i = 7; i >= 0; i--) + { + var bit = (symbol >> i) & 1; + Bit(ref Unsafe.Add(ref probabilities, m), bit, ref low, ref range); + m = (m << 1) | bit; + } + _low = low; + _range = range; + } + + /// + /// A literal after a match: while its bits agree with 's, the probabilities picked by + /// that byte's bit (0x100 or 0x200 on) are used, after the first difference the plain ones. + /// + public void EncodeMatchedLiteral(ref ushort probabilities, uint symbol, uint matchByte) + { + var low = _low; + var range = _range; + uint m = 1; + var offset = 0x100u; // 0x100 while the bits agree, 0 after + for (var i = 7; i >= 0; i--) + { + matchByte <<= 1; + var matchBit = matchByte & 0x100 & offset; // 0x100 or 0 + var bit = (symbol >> i) & 1; + Bit(ref Unsafe.Add(ref probabilities, offset + matchBit + m), bit, ref low, ref range); + m = (m << 1) | bit; + // The bits still agree when this one equals the match byte's. + offset &= ((matchBit >> 8) ^ bit) - 1; + } + _low = low; + _range = range; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -82,42 +149,59 @@ public void EncodeBit(ref ushort probability, int bit) /// The low bits of at probability one half, most significant first. public void EncodeDirectBits(uint value, int count) { + var low = _low; + var range = _range; while (count-- > 0) { - _range >>= 1; - if (((value >> count) & 1) != 0) - _low += _range; - while (_range < RangeCoder.TopValue) + range >>= 1; + low += range & (0 - ((value >> count) & 1)); + if (range < RangeCoder.TopValue) { - _range <<= 8; - ShiftLow(); + range <<= 8; + low = ShiftLow(low); } } + _low = low; + _range = range; } /// A -bit symbol through a binary tree of probabilities, most significant bit first. public void EncodeTree(Span probabilities, int bits, uint symbol) { + if (probabilities.Length < 1 << bits) + throw new ArgumentOutOfRangeException(nameof(probabilities)); + ref var probs = ref MemoryMarshal.GetReference(probabilities); + var low = _low; + var range = _range; uint m = 1; for (var i = bits - 1; i >= 0; i--) { - var bit = (int)((symbol >> i) & 1); - EncodeBit(ref probabilities[(int)m], bit); - m = (m << 1) | (uint)bit; + var bit = (symbol >> i) & 1; + Bit(ref Unsafe.Add(ref probs, m), bit, ref low, ref range); + m = (m << 1) | bit; } + _low = low; + _range = range; } /// Like , least significant bit first. public void EncodeReverseTree(Span probabilities, int bits, uint symbol) { + if (probabilities.Length < 1 << bits) + throw new ArgumentOutOfRangeException(nameof(probabilities)); + ref var probs = ref MemoryMarshal.GetReference(probabilities); + var low = _low; + var range = _range; uint m = 1; for (var i = 0; i < bits; i++) { - var bit = (int)(symbol & 1); + var bit = symbol & 1; symbol >>= 1; - EncodeBit(ref probabilities[(int)m], bit); - m = (m << 1) | (uint)bit; + Bit(ref Unsafe.Add(ref probs, m), bit, ref low, ref range); + m = (m << 1) | bit; } + _low = low; + _range = range; } } diff --git a/src/Filee.Engines/Archives/SevenZipWriter.cs b/src/Filee.Engines/Archives/SevenZipWriter.cs index c02b0107e..e8af27427 100644 --- a/src/Filee.Engines/Archives/SevenZipWriter.cs +++ b/src/Filee.Engines/Archives/SevenZipWriter.cs @@ -80,12 +80,17 @@ public SevenZipWriter(Stream output, ArchiveLevel level, long expectedTotal = -1 } /// - /// The most data one solid block takes at : 16 × the LZMA2 dictionary (64 MiB fast, - /// 256 MiB normal, 1 GiB maximum). Beyond the dictionary a new block costs only a little ratio, and a reader that - /// wants one file decodes at most this much before it. The store level puts every file into its own folder. + /// The most data one solid block takes at : 64 MiB at the fast level, else 16 × the + /// LZMA2 dictionary (256 MiB normal, 1 GiB maximum). Beyond the dictionary a new block costs only a little ratio, + /// and a reader that wants one file decodes at most this much before it. The store level puts every file into its + /// own folder. /// - public static long SolidBlockSize(ArchiveLevel level) => - level == ArchiveLevel.Store ? 0 : 16L * Lzma2Settings.For(level).Dictionary; + public static long SolidBlockSize(ArchiveLevel level) => level switch + { + ArchiveLevel.Store => 0, + ArchiveLevel.Fast => 64L << 20, + _ => 16L * Lzma2Settings.For(level).Dictionary, + }; /// Entries added so far. public int Count => _entries.Count; diff --git a/tests/Filee.Engines.Tests/ArchiveThreadTests.cs b/tests/Filee.Engines.Tests/ArchiveThreadTests.cs index d75db106b..ae44cbac8 100644 --- a/tests/Filee.Engines.Tests/ArchiveThreadTests.cs +++ b/tests/Filee.Engines.Tests/ArchiveThreadTests.cs @@ -1,7 +1,7 @@ -// Archives coded and decoded on several threads (Lzma2Writer blocks, Lzma2Parallel, XzReader blocks, Bzip2Decoder's -// stream): the same bytes as on one thread, the same errors at the same place, 7-Zip reads what Filee writes (skipped -// when it is missing), and cancellation stops the workers. Block sizes are made small so that little data gives many -// blocks. +// Archives coded and decoded on several threads (Lzma2Writer blocks, the match finder's search threads, Lzma2Parallel, +// XzReader blocks, Bzip2Decoder's stream, ReadAheadStream): the same bytes as on one thread, the same errors at the +// same place, 7-Zip reads what Filee writes (skipped when it is missing), and cancellation stops the workers. Block +// sizes are made small so that little data gives many blocks. using Filee.Core.Presets; using Filee.Engines.Archives; @@ -380,4 +380,211 @@ public void A_cancelled_bzip2_stream_stops() } }); } + // ───────── Match finder threads ───────── + + [Fact] + public void The_match_finder_split_over_threads_gives_the_same_bytes() + { + // A dictionary smaller than the block, so tree slots are taken over a dictionary later (by another thread). + var data = Bzip2WriterTests.Mixed(1_400_000); + var settings = new Lzma2Settings(1 << 18, new LzmaEncoderSettings(Depth: 32, Nice: 32, Optimal: true, Tree: true), Store: false); + byte[] Coded(int searchThreads) + { + Lzma2Writer.TestThreads.Value = 1; + Lzma2Writer.TestBlockSize.Value = 1 << 20; + Lzma2Writer.TestSearchAhead.Value = searchThreads > 0; + Lzma2Writer.TestSearchThreads.Value = searchThreads; + try + { + using var output = new MemoryStream(); + var writer = new Lzma2Writer(output, settings, data.Length, Ct); + for (var at = 0; at < data.Length; at += 65_536) + writer.Write(data.AsSpan(at, Math.Min(65_536, data.Length - at))); + writer.Finish(); + return output.ToArray(); + } + finally + { + Lzma2Writer.TestThreads.Value = null; + Lzma2Writer.TestBlockSize.Value = null; + Lzma2Writer.TestSearchAhead.Value = null; + Lzma2Writer.TestSearchThreads.Value = null; + } + } + var alone = Coded(0); + foreach (var threads in new[] { 1, 2, 4 }) + Assert.Equal(alone, Coded(threads)); + var reader = new Lzma2Reader(new ByteInput(new MemoryStream(alone)), 1 << 18, data.Length, Ct); + Assert.Equal(data, ReadAll(new Lzma2Stream(reader))); + } + + [Fact] + public void A_distance_past_the_dictionary_is_damage_even_within_the_windows_guard() + { + // A 100-byte repeat from 4,100 bytes back, once 4,200 bytes are out: fine with an 8 KiB dictionary, beyond a + // 4 KiB one (the decoder's window has a few bytes more than the dictionary; they don't count). + var random = Noise(4_200, 9); + byte[] data = [.. random, .. random.AsSpan(100, 100), .. Noise(300, 10)]; + var settings = new Lzma2Settings(1 << 13, new LzmaEncoderSettings(Depth: 32, Nice: 32, Optimal: true, Tree: true), Store: false); + using var output = new MemoryStream(); + var writer = new Lzma2Writer(output, settings, data.Length, Ct); + writer.Write(data); + writer.Finish(); + var coded = output.ToArray(); + byte[] Decode(uint dictionary) => + ReadAll(new Lzma2Stream(new Lzma2Reader(new ByteInput(new MemoryStream(coded)), dictionary, data.Length, Ct))); + Assert.Equal(data, Decode(1 << 13)); + Assert.Contains("before the start of the window", Assert.Throws(() => Decode(1 << 12)).Message); + } + + // ───────── Read-ahead ───────── + + /// Counts up byte by byte, a thousand at a time, and fails past (-1: never ends). + private sealed class Counting(long failAt) : Stream + { + private long _position; + private int _reads; + + public int Reads => Volatile.Read(ref _reads); + + public bool Disposed { get; private set; } + + public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + Interlocked.Increment(ref _reads); + if (failAt >= 0 && _position >= failAt) + throw new InvalidDataException("the test data is damaged here"); + var n = Math.Min(buffer.Length, 1000); + for (var i = 0; i < n; i++) + buffer[i] = (byte)(_position + i); + _position += n; + return n; + } + + protected override void Dispose(bool disposing) + { + Disposed = true; + base.Dispose(disposing); + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _position; set => throw new NotSupportedException(); } + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } + + private static Stream Ahead(Stream inner, CancellationToken cancellationToken) + { + ReadAheadStream.TestMode.Value = true; + try + { + var stream = ReadAheadStream.Wrap(inner, 10, cancellationToken); + Assert.NotSame(inner, stream); + return stream; + } + finally + { + ReadAheadStream.TestMode.Value = null; + } + } + + [Fact] + public void Read_ahead_hands_out_every_byte_then_the_decoders_error() + { + var inner = new Counting(failAt: 3_456_000); + using var stream = Ahead(inner, Ct); + var buffer = new byte[70_001]; + var random = new Random(3); + long total = 0; + var error = Assert.Throws(() => + { + int read; + while ((read = stream.Read(buffer, 0, random.Next(1, buffer.Length))) > 0) + { + for (var i = 0; i < read; i++) + Assert.Equal((byte)(total + i), buffer[i]); + total += read; + } + }); + Assert.Equal("the test data is damaged here", error.Message); + // Everything the decoder gave before it failed, nothing more. + Assert.Equal(inner.Position, total); + Assert.True(total >= 3_456_000); + } + + [Fact] + public void Read_ahead_ends_where_the_decoder_ends_and_stops_with_the_stream() + { + var data = LzmaTests.Prose(2_500_000); + using (var stream = Ahead(new MemoryStream(data), Ct)) + Assert.Equal(data, ReadAll(stream)); + + // Disposed half way: the worker stops, the decoder is disposed and read no more. + var inner = new Counting(failAt: -1); + var ahead = Ahead(inner, Ct); + var buffer = new byte[100_000]; + ahead.ReadExactly(buffer); + ahead.Dispose(); + Assert.True(inner.Disposed); + var reads = inner.Reads; + Thread.Sleep(50); + Assert.Equal(reads, inner.Reads); + } + + [Fact] + public void A_cancelled_read_ahead_stops() + { + using var cancel = new CancellationTokenSource(); + using var stream = Ahead(new Counting(failAt: -1), cancel.Token); + var buffer = new byte[1 << 16]; + Assert.True(stream.Read(buffer) > 0); + cancel.Cancel(); + Assert.ThrowsAny(() => + { + while (stream.Read(buffer) > 0) + { + } + }); + } + + [Fact] + public async Task BCJ_and_BCJ2_folders_unpack_alike_with_their_coders_on_threads_or_not() + { + Assert.SkipUnless(SevenZipExe is not null, "7-Zip makes the archives this test reads (optional engine, not installed)"); + var dir = fx.NewFolder(); + var source = Directory.CreateDirectory(Path.Combine(dir, "code")).FullName; + for (var i = 0; i < 4; i++) + File.WriteAllBytes(Path.Combine(source, $"part{i}.bin"), SevenZipReaderTests.X86Like(400_000 + i * 54_321, i)); + foreach (var filter in new[] { "BCJ", "BCJ2" }) + { + var archive = Path.Combine(dir, filter + ".7z"); + await Run7zAsync("a", "-t7z", "-mf=" + filter, "-mx5", archive, source); + foreach (var ahead in new[] { false, true }) + { + var opened = ReadAheadStream.Opened; + ReadAheadStream.TestMode.Value = ahead; + var target = fx.NewFolder(); + try + { + using var file = File.OpenRead(archive); + SevenZipReader.Extract(file, Path.GetFileName(archive), target, Ct); + } + finally + { + ReadAheadStream.TestMode.Value = null; + } + if (ahead) + Assert.True(ReadAheadStream.Opened > opened, "no coder was read ahead"); + foreach (var part in Directory.GetFiles(source)) + Assert.Equal(File.ReadAllBytes(part), File.ReadAllBytes(Path.Combine(target, "code", Path.GetFileName(part)))); + } + } + } }