diff --git a/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs b/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs
index 6f6a3a89f..443d6516d 100644
--- a/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs
+++ b/src/Filee.Engines/Archives/Bzip2BlockEncoder.cs
@@ -6,6 +6,10 @@
// bits or less, which every bzip2 decoder takes. The result is a bit string (Bzip2Bits) that starts with the block
// magic; blocks are not byte aligned, so the writer joins them bit by bit.
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
+
namespace Filee.Engines.Archives;
/// Encodes bzip2 blocks; reusable, one block at a time.
@@ -28,6 +32,10 @@ internal sealed class Bzip2BlockEncoder
private readonly uint[][] _codes = CreateTables();
private readonly int[][] _tableFrequencies = CreateTables();
+ // Per symbol, its code length in each table side by side (eight lanes, the unused ones too dear to be chosen), so
+ // a group's cost in every table is one vector sum.
+ private readonly Vector128[] _packedLengths = new Vector128[MaxAlphabet];
+
/// The last block's bits, from its magic to its last Huffman code.
public Bzip2Bits Output { get; } = new();
@@ -231,28 +239,41 @@ private void ChooseTables(int count, int alphabet, int tables, int selectorCount
remaining -= sum;
}
- Span cost = stackalloc int[MaxTables];
+ Span lanes = stackalloc ushort[Vector128.Count];
var symbols = _symbols;
+ if (count > symbols.Length)
+ throw new ArgumentOutOfRangeException(nameof(count));
+ ref var symbol0 = ref MemoryMarshal.GetArrayDataReference(symbols);
+ ref var packed = ref MemoryMarshal.GetArrayDataReference(_packedLengths);
for (var pass = 0; pass < passes; pass++)
{
for (var t = 0; t < tables; t++)
Array.Clear(_tableFrequencies[t], 0, alphabet);
+ for (var v = 0; v < alphabet; v++)
+ {
+ // A group holds at most 50 symbols of at most 17 bits: 255 a lane never wins and never overflows.
+ lanes.Fill(255);
+ for (var t = 0; t < tables; t++)
+ lanes[t] = _lengths[t][v];
+ _packedLengths[v] = Vector128.Create(lanes);
+ }
for (var g = 0; g < selectorCount; g++)
{
var start = g * GroupSize;
var end = Math.Min(count, start + GroupSize);
- cost.Clear();
+ var cost = Vector128.Zero;
for (var i = start; i < end; i++)
- {
- var symbol = symbols[i];
- for (var t = 0; t < tables; t++)
- cost[t] += _lengths[t][symbol];
- }
+ cost += Unsafe.Add(ref packed, Unsafe.Add(ref symbol0, i)); // symbols are below the alphabet
var best = 0;
+ var bestCost = cost.GetElement(0);
for (var t = 1; t < tables; t++)
{
- if (cost[t] < cost[best])
+ var c = cost.GetElement(t);
+ if (c < bestCost)
+ {
best = t;
+ bestCost = c;
+ }
}
_selectors[g] = (byte)best;
var chosen = _tableFrequencies[best];
diff --git a/src/Filee.Engines/Archives/Bzip2Writer.cs b/src/Filee.Engines/Archives/Bzip2Writer.cs
index 3da25d791..baf75e8af 100644
--- a/src/Filee.Engines/Archives/Bzip2Writer.cs
+++ b/src/Filee.Engines/Archives/Bzip2Writer.cs
@@ -1,7 +1,7 @@
// Writes .bz2 files with Filee's own encoder, from the bzip2 format: the "BZh9" stream header, blocks of at most
// 900,000 bytes, and the end marker with the CRC of all block CRCs. The input goes through bzip2's first run-length
// step here (runs of 4 to 255 equal bytes become 4 bytes and a count) while the block CRC is taken over the bytes as
-// they came; full blocks are encoded (Bzip2BlockEncoder) on up to four threads at once, and written in order. Memory
+// they came; full blocks are encoded (Bzip2BlockEncoder) on up to eight threads at once, and written in order. Memory
// stays bounded by the number of blocks in flight (about 20 MB each while sorting). It is a write-only stream, so a TAR
// can be compressed while it is written (.tar.bz2) without a temporary file.
@@ -45,8 +45,8 @@ private sealed class Slot
public Task? Work { get; set; }
}
- /// Blocks encoded at once by default: a few cores, leaving the rest to the machine.
- internal static int DefaultThreads => Math.Clamp(Environment.ProcessorCount / 2, 1, 4);
+ /// Blocks encoded at once by default: one per processor, at most eight (about 20 MB each while sorting).
+ internal static int DefaultThreads => Math.Clamp(Environment.ProcessorCount, 1, 8);
/// Blocks encoded at once (1 encodes on the writing thread).
public Bzip2Writer(Stream output, ArchiveLevel level, CancellationToken cancellationToken = default, bool leaveOpen = false,
diff --git a/src/Filee.Engines/Archives/CompressedFile.cs b/src/Filee.Engines/Archives/CompressedFile.cs
index 95319fe87..e11345e2f 100644
--- a/src/Filee.Engines/Archives/CompressedFile.cs
+++ b/src/Filee.Engines/Archives/CompressedFile.cs
@@ -40,7 +40,8 @@ public static void Decompress(string archive, string format, string outputFile,
try
{
long declared = -1;
- using (var decoder = Open(format, input, name, ref declared))
+ // Decoded on a thread of its own while this one writes the file.
+ using (var decoder = ReadAheadStream.Wrap(Open(format, input, name, ref declared), declared, cancellationToken))
using (var output = new FileStream(outputFile, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16))
{
// Past the room on the drive is "too large"; past a header's size (within that room) is damage.
diff --git a/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs b/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs
index 63c1477f8..51bc4ce7f 100644
--- a/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs
+++ b/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs
@@ -12,7 +12,9 @@
// What a segment decodes to is held in memory until it is read, so the threads and the size of a segment are
// bounded: at most one segment per processor, all of them in an eighth of the memory, none past 512 MiB. A segment
// that fails on its thread is decoded again, from its start, on the reader's thread as it is read: so damaged data
-// gives the very bytes and the very error, at the same place, as the one-thread decoder.
+// gives the very bytes and the very error, at the same place, as the one-thread decoder. Workers hand out what they
+// decoded as they go (Decoded.Publish), so the segment being read streams out while the next ones decode; after a
+// failure the reader's own decoder skips what was handed out already.
using System.Runtime.ExceptionServices;
@@ -158,21 +160,70 @@ public static int Workers(long unpacked, long packed, uint dictionary)
return segments;
}
- /// What a worker decoded: the output, how much of it is good, and the error that stopped it, if any.
+ ///
+ /// What a worker decodes: the output, how much of it is good, and the error that stopped it, if any. The worker
+ /// publishes the output as it goes, so the reader hands out the start of a segment while the rest is decoded.
+ ///
internal sealed class Decoded(byte[] output)
{
+ /// The worker publishes after every this many bytes.
+ public const int Step = 1 << 20;
+
+ private readonly object _gate = new();
+ private int _ready;
+ private bool _done;
+
public byte[] Output { get; } = output;
+
+ /// Bytes the worker decoded so far (its own count; readers go by ).
public int Count { get; set; }
+
public ExceptionDispatchInfo? Error { get; set; }
+
+ /// Called by the worker: the first bytes of the output are final.
+ public void Publish()
+ {
+ lock (_gate)
+ {
+ _ready = Count;
+ Monitor.PulseAll(_gate);
+ }
+ }
+
+ /// Called by the worker when it stops, with set if it failed.
+ public void Finish()
+ {
+ lock (_gate)
+ {
+ if (Error is null)
+ _ready = Count;
+ _done = true;
+ Monitor.PulseAll(_gate);
+ }
+ }
+
+ ///
+ /// Waits until more than bytes are published or the worker stopped; returns the
+ /// bytes published (all that will be when ).
+ ///
+ public int WaitBeyond(int position, out bool done)
+ {
+ lock (_gate)
+ {
+ while (_ready <= position && !_done)
+ Monitor.Wait(_gate);
+ done = _done;
+ return _ready;
+ }
+ }
}
///
- /// Decodes one segment: bytes of chunks followed by the end byte (the caller puts it at
- /// packed[length]), expected to give bytes.
+ /// Decodes one segment into : bytes of chunks followed by the
+ /// end byte (the caller puts it at packed[length]), expected to give bytes.
///
- public static Decoded DecodeSegment(byte[] packed, int length, uint dictionary, int unpacked, CancellationToken cancellationToken)
+ public static void DecodeSegment(Decoded result, byte[] packed, int length, uint dictionary, int unpacked, CancellationToken cancellationToken)
{
- var result = new Decoded(GC.AllocateUninitializedArray(unpacked));
try
{
var reader = new Lzma2Reader(new ByteInput(new MemoryStream(packed, 0, length + 1, writable: false)), dictionary, unpacked,
@@ -180,17 +231,40 @@ public static Decoded DecodeSegment(byte[] packed, int length, uint dictionary,
var output = result.Output;
while (result.Count < unpacked)
{
- var read = reader.Read(output.AsSpan(result.Count));
+ var read = reader.Read(output.AsSpan(result.Count, Math.Min(Decoded.Step, unpacked - result.Count)));
if (read == 0)
throw new InvalidDataException("the LZMA2 chunk is damaged");
result.Count += read;
+ result.Publish();
}
}
catch (Exception ex)
{
result.Error = ExceptionDispatchInfo.Capture(ex);
}
- return result;
+ finally
+ {
+ result.Finish();
+ }
+ }
+
+ ///
+ /// Reads and drops bytes of : data decoded again after a worker
+ /// failed, up to where the reader had got. gets them on the way.
+ ///
+ public static void Skip(Lzma2Reader reader, long count, Action>? seen = null)
+ {
+ if (count <= 0)
+ return;
+ var scratch = new byte[Math.Min(count, 1 << 16)];
+ while (count > 0)
+ {
+ var read = reader.Read(scratch.AsSpan(0, (int)Math.Min(count, scratch.Length)));
+ if (read == 0)
+ return;
+ seen?.Invoke(scratch.AsSpan(0, read));
+ count -= read;
+ }
}
/// The segments of a 7z LZMA2 coder, decoded ahead on worker threads and read in order.
@@ -201,10 +275,11 @@ private sealed class SegmentStream : Stream
private readonly uint _dictionary;
private readonly int _workers;
private readonly CancellationTokenSource _cancel;
- private readonly Queue<(Segment Segment, Task Work)> _ahead = new();
+ private readonly Queue<(Segment Segment, Decoded Work)> _ahead = new();
private int _next;
- private Decoded? _current;
+ private (Segment Segment, Decoded Work)? _current;
private int _position;
+ private int _ready;
private Lzma2Reader? _replay; // a failed segment, decoded again here
public SegmentStream(PackedWindow packed, List segments, uint dictionary, int workers, CancellationToken cancellationToken)
@@ -234,29 +309,34 @@ public override int Read(Span buffer)
}
if (_current is { } current)
{
- if (_position < current.Count)
+ if (_position < _ready)
{
- var count = Math.Min(buffer.Length, current.Count - _position);
- current.Output.AsSpan(_position, count).CopyTo(buffer);
+ var count = Math.Min(buffer.Length, _ready - _position);
+ current.Work.Output.AsSpan(_position, count).CopyTo(buffer);
_position += count;
return count;
}
+ _ready = current.Work.WaitBeyond(_position, out var done);
+ if (_position < _ready || !done)
+ continue;
_current = null;
+ if (current.Work.Error is { } error)
+ {
+ // Decoded again here, as the one-thread reader reads it, from where the worker's output ran out.
+ if (error.SourceException is OperationCanceledException)
+ error.Throw();
+ _replay = Open(current.Segment, _cancel.Token);
+ Skip(_replay, _position);
+ }
+ continue;
}
_cancel.Token.ThrowIfCancellationRequested();
Start();
if (!_ahead.TryDequeue(out var next))
return 0;
- var decoded = next.Work.GetAwaiter().GetResult();
- if (decoded.Error is { } error)
- {
- if (error.SourceException is OperationCanceledException)
- error.Throw();
- _replay = Open(next.Segment, _cancel.Token);
- continue;
- }
- _current = decoded;
+ _current = next;
_position = 0;
+ _ready = 0;
}
}
@@ -269,7 +349,9 @@ private void Start()
var packed = ReadPacked(segment);
var dictionary = _dictionary;
var token = _cancel.Token;
- _ahead.Enqueue((segment, Task.Run(() => DecodeSegment(packed, segment.Length, dictionary, segment.Unpacked, token), CancellationToken.None)));
+ var decoded = new Decoded(GC.AllocateUninitializedArray(segment.Unpacked));
+ _ = Task.Run(() => DecodeSegment(decoded, packed, segment.Length, dictionary, segment.Unpacked, token), CancellationToken.None);
+ _ahead.Enqueue((segment, decoded));
}
}
@@ -321,7 +403,10 @@ public int ReadAt(long at, Span target)
return 0;
if (target.Length > length - at)
target = target[..(int)(length - at)];
- shared.Position = offset + at;
- return shared.ReadAtLeast(target, target.Length, throwOnEndOfStream: false);
+ lock (shared) // as SharedWindowStream: other coders of the folder may read on other threads
+ {
+ shared.Position = offset + at;
+ return shared.ReadAtLeast(target, target.Length, throwOnEndOfStream: false);
+ }
}
}
diff --git a/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs b/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs
index d919df87e..7cb81387f 100644
--- a/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs
+++ b/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs
@@ -8,7 +8,7 @@
// A block's first chunk resets the dictionary and sets the properties; after a first stored chunk (0x01) the decoder
// needs the properties again, so the next LZMA chunk carries them (0xC0).
// With processors to spare (fewer blocks than half of them), each block's binary-tree match finder searches ahead on
-// a thread of its own (MatchFinder.StartAhead), which gives the same bytes too.
+// threads of its own (MatchFinder.StartAhead, the trees split between them by hash), which gives the same bytes too.
using System.Numerics;
using Filee.Core.Presets;
@@ -18,9 +18,11 @@ namespace Filee.Engines.Archives.Lzma;
/// Dictionary and search effort for one .
internal readonly record struct Lzma2Settings(int Dictionary, LzmaEncoderSettings Encoder, bool Store)
{
+
///
/// The settings for , with the dictionary cut to the input when its size is known. Memory
- /// for one block's encoder is about 8 × the dictionary with the hash chain (fast: 4 MiB → ~32 MB) and 10–11 × with
+ /// for one block's encoder is about 8 × the dictionary with the hash chain (fast: 256 KiB → ~2 MB; small, so the
+ /// tables mostly stay in the processor's cache, and the lazy parse makes up the ratio) and 10–11 × with
/// the binary tree (normal: 16 MiB → ~180 MB, maximum: 64 MiB → ~660 MB, as xz -9), plus the block itself while it
/// is coded (); unpacking needs only the dictionary.
///
@@ -29,7 +31,7 @@ public static Lzma2Settings For(ArchiveLevel level, long expectedSize = -1)
var settings = level switch
{
ArchiveLevel.Store => new Lzma2Settings(1 << 12, default, Store: true),
- ArchiveLevel.Fast => new Lzma2Settings(1 << 22, new LzmaEncoderSettings(Depth: 8, Nice: 32, Optimal: false, Tree: false), false),
+ ArchiveLevel.Fast => new Lzma2Settings(1 << 18, new LzmaEncoderSettings(Depth: 8, Nice: 32, Optimal: false, Tree: false), false),
ArchiveLevel.Maximum => new Lzma2Settings(1 << 26, new LzmaEncoderSettings(Depth: 64, Nice: 64, Optimal: true, Tree: true), false),
_ => new Lzma2Settings(1 << 24, new LzmaEncoderSettings(Depth: 32, Nice: 32, Optimal: true, Tree: true), false),
};
@@ -43,8 +45,8 @@ public static Lzma2Settings For(ArchiveLevel level, long expectedSize = -1)
///
/// The input coded as one block, from one dictionary reset to the next, at least 1 MiB and at most 256 MiB: one
- /// dictionary at the fast level (many blocks for many threads; about 0.2 % larger output), two at normal (about
- /// 0.15 %), four at maximum. Matches never reach into the block before, which costs that little ratio.
+ /// dictionary at the fast level (that is 1 MiB: many blocks for many threads), two at normal (about 0.15 % larger
+ /// output), four at maximum. Matches never reach into the block before, which costs that little ratio.
///
public int BlockSize => (int)Math.Clamp((long)Dictionary * (!Encoder.Optimal ? 1 : Encoder.Depth >= 64 ? 4 : 2), 1 << 20, 1 << 28);
}
@@ -60,6 +62,9 @@ internal sealed class Lzma2Writer
/// Tests turn the match finder's own thread on or off (normally on with processors to spare).
internal static readonly AsyncLocal TestSearchAhead = new();
+ /// Tests set how many threads the match finder of each block searches on (normally from the processors).
+ internal static readonly AsyncLocal TestSearchThreads = new();
+
private readonly Stream _output;
private readonly Lzma2Settings _settings;
private readonly Action? _onBlock;
@@ -67,7 +72,7 @@ internal sealed class Lzma2Writer
private readonly int _blockSize;
private readonly int _firstCapacity;
private readonly int _threads;
- private readonly bool _searchAhead;
+ private readonly int _searchThreads; // per block: 0 for none, else the match finder's tree threads
private readonly bool _check;
private readonly Stack _free = new();
private readonly Queue<(Lzma2BlockEncoder Encoder, Task Work)> _pending = new();
@@ -96,8 +101,11 @@ public Lzma2Writer(Stream output, Lzma2Settings settings, long expectedSize, Can
? (int)Math.Min(_blockSize, Math.Max(expectedSize, 1) + LzmaModel.MatchMaxLength + 4)
: _blockSize;
_threads = TestThreads.Value ?? Threads(settings, _blockSize, expectedSize);
- _searchAhead = settings.Encoder is { Tree: true, Optimal: true }
- && (TestSearchAhead.Value ?? 2 * _threads <= Environment.ProcessorCount);
+ // Processors left over by the blocks go to their match finders: the trees are split between threads by hash.
+ var searchAhead = settings.Encoder is { Tree: true, Optimal: true }
+ && (TestSearchAhead.Value ?? 2 * _threads <= Environment.ProcessorCount);
+ _searchThreads = !searchAhead ? 0
+ : TestSearchThreads.Value ?? Math.Clamp(Environment.ProcessorCount / _threads - 1, 1, MatchFinder.MaxSearchThreads);
}
/// The dictionary size the decoder needs (for the filter properties).
@@ -187,12 +195,12 @@ private void Dispatch(Lzma2BlockEncoder encoder)
_cancellationToken.ThrowIfCancellationRequested();
if (_threads == 1)
{
- encoder.Encode(_cancellationToken, _searchAhead);
+ encoder.Encode(_cancellationToken, _searchThreads);
Emit(encoder);
_free.Push(encoder);
return;
}
- var (token, ahead) = (_cancellationToken, _searchAhead);
+ var (token, ahead) = (_cancellationToken, _searchThreads);
_pending.Enqueue((encoder, Task.Run(() => encoder.Encode(token, ahead), CancellationToken.None)));
}
@@ -311,8 +319,8 @@ public void Reset()
}
/// Codes the whole block.
- /// Lets the match finder search on a thread of its own (binary tree, large blocks).
- public void Encode(CancellationToken cancellationToken, bool searchAhead = false)
+ /// Lets the match finder search ahead on this many threads of its own (binary tree, large blocks; 0 for none).
+ public void Encode(CancellationToken cancellationToken, int searchThreads = 0)
{
_first = true;
_needProperties = false;
@@ -325,8 +333,8 @@ public void Encode(CancellationToken cancellationToken, bool searchAhead = false
WriteStored(_stored.AsSpan(0, _storedCount));
return;
}
- if (searchAhead && _settings.Encoder.Tree && _mf.Filled >= 1 << 18)
- _mf.StartAhead(cancellationToken);
+ if (searchThreads > 0 && _settings.Encoder.Tree && _mf.Filled >= 1 << 18)
+ _mf.StartAhead(cancellationToken, searchThreads);
try
{
while (_mf.Position < _mf.Filled)
diff --git a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs
index 8937503c5..7066a4e0f 100644
--- a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs
+++ b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs
@@ -8,6 +8,7 @@
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
namespace Filee.Engines.Archives.Lzma;
@@ -190,6 +191,16 @@ internal sealed class LzmaDecoder
private const int FirstWindow = 1 << 16;
+ ///
+ /// Bytes the full window has beyond the dictionary. A match copy may write up to − 1
+ /// bytes past its end (whole chunks are quicker than the exact tail); with this much room, the bytes it overwrites
+ /// lie farther back than any distance can reach before they are written again.
+ ///
+ private const int Guard = 32;
+
+ /// Match copies move this many bytes at a time where they may (see ).
+ private const int CopyChunk = 16;
+
// The probabilities, in one array: the single-bit decisions, the slot and distance trees, the two length models
// and, last, the literal coders (0x300 per context, 2^(lc + lp) contexts).
private const int States = LzmaModel.States;
@@ -225,6 +236,10 @@ internal sealed class LzmaDecoder
private int _pending; // bytes of the current match not yet copied out
private ushort[] _probabilities = [];
+
+ // Copies MatchRun decoded but has not done yet: target position, distance and length, three ints each.
+ private const int MaxDeferred = 32;
+ private readonly int[] _deferred = new int[MaxDeferred * 3];
private int _runStop; // where the current Run stops
private int _runTotalBase; // pos + this has the low bits of the total
private int _lc;
@@ -426,9 +441,9 @@ private int LiteralRun(RangeDecoder rc) where TInput : struct, IInputMod
{
uint p0 = Unsafe.Add(ref literal, symbol * 2);
uint p1 = Unsafe.Add(ref literal, symbol * 2 + 1);
- var one = BitMask(p, ref Unsafe.Add(ref literal, symbol), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
- p = p0 ^ ((p0 ^ p1) & one);
- symbol = (symbol << 1) - one;
+ var zero = ZeroMask(p, ref Unsafe.Add(ref literal, symbol), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ p = p1 ^ ((p0 ^ p1) & zero);
+ symbol = (symbol << 1) + 1 + zero;
}
while (symbol < 0x100);
}
@@ -451,11 +466,11 @@ private int LiteralRun(RangeDecoder rc) where TInput : struct, IInputMod
var index1 = offset1 + (after & offset1) + symbol * 2 + 1;
uint p0 = Unsafe.Add(ref literal, index0);
uint p1 = Unsafe.Add(ref literal, index1);
- var one = BitMask(p, ref Unsafe.Add(ref literal, index), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
- p = p0 ^ ((p0 ^ p1) & one);
- index = index0 ^ ((index0 ^ index1) & one);
- offset = offset0 ^ ((offset0 ^ offset1) & one);
- symbol = (symbol << 1) - one;
+ var zero = ZeroMask(p, ref Unsafe.Add(ref literal, index), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ p = p1 ^ ((p0 ^ p1) & zero);
+ index = index1 ^ ((index0 ^ index1) & zero);
+ offset = offset1 ^ ((offset0 ^ offset1) & zero);
+ symbol = (symbol << 1) + 1 + zero;
match = after;
}
while (symbol < 0x100);
@@ -500,6 +515,8 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput
ref var input = ref MemoryMarshal.GetArrayDataReference(rc.Buffer);
var inEnd = rc.End;
rc.Take(out var range, out var code, out var inPos);
+ ref var deferred = ref MemoryMarshal.GetArrayDataReference(_deferred);
+ var copies = 0;
int next;
while (true)
{
@@ -549,28 +566,37 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput
lengthModel = MatchLengthAt;
}
+ int copyLength;
if (shortRep)
{
state = state < LzmaModel.LiteralStates ? 9 : 11;
- window[pos] = window[ByteAt(pos, rep0, window.Length)];
- pos++;
+ copyLength = 1;
}
else
{
- // The length, from the match or the repeat length model.
+ // The length, from the match or the repeat length model: one of three trees (one loop for all three
+ // keeps the method small enough for the coder to stay in registers).
ref var model = ref Unsafe.Add(ref probs, lengthModel);
+ int tree, bits;
if (Bit(ref model, ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0)
{
- length = Tree(ref Unsafe.Add(ref model, LowAt + (posState << LengthModel.LowBits)), LengthModel.LowBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ tree = LowAt + (posState << LengthModel.LowBits);
+ bits = LengthModel.LowBits;
+ length = 0;
}
else if (Bit(ref Unsafe.Add(ref model, Choice2At), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0)
{
- length = LengthModel.LowSymbols + Tree(ref Unsafe.Add(ref model, MidAt + (posState << LengthModel.MidBits)), LengthModel.MidBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ tree = MidAt + (posState << LengthModel.MidBits);
+ bits = LengthModel.MidBits;
+ length = LengthModel.LowSymbols;
}
else
{
- length = LengthModel.LowSymbols + LengthModel.MidSymbols + Tree(ref Unsafe.Add(ref model, HighAt), LengthModel.HighBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ tree = HighAt;
+ bits = LengthModel.HighBits;
+ length = LengthModel.LowSymbols + LengthModel.MidSymbols;
}
+ length += Tree(ref Unsafe.Add(ref model, tree), bits, ref range, ref code, ref input, ref inPos, ref inEnd, rc);
if (lengthModel == RepLengthAt)
{
@@ -588,12 +614,24 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput
break;
}
}
- if (rep0 >= (_full ? (uint)window.Length : (uint)pos))
+ if (rep0 >= (_full ? _dictionary : Math.Min((uint)pos, _dictionary)))
throw new InvalidDataException("the LZMA data refers to bytes before the start of the window");
var matchLength = (int)length + LzmaModel.MatchMinLength;
- var fits = Math.Min(matchLength, stop - pos);
- _pending = matchLength - fits;
- pos = Copy(window, pos, (int)rep0 + 1, fits);
+ copyLength = Math.Min(matchLength, stop - pos);
+ _pending = matchLength - copyLength;
+ }
+
+ // The bytes are copied later, several matches at a time: the copies' sources are far apart in the window
+ // and mostly not in the cache, and done back to back their loads overlap. Nothing before the copies reads
+ // the window (only literals do), so the bytes come out the same.
+ Unsafe.Add(ref deferred, copies * 3) = pos;
+ Unsafe.Add(ref deferred, copies * 3 + 1) = (int)rep0 + 1;
+ Unsafe.Add(ref deferred, copies * 3 + 2) = copyLength;
+ pos += copyLength;
+ if (++copies == MaxDeferred)
+ {
+ CopyDeferred(window, copies);
+ copies = 0;
}
if (pos >= stop || (!TInput.Checked && inPos > inEnd - InputMargin))
@@ -608,6 +646,7 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput
break;
}
}
+ CopyDeferred(window, copies);
_pos = pos;
_state = state;
_rep0 = rep0;
@@ -618,6 +657,14 @@ private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput
return next;
}
+ /// Does the first copies MatchRun put off, in order.
+ private void CopyDeferred(byte[] window, int count)
+ {
+ ref var deferred = ref MemoryMarshal.GetArrayDataReference(_deferred);
+ for (var i = 0; i < count; i++)
+ Copy(window, Unsafe.Add(ref deferred, i * 3), Unsafe.Add(ref deferred, i * 3 + 1), Unsafe.Add(ref deferred, i * 3 + 2));
+ }
+
/// The index of the byte + 1 before .
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static long ByteAt(int pos, uint distance, int windowLength)
@@ -639,6 +686,20 @@ private static int Copy(byte[] window, int pos, int distance, int count)
if (source < 0)
source += windowLength;
ref var win = ref MemoryMarshal.GetArrayDataReference(window);
+ // Most matches: the source at least a chunk behind (or, wrapped, far ahead) and room for whole chunks on both
+ // sides. Chunks read only bytes written before them, and the few bytes written past the end are written
+ // again before anything reads them (Guard).
+ if (distance >= CopyChunk && source + count <= windowLength - CopyChunk && pos + count <= windowLength - CopyChunk)
+ {
+ var i = 0;
+ do
+ {
+ Vector128.LoadUnsafe(ref win, (nuint)(source + i)).StoreUnsafe(ref win, (nuint)(pos + i));
+ i += CopyChunk;
+ }
+ while (i < count);
+ return pos + count;
+ }
if (source + count <= windowLength)
{
// A forward copy is what LZMA means. It equals a block move when source and target don't overlap or the
@@ -680,13 +741,13 @@ private static int Copy(byte[] window, int pos, int distance, int count)
return pos + count;
}
- /// The write position reached the end of the window: grow it while it is below the dictionary, else wrap.
+ /// The write position reached the end of the window: grow it while it is below the dictionary (and the guard), else wrap.
private void WindowEnd()
{
- if ((uint)_window.Length < _dictionary)
+ if ((uint)_window.Length < _dictionary + Guard)
{
- // Still filling: grow (twice the size, up to the dictionary) instead of wrapping.
- Array.Resize(ref _window, (int)Math.Min(_dictionary, (long)_window.Length * 2));
+ // Still filling: grow (twice the size, up to the dictionary and the guard) instead of wrapping.
+ Array.Resize(ref _window, (int)Math.Min(_dictionary + Guard, (long)_window.Length * 2));
}
else
{
@@ -761,41 +822,49 @@ private static uint Bit(ref ushort probability, ref uint range, ref uint
}
///
- /// One adaptive bit without a branch on its value: returns all ones for a 1 and 0 for a 0, so a tree index moves
- /// on with (m << 1) − mask. The probability update is the same as in :
- /// p − ((p − 2017) >> 5) with an arithmetic shift is p + ((2048 − p) >> 5) for every p in 0 … 2048.
+ /// One adaptive bit, with the probability already read into , without a branch on its value:
+ /// returns all ones for a 0 and 0 for a 1 (the borrow of code − bound), so a tree index moves on with
+ /// (m << 1) + 1 + mask and a choice between two values is b ^ ((a ^ b) & mask). The
+ /// probability update is the same as in : p − ((p − 2017) >> 5) with an arithmetic
+ /// shift is p + ((2048 − p) >> 5) for every p in 0 … 2048.
///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static uint BitMask(ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc)
- where TInput : struct, IInputMode =>
- BitMask(probability, ref probability, ref range, ref code, ref input, ref inPos, ref inEnd, rc);
-
- /// with the probability already read into .
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static uint BitMask(uint p, ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc)
+ private static uint ZeroMask(uint p, ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc)
where TInput : struct, IInputMode
{
var bound = (range >> RangeCoder.ProbabilityBits) * p;
- var one = (uint)(((long)bound - code - 1) >> 63); // code >= bound
- range = bound + ((range - 2 * bound) & one); // bound for a 0, range − bound for a 1
- code -= bound & one;
- probability = (ushort)(p - (uint)((int)(p - (~one & (RangeCoder.ProbabilityOne - (1 << RangeCoder.MoveBits) + 1))) >> RangeCoder.MoveBits));
+ var zero = (uint)(((ulong)code - bound) >> 32); // code < bound: the 64-bit difference wraps
+ range = range - bound + ((2 * bound - range) & zero); // bound for a 0, range − bound for a 1
+ code -= bound & ~zero;
+ probability = (ushort)(p - (uint)((int)(p - (zero & (RangeCoder.ProbabilityOne - (1 << RangeCoder.MoveBits) + 1))) >> RangeCoder.MoveBits));
if (range < Top)
{
range <<= 8;
code = code << 8 | Next(ref input, ref inPos, ref inEnd, rc);
}
- return one;
+ return zero;
}
- /// A -bit symbol from a binary tree of probabilities (index 1 is the root).
+ ///
+ /// A -bit symbol from a binary tree of probabilities (index 1 is the root). Like the
+ /// literals, each step reads both children's probabilities before its bit is known (all but the last, whose
+ /// children lie past the tree), so the next step doesn't wait for a load that depends on this bit.
+ ///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint Tree(ref ushort probs, int bits, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc)
where TInput : struct, IInputMode
{
uint m = 1;
- for (var i = 0; i < bits; i++)
- m = (m << 1) - BitMask(ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ uint p = Unsafe.Add(ref probs, 1);
+ for (var i = 1; i < bits; i++)
+ {
+ uint p0 = Unsafe.Add(ref probs, m * 2);
+ uint p1 = Unsafe.Add(ref probs, m * 2 + 1);
+ var zero = ZeroMask(p, ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ p = p1 ^ ((p0 ^ p1) & zero);
+ m = (m << 1) + 1 + zero;
+ }
+ m = (m << 1) + 1 + ZeroMask(p, ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
return m - (1u << bits);
}
@@ -806,13 +875,19 @@ private static uint ReverseTree(ref ushort probs, int bits, ref uint ran
{
uint m = 1;
uint symbol = 0;
- for (var i = 0; i < bits; i++)
+ uint p = Unsafe.Add(ref probs, 1);
+ for (var i = 0; i < bits - 1; i++)
{
- var bit = 0u - BitMask(ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ uint p0 = Unsafe.Add(ref probs, m * 2);
+ uint p1 = Unsafe.Add(ref probs, m * 2 + 1);
+ var zero = ZeroMask(p, ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ p = p1 ^ ((p0 ^ p1) & zero);
+ var bit = zero + 1;
m = (m << 1) + bit;
symbol |= bit << i;
}
- return symbol;
+ var last = 1 + ZeroMask(p, ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc);
+ return symbol | (last << (bits - 1));
}
/// A match distance minus 1 (0xFFFFFFFF is the end marker): slot, then reverse-coded or direct bits.
@@ -834,7 +909,9 @@ private static uint DecodeDistance(ref ushort probs, uint length, ref ui
{
range >>= 1;
code -= range;
- var t = 0u - (code >> 31); // all ones when the subtraction went below zero (bit 0)
+ // All ones when the subtraction went below zero (bit 0). The range is now below 2^31 and the code was below
+ // twice the range plus one, so a difference that didn't go below zero is below 2^31 too: the sign tells.
+ var t = (uint)((int)code >> 31);
code += range & t;
if (code == range)
throw new InvalidDataException("the LZMA data is damaged");
diff --git a/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs b/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs
index 2859f385d..3bec65072 100644
--- a/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs
+++ b/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs
@@ -3,8 +3,10 @@
// and a new match from the MatchFinder.
//
// Two parsers:
-// - Greedy (the fast level): take the longest match. A repeat is preferred over a slightly longer new match, since it
-// needs no distance; literal against short repeat is decided by their bit prices.
+// - Lazy (the fast level): take the longest match, unless the next position starts a better one (longer, or as long
+// and nearer), in which case a literal goes first. A repeat is preferred over a slightly longer new match, since
+// it needs no distance, and a match one shorter is preferred when it is far nearer; literal against short repeat
+// is decided by their bit prices.
// - Price-based (normal and maximum): a shortest-path search over the next positions. Every position in a window
// gets all its matches (one per length, the nearest distance), its four repeats, a literal and a short repeat;
// each choice is priced in 1/16 bits with the current probabilities and the cheapest way to reach every later
@@ -13,6 +15,7 @@
// its end is coded. Prices are refreshed from the probabilities every few KB.
using System.Numerics;
+using System.Runtime.CompilerServices;
namespace Filee.Engines.Archives.Lzma;
@@ -34,6 +37,12 @@ internal sealed class LzmaEncoder : LzmaModel
private int _cachedLength;
private uint _cachedDistance;
+ // The lazy parse's look at the next position: its matches, kept for the next call when a literal went first.
+ private int _nextAt = -1;
+ private int _nextCount;
+ private readonly int[] _nextLengths = new int[MatchMaxLength + 1];
+ private readonly uint[] _nextDistances = new uint[MatchMaxLength + 1];
+
// The price-based parser: matches of the current position, the path nodes, and price tables.
private const int OptimalWindow = 2048;
private const int LengthSymbols = MatchMaxLength - MatchMinLength + 1;
@@ -91,6 +100,7 @@ public void Restart()
ResetState();
_total = 0;
_cachedAt = -1;
+ _nextAt = -1;
}
/// The window moved back by bytes (a slide); the cached search moves with it.
@@ -98,6 +108,8 @@ public void Shifted(int shift)
{
if (_cachedAt >= 0)
_cachedAt -= shift;
+ if (_nextAt >= 0)
+ _nextAt -= shift;
}
private void EncodeOne(RangeEncoder rc, int end)
@@ -106,30 +118,26 @@ private void EncodeOne(RangeEncoder rc, int end)
var pos = _mf.Position;
var available = Math.Min(MatchMaxLength, end - pos);
- var mainLength = _mf.Find(available, out var mainDistance);
-
- // The longest repeat of one of the last four distances.
- var repLength = 0;
- var repIndex = 0;
- if (available >= 2)
+ // This position's matches (searched by the last call when it looked ahead), each longer than the one before.
+ int count;
+ if (_nextAt == pos)
{
- for (var i = 0; i < 4; i++)
+ count = _nextCount;
+ for (var k = 0; k < count; k++)
{
- var distance = Rep(i);
- if (distance >= _total)
- continue;
- var from = pos - (int)distance - 1;
- if (buffer[from] != buffer[pos] || buffer[from + 1] != buffer[pos + 1])
- continue;
- var length = MatchFinder.Common(buffer, pos, from, available);
- if (length > repLength)
- {
- repLength = length;
- repIndex = i;
- }
+ _lengths[k] = Math.Min(_nextLengths[k], available);
+ _distances[k] = _nextDistances[k];
}
}
+ else
+ {
+ count = _mf.FindAll(pos, available, _lengths, _distances);
+ }
+ _nextAt = -1;
+ var mainLength = count > 0 ? _lengths[count - 1] : 0;
+ var mainDistance = count > 0 ? _distances[count - 1] : 0;
+ var repLength = LongestRep(buffer, pos, available, out var repIndex);
if (repLength >= _settings.Nice)
{
EmitRep(rc, repIndex, repLength);
@@ -141,6 +149,14 @@ private void EncodeOne(RangeEncoder rc, int end)
return;
}
+ // One byte shorter but far nearer is cheaper.
+ while (count > 1 && _lengths[count - 2] + 1 == mainLength && FarNearer(_distances[count - 2], mainDistance))
+ {
+ count--;
+ mainLength = _lengths[count - 1];
+ mainDistance = _distances[count - 1];
+ }
+
// Short matches far back cost more than the literals they replace.
if (mainLength == 2 && mainDistance >= 128 || mainLength == 3 && mainDistance >= 1 << 14)
mainLength = 0;
@@ -153,10 +169,84 @@ private void EncodeOne(RangeEncoder rc, int end)
return;
}
- if (mainLength >= 2)
- EmitMatch(rc, mainLength, mainDistance);
- else
+ if (mainLength < 2)
+ {
EmitLiteralOrShortRep(rc);
+ return;
+ }
+
+ // A look at the next position: a better match there (or a repeat nearly as long) makes this one a literal.
+ if (available > 2)
+ {
+ var nextAvailable = Math.Min(MatchMaxLength, end - pos - 1);
+ _nextCount = _mf.FindAll(pos + 1, nextAvailable, _nextLengths, _nextDistances);
+ _nextAt = pos + 1;
+ if (_nextCount > 0)
+ {
+ var nextLength = _nextLengths[_nextCount - 1];
+ var nextDistance = _nextDistances[_nextCount - 1];
+ if (nextLength >= mainLength && nextDistance < mainDistance
+ || nextLength == mainLength + 1 && !FarNearer(mainDistance, nextDistance)
+ || nextLength > mainLength + 1
+ || nextLength + 1 >= mainLength && mainLength >= 3 && FarNearer(nextDistance, mainDistance))
+ {
+ EmitLiteral(rc);
+ return;
+ }
+ }
+ if (RepReaches(buffer, pos + 1, nextAvailable, Math.Max(mainLength - 1, 2)))
+ {
+ EmitLiteral(rc);
+ return;
+ }
+ }
+ EmitMatch(rc, mainLength, mainDistance);
+ }
+
+ /// True when the distance is less than 1/128 of .
+ private static bool FarNearer(uint near, uint far) => far >> 7 > near;
+
+ /// The longest repeat of one of the last four distances at (0 when none of 2 or more).
+ private int LongestRep(byte[] buffer, int pos, int available, out int repIndex)
+ {
+ repIndex = 0;
+ var repLength = 0;
+ if (available < 2)
+ return 0;
+ for (var i = 0; i < 4; i++)
+ {
+ var distance = Rep(i);
+ if (distance >= _total)
+ continue;
+ var from = pos - (int)distance - 1;
+ if (buffer[from] != buffer[pos] || buffer[from + 1] != buffer[pos + 1])
+ continue;
+ var length = MatchFinder.Common(buffer, pos, from, available);
+ if (length > repLength)
+ {
+ repLength = length;
+ repIndex = i;
+ }
+ }
+ return repLength;
+ }
+
+ /// True when one of the last four distances repeats at least bytes at (one byte past the cursor).
+ private bool RepReaches(byte[] buffer, int pos, int available, int length)
+ {
+ if (available < length)
+ return false;
+ for (var i = 0; i < 4; i++)
+ {
+ var distance = Rep(i);
+ if (distance > _total)
+ continue;
+ var from = pos - (int)distance - 1;
+ if (buffer[from] == buffer[pos] && buffer[from + 1] == buffer[pos + 1]
+ && MatchFinder.Common(buffer, pos, from, length) == length)
+ return true;
+ }
+ return false;
}
// ───────── Price-based parse ─────────
@@ -430,6 +520,7 @@ private void RefreshPrices()
_alignPrices[i] = BitPrice.ReverseTree(Align, AlignBits, i);
}
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
private uint Rep(int index) => index switch { 0 => Rep0, 1 => Rep1, 2 => Rep2, _ => Rep3 };
private void EmitLiteralOrShortRep(RangeEncoder rc)
@@ -469,35 +560,12 @@ private void EmitLiteral(RangeEncoder rc)
var pos = _mf.Position;
var posState = (int)_total & PbMask;
rc.EncodeBit(ref IsMatch[(State << 4) + posState], 0);
- var probs = Literal.AsSpan(LiteralBase(_total, _total > 0 ? buffer[pos - 1] : 0), 0x300);
+ ref var probs = ref Literal.AsSpan(LiteralBase(_total, _total > 0 ? buffer[pos - 1] : 0), 0x300)[0];
uint symbol = buffer[pos];
if (State >= LiteralStates)
- {
- // Matched literal: while the bits agree with the byte at the last distance, a second set of
- // probabilities (chosen by that byte's bit) is used.
- uint matchByte = buffer[pos - (int)Rep0 - 1];
- uint m = 1;
- var same = true;
- for (var i = 7; i >= 0; i--)
- {
- var bit = (int)((symbol >> i) & 1);
- if (same)
- {
- var matchBit = (matchByte >> i) & 1;
- rc.EncodeBit(ref probs[(int)(((1 + matchBit) << 8) + m)], bit);
- same = matchBit == (uint)bit;
- }
- else
- {
- rc.EncodeBit(ref probs[(int)m], bit);
- }
- m = (m << 1) | (uint)bit;
- }
- }
+ rc.EncodeMatchedLiteral(ref probs, symbol, buffer[pos - (int)Rep0 - 1]);
else
- {
- rc.EncodeTree(probs, 8, symbol);
- }
+ rc.EncodeLiteral(ref probs, symbol);
State = AfterLiteral(State);
Advance(1);
}
diff --git a/src/Filee.Engines/Archives/Lzma/MatchFinder.cs b/src/Filee.Engines/Archives/Lzma/MatchFinder.cs
index 37522d4ca..f3d5eb3d5 100644
--- a/src/Filee.Engines/Archives/Lzma/MatchFinder.cs
+++ b/src/Filee.Engines/Archives/Lzma/MatchFinder.cs
@@ -12,10 +12,14 @@
// is used up. Positions are stored as buffer index + an offset that grows with each slide, so the tables never need
// rewriting when the window moves; only every ~1 GiB of input are they rebased (Normalize). 0 marks an empty slot.
//
-// Searching ahead (binary tree, for the price-based parse, with the whole input in the buffer): a thread of its own
-// searches every position in order, as the parse would search it or step over it (both walk the tree the same way),
-// and hands the matches over in batches; the parse takes each position's list cut to the length it allows there.
-// The lists are exactly those a search in the parse would find, so the output is the same bytes either way.
+// Searching ahead (binary tree, for the price-based parse, with the whole input in the buffer): threads of their own
+// search every position in order, as the parse would search it or step over it (both walk the tree the same way),
+// and hand the matches over in batches; the parse takes each position's list cut to the length it allows there.
+// A position's tree holds only positions with the same four-byte hash, so the trees are split between the threads
+// by hash: each thread walks the positions of its own hashes, in order, and touches no other tree. The last thread
+// to finish a batch adds the close matches of the two- and three-byte tables (kept in order, by one thread at a
+// time) and joins them with the tree's as one search would. The lists are exactly those a search in the parse would
+// find, so the output is the same bytes whatever the number of threads, or with none.
using System.Buffers.Binary;
using System.Collections.Concurrent;
@@ -31,10 +35,14 @@ internal sealed class MatchFinder
private const int Hash3Bits = 16;
private const int MinAhead = 1 << 16;
+ /// The most threads searches on.
+ public const int MaxSearchThreads = Ahead.MaxThreads;
+
/// Tests lower the offset at which positions are rebased (normally 1 GiB).
internal static readonly AsyncLocal TestNormalizeAt = new();
private readonly int _dictionary;
+ private readonly int _reach; // how far back the tree walks: the dictionary less the search threads' leeway
private readonly int _mask;
private readonly bool _tree;
private readonly int[] _links; // chain: one previous position per slot; tree: left and right child per slot
@@ -58,6 +66,10 @@ public MatchFinder(int dictionary, int ahead, int depth, int nice, bool tree)
if (!BitOperations.IsPow2(dictionary) || dictionary < 1 << 12 || dictionary > 1 << 28)
throw new ArgumentOutOfRangeException(nameof(dictionary));
_dictionary = dictionary;
+ // A tree slot is reused a dictionary later, possibly by another search thread (they split the trees by hash
+ // and may be up to Ahead.MaxLag positions apart): walks stop that far short of the dictionary, so no thread
+ // reads a slot another has taken over. The same with one thread or none, so the bytes don't depend on it.
+ _reach = tree ? dictionary - Math.Min(Ahead.MaxLag, dictionary >> 2) : dictionary;
_mask = dictionary - 1;
_tree = tree;
_links = new int[tree ? 2L * dictionary : dictionary];
@@ -155,12 +167,6 @@ static void Rebase(int[] table, int delta)
_offset -= delta;
}
- ///
- /// Enters in the tables and returns the longest match there of at most
- /// bytes (0 when there is none of at least 2), with its distance − 1.
- ///
- public int Find(int maxLength, out uint distance) => Search(Position, maxLength, null, null, out distance);
-
///
/// Enters in the tables and lists its matches of at most
/// bytes, each longer than the one before; returns how many.
@@ -169,15 +175,17 @@ public int FindAll(int pos, int maxLength, int[] lengths, uint[] distances) =>
_ahead is { } ahead ? ahead.Take(pos, maxLength, lengths, distances) : Search(pos, maxLength, lengths, distances, out _);
///
- /// Starts searching every position from to on a thread of its own
- /// (binary tree only); then takes its results and does nothing. Call
- /// when done.
+ /// Starts searching every position from to on
+ /// threads of their own (binary tree only); then takes their
+ /// results and does nothing. Call when done.
///
- public void StartAhead(CancellationToken cancellationToken)
+ public void StartAhead(CancellationToken cancellationToken, int threads = 1)
{
if (!_tree)
throw new InvalidOperationException("only the binary tree searches ahead");
- _ahead = new Ahead(this, Position, Filled, cancellationToken);
+ if (threads > 1 && _dictionary < 4 * Ahead.MaxLag)
+ threads = 1; // the walks' leeway (a quarter of the dictionary at most) would not cover the threads' lag
+ _ahead = new Ahead(this, Position, Filled, Math.Clamp(threads, 1, Ahead.MaxThreads), cancellationToken);
}
/// Stops the search thread (if any) and waits for it.
@@ -194,6 +202,7 @@ private ref struct Found(int maxLength, int[]? lengths, uint[]? distances)
public uint Distance;
public int Count;
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Offer(int length, int gap)
{
length = Math.Min(length, maxLength);
@@ -210,6 +219,53 @@ public void Offer(int length, int gap)
}
}
+ /// The four-byte hash bucket of (which has four bytes after it).
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private int Hash4(int pos) => (int)((BinaryPrimitives.ReadUInt32LittleEndian(Buffer.AsSpan(pos)) * 2654435761u) >> _hash4Shift);
+
+ ///
+ /// The tree part of a search at (which has four bytes after it): enters it in its tree
+ /// and offers what the walk finds.
+ ///
+ private void TreeSearch(int pos, int h4, ref Found found)
+ {
+ var stored = pos + _offset;
+ ref var slot4 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), h4);
+ var candidate = slot4;
+ slot4 = stored;
+ TreeWalk(pos, stored, candidate, ref found);
+ }
+
+ ///
+ /// The two- and three-byte part of a search at (which has four bytes after it): enters it
+ /// in those tables and offers their candidates.
+ ///
+ private void NearSearch(int pos, int maxLength, ref Found found)
+ {
+ var buffer = Buffer;
+ var value = BinaryPrimitives.ReadUInt32LittleEndian(buffer.AsSpan(pos));
+ var stored = pos + _offset;
+ var h2 = (int)(value & 0xFFFF);
+ var h3 = (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits));
+ ref var slot2 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), h2);
+ ref var slot3 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), h3);
+ var candidate2 = slot2;
+ slot2 = stored;
+ var candidate3 = slot3;
+ slot3 = stored;
+ if (maxLength < 2)
+ return;
+ foreach (var near in (ReadOnlySpan)[candidate2, candidate3])
+ {
+ if (near > 0 && stored - near < _dictionary && found.Best < maxLength)
+ {
+ var at = near - _offset;
+ if (buffer[at] == buffer[pos])
+ found.Offer(Common(buffer, pos, at, maxLength), stored - near);
+ }
+ }
+ }
+
private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, out uint distance)
{
distance = 0;
@@ -225,12 +281,16 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou
var h2 = (int)(value & 0xFFFF);
var h3 = (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits));
var h4 = (int)((value * 2654435761u) >> _hash4Shift);
- var candidate2 = _head2[h2];
- _head2[h2] = stored;
- var candidate3 = _head3[h3];
- _head3[h3] = stored;
- var candidate = _head4[h4];
- _head4[h4] = stored;
+ // The hashes are below the tables' sizes by construction: 16 bits, Hash3Bits and the shift for _head4.
+ ref var slot2 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), h2);
+ ref var slot3 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), h3);
+ ref var slot4 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), h4);
+ var candidate2 = slot2;
+ slot2 = stored;
+ var candidate3 = slot3;
+ slot3 = stored;
+ var candidate = slot4;
+ slot4 = stored;
var found = new Found(maxLength, lengths, distances);
if (maxLength >= 2)
@@ -288,25 +348,39 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou
private void TreeWalk(int pos, int stored, int candidate, ref Found found)
{
var buffer = Buffer;
- var links = _links;
+ ref var data = ref MemoryMarshal.GetArrayDataReference(buffer);
+ ref var links = ref MemoryMarshal.GetArrayDataReference(_links);
+ var mask = _mask;
+ var reach = _reach;
+ var offset = _offset;
var limit = Math.Min(_nice, Filled - pos);
- var leftSlot = (stored & _mask) << 1; // where the next candidate smaller than pos goes
- var rightSlot = leftSlot + 1; // where the next candidate larger than pos goes
- int leftLength = 0, rightLength = 0; // bytes known to match the bounds on either side
+ // Every candidate checked below lies within the dictionary behind pos, so it and the limit fit the buffer;
+ // every slot is a node's (masked) or the cursor's. The checks stay, cheap next to the walk's cache misses.
+ if ((uint)pos > (uint)(buffer.Length - limit) || limit < 0)
+ throw new ArgumentOutOfRangeException(nameof(pos));
+ ref var cursor = ref Unsafe.Add(ref data, pos);
+ var leftSlot = (stored & mask) << 1; // where the next candidate smaller than pos goes
+ var rightSlot = leftSlot + 1; // where the next candidate larger than pos goes
+ int leftLength = 0, rightLength = 0; // bytes known to match the bounds on either side
for (var depth = _depth; ; depth--)
{
var gap = stored - candidate;
- if (candidate <= 0 || gap <= 0 || gap >= _dictionary || depth == 0)
+ var at = candidate - offset;
+ if (candidate <= 0 || gap <= 0 || gap >= reach || depth == 0 || (uint)at > (uint)pos)
{
- links[leftSlot] = 0;
- links[rightSlot] = 0;
+ Unsafe.Add(ref links, leftSlot) = 0;
+ Unsafe.Add(ref links, rightSlot) = 0;
return;
}
- var at = candidate - _offset;
- var node = (candidate & _mask) << 1;
+ ref var match = ref Unsafe.Add(ref data, at);
+ var node = (candidate & mask) << 1;
+ // Both children are read before the bytes are compared: the two loads (the node's and the candidate's
+ // bytes, both mostly cache misses) then overlap instead of one waiting for the other.
+ var smaller = Unsafe.Add(ref links, node);
+ var larger = Unsafe.Add(ref links, node + 1);
var length = Math.Min(leftLength, rightLength);
// Most candidates differ at the first byte not known to match: only then is the rest compared.
- if (length < limit && buffer[at + length] == buffer[pos + length])
+ if (length < limit && Unsafe.Add(ref match, length) == Unsafe.Add(ref cursor, length))
{
length++;
if (length < limit)
@@ -316,22 +390,22 @@ private void TreeWalk(int pos, int stored, int candidate, ref Found found)
if (length == limit)
{
// Equal as far as the tree looks: the candidate's subtrees become the new root's.
- links[leftSlot] = links[node];
- links[rightSlot] = links[node + 1];
+ Unsafe.Add(ref links, leftSlot) = smaller;
+ Unsafe.Add(ref links, rightSlot) = larger;
return;
}
- if (buffer[at + length] < buffer[pos + length])
+ if (Unsafe.Add(ref match, length) < Unsafe.Add(ref cursor, length))
{
- links[leftSlot] = candidate;
+ Unsafe.Add(ref links, leftSlot) = candidate;
leftSlot = node + 1;
- candidate = links[node + 1];
+ candidate = larger;
leftLength = length;
}
else
{
- links[rightSlot] = candidate;
+ Unsafe.Add(ref links, rightSlot) = candidate;
rightSlot = node;
- candidate = links[node];
+ candidate = smaller;
rightLength = length;
}
}
@@ -378,10 +452,11 @@ private void Insert(int pos)
var h4 = (int)((value * 2654435761u) >> _hash4Shift);
var h3 = (int)(((value & 0xFFFFFF) * 0x9E3779B1u) >> (32 - Hash3Bits));
var stored = p + _offset;
- _head2[(int)(value & 0xFFFF)] = stored;
- _head3[h3] = stored;
- var candidate = _head4[h4];
- _head4[h4] = stored;
+ Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head2), (int)(value & 0xFFFF)) = stored;
+ Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head3), h3) = stored;
+ ref var slot4 = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_head4), h4);
+ var candidate = slot4;
+ slot4 = stored;
if (_tree)
TreeWalk(p, stored, candidate, ref found);
else
@@ -390,72 +465,220 @@ private void Insert(int pos)
_nextInsert = pos;
}
- /// The search thread and the batches of match lists it hands over.
+ /// The search threads and the batches of match lists they hand over.
private sealed class Ahead
{
+ /// The most tree threads one match finder starts.
+ public const int MaxThreads = 4;
+
private const int BatchSize = 4096;
private const int Batches = 8;
+ ///
+ /// How far apart two search threads may be: a thread takes a batch only when the one eight before it was
+ /// handed to the parse, after every thread finished it, so they are at most nine batches (36 Ki positions)
+ /// apart.
+ ///
+ public const int MaxLag = 1 << 16;
+
+ /// One thread's tree matches for the positions of a batch it owns: per position its offset in Data, or -1.
+ private sealed class Part
+ {
+ public readonly int[] Offsets = new int[BatchSize];
+ public int[] Data = new int[BatchSize * 4];
+ }
+
private sealed class Batch
{
- public int First;
+ public int First = -1; // -1: free
+ public int Parts; // tree threads done with it
+ public bool Ready;
public readonly int[] Offsets = new int[BatchSize];
public int[] Data = new int[BatchSize * 4]; // per position: count, then length and distance per match
+ public Part[] Trees = [];
}
- private readonly BlockingCollection _free = new();
- private readonly BlockingCollection _full = new();
+ private readonly MatchFinder _mf;
+ private readonly int _start;
+ private readonly int _end;
+ private readonly int _threads;
+ private readonly Batch[] _ring = new Batch[Batches];
+ private readonly object _gate = new();
private readonly CancellationTokenSource _stop;
- private readonly Task _producer;
+ private readonly Task[] _producers;
+ private readonly int[] _lengths = new int[LzmaModel.MatchMaxLength + 1];
+ private readonly uint[] _distances = new uint[LzmaModel.MatchMaxLength + 1];
+ private int _nextMerge; // the first batch (by number) not merged yet
+ private int _taken; // the first batch (by number) the parse has not handed back
private ExceptionDispatchInfo? _error;
private Batch? _current;
- public Ahead(MatchFinder mf, int start, int end, CancellationToken cancellationToken)
+ public Ahead(MatchFinder mf, int start, int end, int threads, CancellationToken cancellationToken)
{
+ _mf = mf;
+ _start = start;
+ _end = end;
+ _threads = threads;
for (var i = 0; i < Batches; i++)
- _free.Add(new Batch());
+ {
+ _ring[i] = new Batch { Trees = new Part[threads] };
+ for (var k = 0; k < threads; k++)
+ _ring[i].Trees[k] = new Part();
+ }
_stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var token = _stop.Token;
- _producer = Task.Factory.StartNew(() => Produce(mf, start, end, token), CancellationToken.None,
- TaskCreationOptions.LongRunning, TaskScheduler.Default);
+ _producers = new Task[threads];
+ for (var k = 0; k < threads; k++)
+ {
+ var part = k;
+ _producers[k] = Task.Factory.StartNew(() => Produce(part, token), CancellationToken.None,
+ TaskCreationOptions.LongRunning, TaskScheduler.Default);
+ }
}
- private void Produce(MatchFinder mf, int start, int end, CancellationToken token)
+ private int BatchCount => (_end - _start + BatchSize - 1) / BatchSize;
+
+ /// Thread : the tree searches of its hashes, batch by batch.
+ private void Produce(int part, CancellationToken token)
{
try
{
+ var mf = _mf;
var lengths = new int[LzmaModel.MatchMaxLength + 1];
var distances = new uint[LzmaModel.MatchMaxLength + 1];
- for (var first = start; first < end; first += BatchSize)
+ for (var number = 0; number < BatchCount; number++)
{
- var batch = _free.Take(token);
- batch.First = first;
+ var batch = _ring[number % Batches];
+ lock (_gate)
+ {
+ // The slot is free once the parse handed back the batch eight before.
+ while (number - _taken >= Batches && _error is null)
+ {
+ token.ThrowIfCancellationRequested();
+ Monitor.Wait(_gate, 100);
+ }
+ if (_error is not null)
+ return;
+ }
+ var first = _start + number * BatchSize;
+ var last = Math.Min(_end, first + BatchSize);
+ var tree = batch.Trees[part];
var at = 0;
- var last = Math.Min(end, first + BatchSize);
for (var p = first; p < last; p++)
{
- batch.Offsets[p - first] = at;
- var count = mf.Search(p, Math.Min(LzmaModel.MatchMaxLength, end - p), lengths, distances, out _);
- if (at + 1 + 2 * count > batch.Data.Length)
- Array.Resize(ref batch.Data, Math.Max(batch.Data.Length * 2, at + 1 + 2 * count));
- var data = batch.Data;
- data[at++] = count;
- for (var k = 0; k < count; k++)
+ var h4 = p + 4 <= mf.Filled ? mf.Hash4(p) : -1;
+ if (h4 < 0 || (_threads > 1 && h4 % _threads != part))
+ {
+ tree.Offsets[p - first] = -1;
+ continue;
+ }
+ var found = new Found(Math.Min(LzmaModel.MatchMaxLength, _end - p), lengths, distances);
+ mf.TreeSearch(p, h4, ref found);
+ if (at + 1 + 2 * found.Count > tree.Data.Length)
+ Array.Resize(ref tree.Data, Math.Max(tree.Data.Length * 2, at + 1 + 2 * found.Count));
+ tree.Offsets[p - first] = at;
+ var data = tree.Data;
+ data[at++] = found.Count;
+ for (var k = 0; k < found.Count; k++)
{
data[at++] = lengths[k];
data[at++] = (int)distances[k];
}
}
- _full.Add(batch, token);
+ token.ThrowIfCancellationRequested();
+ bool merge;
+ lock (_gate)
+ {
+ if (batch.Parts == 0)
+ batch.First = first;
+ merge = ++batch.Parts == _threads;
+ }
+ if (merge)
+ MergeReady();
}
}
catch (Exception ex)
{
- _error = ExceptionDispatchInfo.Capture(ex);
+ lock (_gate)
+ {
+ _error ??= ExceptionDispatchInfo.Capture(ex);
+ Monitor.PulseAll(_gate);
+ }
+ }
+ }
+
+ /// Joins the batches whose trees are all done, in order (one thread at a time).
+ private void MergeReady()
+ {
+ lock (_mergeGate)
+ {
+ while (true)
+ {
+ Batch batch;
+ lock (_gate)
+ {
+ if (_nextMerge >= BatchCount)
+ return;
+ batch = _ring[_nextMerge % Batches];
+ if (batch.Parts != _threads || batch.Ready)
+ return;
+ }
+ Merge(batch);
+ lock (_gate)
+ {
+ batch.Ready = true;
+ _nextMerge++;
+ Monitor.PulseAll(_gate);
+ }
+ }
}
- finally
+ }
+
+ private readonly object _mergeGate = new();
+
+ /// Adds the close matches to every position of the batch and joins them with its tree's.
+ private void Merge(Batch batch)
+ {
+ var mf = _mf;
+ var buffer = mf.Buffer;
+ var first = batch.First;
+ var last = Math.Min(_end, first + BatchSize);
+ var at = 0;
+ for (var p = first; p < last; p++)
{
- _full.CompleteAdding();
+ batch.Offsets[p - first] = at;
+ var maxLength = Math.Min(LzmaModel.MatchMaxLength, _end - p);
+ var found = new Found(maxLength, _lengths, _distances);
+ var count = 0;
+ if (p + 4 <= mf.Filled)
+ {
+ mf.NearSearch(p, maxLength, ref found);
+ // The tree's matches longer than the close ones, as one search offers them after those.
+ var owner = _threads > 1 ? mf.Hash4(p) % _threads : 0;
+ var tree = batch.Trees[owner];
+ var from = tree.Offsets[p - first];
+ var treeData = tree.Data;
+ var treeCount = treeData[from++];
+ for (var k = 0; k < treeCount; k++, from += 2)
+ found.Offer(treeData[from], treeData[from + 1] + 1);
+ // The tree compares at most the nice length; a match that long may go on.
+ if (found.Best >= mf._nice && found.Best < maxLength)
+ {
+ var longer = Common(buffer, p, p - (int)found.Distance - 1, maxLength);
+ found.Best = longer;
+ _lengths[found.Count - 1] = longer;
+ }
+ count = found.Best < 2 ? 0 : found.Count;
+ }
+ if (at + 1 + 2 * count > batch.Data.Length)
+ Array.Resize(ref batch.Data, Math.Max(batch.Data.Length * 2, at + 1 + 2 * count));
+ var data = batch.Data;
+ data[at++] = count;
+ for (var k = 0; k < count; k++)
+ {
+ data[at++] = _lengths[k];
+ data[at++] = (int)_distances[k];
+ }
}
}
@@ -464,15 +687,26 @@ public int Take(int pos, int maxLength, int[] lengths, uint[] distances)
{
while (_current is null || pos >= _current.First + BatchSize)
{
- if (_current is not null)
- _free.Add(_current);
- _current = null;
- if (!_full.TryTake(out var next, Timeout.Infinite))
+ lock (_gate)
{
+ if (_current is not null)
+ {
+ // Handed back: its slot takes a later batch.
+ _current.First = -1;
+ _current.Parts = 0;
+ _current.Ready = false;
+ _current = null;
+ _taken++;
+ Monitor.PulseAll(_gate);
+ }
+ if (_taken >= BatchCount)
+ throw new InvalidOperationException("a position past the input was searched");
+ var batch = _ring[_taken % Batches];
+ while (!batch.Ready && _error is null)
+ Monitor.Wait(_gate);
_error?.Throw();
- throw new InvalidOperationException("a position past the input was searched");
+ _current = batch;
}
- _current = next;
}
if (maxLength < 2)
return 0;
@@ -496,17 +730,17 @@ public int Take(int pos, int maxLength, int[] lengths, uint[] distances)
public void Stop()
{
_stop.Cancel();
+ lock (_gate)
+ Monitor.PulseAll(_gate);
try
{
- _producer.Wait();
+ Task.WaitAll(_producers);
}
catch (AggregateException)
{
- // The producer catches everything.
+ // The producers catch everything.
}
_stop.Dispose();
- _free.Dispose();
- _full.Dispose();
}
}
}
diff --git a/src/Filee.Engines/Archives/Lzma/RangeEncoder.cs b/src/Filee.Engines/Archives/Lzma/RangeEncoder.cs
index d943d509e..2bc6612b5 100644
--- a/src/Filee.Engines/Archives/Lzma/RangeEncoder.cs
+++ b/src/Filee.Engines/Archives/Lzma/RangeEncoder.cs
@@ -1,8 +1,13 @@
// The LZMA range encoder, the mirror of RangeDecoder: a 33-bit low end (the extra bit is a carry into bytes already
// produced), a 32-bit range, and a byte held back plus a count of 0xFF bytes after it, because a later carry can still
// change them. Output goes to a byte array that holds one LZMA2 chunk.
+//
+// The multi-bit codings (trees, literals, direct bits) keep the low end and the range in locals for all their bits
+// and code each bit without a branch on its value (the bits of a literal or a distance are close to random): a mask
+// of the bit picks the new range and what the low end gains, as in the decoder.
using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
namespace Filee.Engines.Archives.Lzma;
@@ -39,21 +44,83 @@ public void Flush()
ShiftLow();
}
- private void ShiftLow()
+ private void ShiftLow() => _low = ShiftLow(_low);
+
+ /// Moves the top byte of out (held back while a carry may still reach it); returns the new low end.
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private ulong ShiftLow(ulong low)
{
- if ((uint)_low < 0xFF000000u || (_low >> 32) != 0)
+ if ((uint)low < 0xFF000000u || (low >> 32) != 0)
{
- var carry = (byte)(_low >> 32);
+ var carry = (byte)(low >> 32);
var value = _cache;
do
{
_buffer[_count++] = (byte)(value + carry);
value = 0xFF;
} while (--_cacheSize != 0);
- _cache = (byte)((uint)_low >> 24);
+ _cache = (byte)((uint)low >> 24);
}
_cacheSize++;
- _low = (_low & 0x00FFFFFF) << 8;
+ return (low & 0x00FFFFFF) << 8;
+ }
+
+ /// One bit with the adaptive probability , on the caller's locals, without a branch on the bit.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private void Bit(ref ushort probability, uint bit, ref ulong low, ref uint range)
+ {
+ uint p = probability;
+ var bound = (range >> RangeCoder.ProbabilityBits) * p;
+ var one = 0 - bit;
+ low += bound & one;
+ range = bound + ((range - 2 * bound) & one);
+ // p + ((2048 − p) >> 5) for a 0 and p − (p >> 5) for a 1, as in the decoder.
+ probability = (ushort)(p - (uint)((int)(p - (~one & (RangeCoder.ProbabilityOne - (1 << RangeCoder.MoveBits) + 1))) >> RangeCoder.MoveBits));
+ if (range < RangeCoder.TopValue)
+ {
+ range <<= 8;
+ low = ShiftLow(low);
+ }
+ }
+
+ /// A literal (8 bits through the tree at , most significant first).
+ public void EncodeLiteral(ref ushort probabilities, uint symbol)
+ {
+ var low = _low;
+ var range = _range;
+ uint m = 1;
+ for (var i = 7; i >= 0; i--)
+ {
+ var bit = (symbol >> i) & 1;
+ Bit(ref Unsafe.Add(ref probabilities, m), bit, ref low, ref range);
+ m = (m << 1) | bit;
+ }
+ _low = low;
+ _range = range;
+ }
+
+ ///
+ /// A literal after a match: while its bits agree with 's, the probabilities picked by
+ /// that byte's bit (0x100 or 0x200 on) are used, after the first difference the plain ones.
+ ///
+ public void EncodeMatchedLiteral(ref ushort probabilities, uint symbol, uint matchByte)
+ {
+ var low = _low;
+ var range = _range;
+ uint m = 1;
+ var offset = 0x100u; // 0x100 while the bits agree, 0 after
+ for (var i = 7; i >= 0; i--)
+ {
+ matchByte <<= 1;
+ var matchBit = matchByte & 0x100 & offset; // 0x100 or 0
+ var bit = (symbol >> i) & 1;
+ Bit(ref Unsafe.Add(ref probabilities, offset + matchBit + m), bit, ref low, ref range);
+ m = (m << 1) | bit;
+ // The bits still agree when this one equals the match byte's.
+ offset &= ((matchBit >> 8) ^ bit) - 1;
+ }
+ _low = low;
+ _range = range;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -82,42 +149,59 @@ public void EncodeBit(ref ushort probability, int bit)
/// The low bits of at probability one half, most significant first.
public void EncodeDirectBits(uint value, int count)
{
+ var low = _low;
+ var range = _range;
while (count-- > 0)
{
- _range >>= 1;
- if (((value >> count) & 1) != 0)
- _low += _range;
- while (_range < RangeCoder.TopValue)
+ range >>= 1;
+ low += range & (0 - ((value >> count) & 1));
+ if (range < RangeCoder.TopValue)
{
- _range <<= 8;
- ShiftLow();
+ range <<= 8;
+ low = ShiftLow(low);
}
}
+ _low = low;
+ _range = range;
}
/// A -bit symbol through a binary tree of probabilities, most significant bit first.
public void EncodeTree(Span probabilities, int bits, uint symbol)
{
+ if (probabilities.Length < 1 << bits)
+ throw new ArgumentOutOfRangeException(nameof(probabilities));
+ ref var probs = ref MemoryMarshal.GetReference(probabilities);
+ var low = _low;
+ var range = _range;
uint m = 1;
for (var i = bits - 1; i >= 0; i--)
{
- var bit = (int)((symbol >> i) & 1);
- EncodeBit(ref probabilities[(int)m], bit);
- m = (m << 1) | (uint)bit;
+ var bit = (symbol >> i) & 1;
+ Bit(ref Unsafe.Add(ref probs, m), bit, ref low, ref range);
+ m = (m << 1) | bit;
}
+ _low = low;
+ _range = range;
}
/// Like , least significant bit first.
public void EncodeReverseTree(Span probabilities, int bits, uint symbol)
{
+ if (probabilities.Length < 1 << bits)
+ throw new ArgumentOutOfRangeException(nameof(probabilities));
+ ref var probs = ref MemoryMarshal.GetReference(probabilities);
+ var low = _low;
+ var range = _range;
uint m = 1;
for (var i = 0; i < bits; i++)
{
- var bit = (int)(symbol & 1);
+ var bit = symbol & 1;
symbol >>= 1;
- EncodeBit(ref probabilities[(int)m], bit);
- m = (m << 1) | (uint)bit;
+ Bit(ref Unsafe.Add(ref probs, m), bit, ref low, ref range);
+ m = (m << 1) | bit;
}
+ _low = low;
+ _range = range;
}
}
diff --git a/src/Filee.Engines/Archives/Lzma/XzReader.cs b/src/Filee.Engines/Archives/Lzma/XzReader.cs
index 5b411e341..e4b391418 100644
--- a/src/Filee.Engines/Archives/Lzma/XzReader.cs
+++ b/src/Filee.Engines/Archives/Lzma/XzReader.cs
@@ -12,7 +12,8 @@
// whole, then decoded and checked on a worker thread while the next block headers are read (Lzma2Parallel bounds how
// many and how large). Blocks are handed out in order; a block that fails on its thread is decoded again on the
// reader's thread as it is read, so damaged data gives the same bytes and the same error, at the same place, as the
-// one-thread reader. An error in a header read ahead is thrown after the blocks before it.
+// one-thread reader. An error in a header read ahead is thrown after the blocks before it. A worker hands out its
+// block as it decodes it (the check is compared at the block's end, as the one-thread reader does).
using System.Buffers.Binary;
using System.Runtime.ExceptionServices;
@@ -50,7 +51,8 @@ private enum Step { StreamHeader, BlockOrIndex, BlockData, StreamPadding, Done }
// Blocks decoding on worker threads, in order; the one being handed out; an error met while reading ahead.
private readonly Queue _ahead = new();
- private Lzma2Parallel.Decoded? _current;
+ private AheadBlock? _current;
+ private int _currentReady;
private Replay? _replay;
private int _currentPosition;
private int _aheadLimit = Environment.ProcessorCount;
@@ -98,32 +100,39 @@ public override int Read(Span buffer)
}
if (_current is { } current)
{
- if (_currentPosition < current.Count)
+ if (_currentPosition < _currentReady)
{
- var count = Math.Min(buffer.Length, current.Count - _currentPosition);
- current.Output.AsSpan(_currentPosition, count).CopyTo(buffer);
+ var count = Math.Min(buffer.Length, _currentReady - _currentPosition);
+ current.Work.Output.AsSpan(_currentPosition, count).CopyTo(buffer);
_currentPosition += count;
_position += count;
return count;
}
+ // The worker hands its block out as it decodes it.
+ _currentReady = current.Work.WaitBeyond(_currentPosition, out var done);
+ if (_currentPosition < _currentReady || !done)
+ continue;
_current = null;
- }
- ReadAhead();
- if (_ahead.TryDequeue(out var next))
- {
- var decoded = next.Work.GetAwaiter().GetResult();
- if (decoded.Error is { } error)
+ if (current.Work.Error is { } error)
{
- // Decoded again here, as the one-thread reader reads it.
+ // Decoded again here, as the one-thread reader reads it, past what the worker handed out.
if (error.SourceException is OperationCanceledException)
error.Throw();
- var input = new ByteInput(new MemoryStream(next.Data, 0, next.Length, writable: false));
- _replay = new Replay(next, input, new Lzma2Reader(input, next.Dictionary, next.DeclaredUnpacked, _cancellationToken),
+ var input = new ByteInput(new MemoryStream(current.Data, 0, current.Length, writable: false));
+ var again = new Replay(current, input, new Lzma2Reader(input, current.Dictionary, current.DeclaredUnpacked, _cancellationToken),
Check.Create(_checkType));
- continue;
+ Lzma2Parallel.Skip(again.Reader, _currentPosition, again.Check.Append);
+ again.Unpacked = _currentPosition;
+ _replay = again;
}
- _current = decoded;
+ continue;
+ }
+ ReadAhead();
+ if (_ahead.TryDequeue(out var next))
+ {
+ _current = next;
_currentPosition = 0;
+ _currentReady = 0;
continue;
}
if (_deferred is { } deferred)
@@ -286,14 +295,15 @@ private bool TryDecodeAhead(uint dictionary)
_blocks.Add((_blockHeaderSize + _declaredPacked + checkSize, _declaredUnpacked));
_cancel ??= CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken);
var (token, packed, unpacked, checkType, name) = (_cancel.Token, _declaredPacked, (int)_declaredUnpacked, _checkType, _name);
- var work = Task.Run(() => DecodeBlock(data, read, dictionary, packed, unpacked, checkType, name, token), CancellationToken.None);
+ var work = new Lzma2Parallel.Decoded(GC.AllocateUninitializedArray(unpacked));
+ _ = Task.Run(() => DecodeBlock(work, data, read, dictionary, packed, unpacked, checkType, name, token), CancellationToken.None);
_ahead.Enqueue(new AheadBlock(work, data, read, dictionary, packed, unpacked));
_step = Step.BlockOrIndex;
return true;
}
/// A block read whole and decoding on a worker thread, with what it takes to decode it again here.
- private sealed record AheadBlock(Task Work, byte[] Data, int Length, uint Dictionary, long DeclaredPacked,
+ private sealed record AheadBlock(Lzma2Parallel.Decoded Work, byte[] Data, int Length, uint Dictionary, long DeclaredPacked,
int DeclaredUnpacked);
/// A block that failed on its thread, decoded again on this one as it is read.
@@ -307,10 +317,9 @@ private sealed class Replay(AheadBlock block, ByteInput input, Lzma2Reader reade
}
/// A block read whole (data, padding and check), decoded and checked as does.
- private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint dictionary, long declaredPacked, int declaredUnpacked,
- int checkType, string name, CancellationToken cancellationToken)
+ private static void DecodeBlock(Lzma2Parallel.Decoded result, byte[] data, int length, uint dictionary, long declaredPacked,
+ int declaredUnpacked, int checkType, string name, CancellationToken cancellationToken)
{
- var result = new Lzma2Parallel.Decoded(GC.AllocateUninitializedArray(declaredUnpacked));
try
{
var input = new ByteInput(new MemoryStream(data, 0, length, writable: false));
@@ -319,7 +328,9 @@ private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint d
var output = result.Output;
while (true)
{
- var read = result.Count < declaredUnpacked ? reader.Read(output.AsSpan(result.Count)) : 0;
+ var read = result.Count < declaredUnpacked
+ ? reader.Read(output.AsSpan(result.Count, Math.Min(Lzma2Parallel.Decoded.Step, declaredUnpacked - result.Count)))
+ : 0;
if (read == 0)
{
// All of it, or the data ended; one more byte would be past the header's size.
@@ -330,6 +341,7 @@ private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint d
}
check.Append(output.AsSpan(result.Count, read));
result.Count += read;
+ result.Publish();
}
CheckBlockEnd(input, input.Consumed, declaredPacked, declaredUnpacked, result.Count, checkType, check, name);
}
@@ -337,7 +349,10 @@ private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint d
{
result.Error = ExceptionDispatchInfo.Capture(ex);
}
- return result;
+ finally
+ {
+ result.Finish();
+ }
}
private void FinishBlock()
diff --git a/src/Filee.Engines/Archives/ReadAheadStream.cs b/src/Filee.Engines/Archives/ReadAheadStream.cs
new file mode 100644
index 000000000..e8d1d93b8
--- /dev/null
+++ b/src/Filee.Engines/Archives/ReadAheadStream.cs
@@ -0,0 +1,197 @@
+// Runs a decoder on a thread of its own, a few blocks ahead of whoever reads it. A decoder that has to go through its
+// data in order (one LZMA stream, a BCJ filter after it, the sub-streams of BCJ2) then works while the reader checks
+// CRCs and writes files, or while the next stage of the chain works on what came before; the slowest stage sets the
+// pace instead of the sum of them all.
+//
+// What the reader gets is exactly what the decoder gives, in the same order. The worker asks the decoder for
+// PieceSize bytes at a time (what ArchiveFiles.CopyLimited asks for), so an error comes after the same bytes, give or
+// take one piece: the decoder's exception itself is thrown to the reader once the bytes before it are read. Disposing
+// stops the worker and waits for it, so the decoder (and the archive stream under it) is never used after that.
+
+using System.Runtime.ExceptionServices;
+
+namespace Filee.Engines.Archives;
+
+/// A read-only stream that reads its inner stream ahead on a worker thread.
+internal sealed class ReadAheadStream : Stream
+{
+ /// Streams shorter than this are read directly: a thread would cost more than it gives.
+ public const long MinSize = 1 << 20;
+
+ /// Bytes handed over at a time.
+ private const int BlockSize = 1 << 20;
+
+ /// Bytes the worker asks the decoder for at a time.
+ private const int PieceSize = 1 << 16;
+
+ /// Blocks read ahead at most (memory: this many blocks, plus the one being read).
+ private const int Depth = 4;
+
+ /// Tests turn read-ahead off (false) or on for any size (true).
+ internal static readonly AsyncLocal TestMode = new();
+
+ /// Streams opened with a worker so far (for tests).
+ internal static int Opened;
+
+ private readonly Stream _inner;
+ private readonly CancellationTokenSource _cancel;
+ private readonly object _gate = new();
+ private readonly Queue _filled = new();
+ private readonly Stack _free = new();
+ private readonly Thread _worker;
+ private Block? _current;
+ private int _at;
+ private bool _stopping;
+ private bool _workerDone;
+
+ private sealed record Block(byte[] Data, int Count, bool End, ExceptionDispatchInfo? Error);
+
+ private ReadAheadStream(Stream inner, CancellationToken cancellationToken)
+ {
+ _inner = inner;
+ _cancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ for (var i = 0; i < Depth + 1; i++)
+ _free.Push(GC.AllocateUninitializedArray(BlockSize));
+ Interlocked.Increment(ref Opened);
+ _worker = new Thread(Work) { IsBackground = true, Name = "Filee read-ahead" };
+ _worker.Start();
+ }
+
+ ///
+ /// read ahead on a worker thread when that pays off (more than one processor, at least
+ /// bytes, or -1 for not known); else itself.
+ /// The result owns .
+ ///
+ public static Stream Wrap(Stream inner, long size, CancellationToken cancellationToken)
+ {
+ var mode = TestMode.Value;
+ if (mode == false || (mode is null && (Environment.ProcessorCount < 2 || (size >= 0 && size < MinSize))))
+ return inner;
+ return new ReadAheadStream(inner, cancellationToken);
+ }
+
+ private void Work()
+ {
+ var token = _cancel.Token;
+ try
+ {
+ while (true)
+ {
+ byte[] data;
+ lock (_gate)
+ {
+ while (_free.Count == 0 && !_stopping)
+ Monitor.Wait(_gate);
+ if (_stopping)
+ return;
+ data = _free.Pop();
+ }
+ var count = 0;
+ var end = false;
+ ExceptionDispatchInfo? error = null;
+ try
+ {
+ while (count < data.Length)
+ {
+ token.ThrowIfCancellationRequested();
+ var read = _inner.Read(data, count, Math.Min(PieceSize, data.Length - count));
+ if (read <= 0)
+ {
+ end = true;
+ break;
+ }
+ count += read;
+ }
+ }
+ catch (Exception ex)
+ {
+ error = ExceptionDispatchInfo.Capture(ex);
+ }
+ lock (_gate)
+ {
+ _filled.Enqueue(new Block(data, count, end, error));
+ Monitor.PulseAll(_gate);
+ }
+ if (end || error is not null)
+ return;
+ }
+ }
+ finally
+ {
+ lock (_gate)
+ {
+ _workerDone = true;
+ Monitor.PulseAll(_gate);
+ }
+ }
+ }
+
+ public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
+
+ public override int Read(Span buffer)
+ {
+ if (buffer.IsEmpty)
+ return 0;
+ while (true)
+ {
+ if (_current is { } current)
+ {
+ if (_at < current.Count)
+ {
+ var count = Math.Min(buffer.Length, current.Count - _at);
+ current.Data.AsSpan(_at, count).CopyTo(buffer);
+ _at += count;
+ return count;
+ }
+ current.Error?.Throw();
+ if (current.End)
+ return 0;
+ lock (_gate)
+ {
+ _free.Push(current.Data);
+ Monitor.PulseAll(_gate);
+ }
+ _current = null;
+ }
+ lock (_gate)
+ {
+ while (_filled.Count == 0)
+ {
+ if (_workerDone)
+ throw new ObjectDisposedException(nameof(ReadAheadStream));
+ Monitor.Wait(_gate);
+ }
+ _current = _filled.Dequeue();
+ _at = 0;
+ }
+ }
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing && !_stopping)
+ {
+ lock (_gate)
+ {
+ _stopping = true;
+ Monitor.PulseAll(_gate);
+ }
+ _cancel.Cancel();
+ // The decoder and the archive under it are left alone once this returns.
+ _worker.Join();
+ _cancel.Dispose();
+ _inner.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+
+ public override bool CanRead => true;
+ public override bool CanSeek => false;
+ public override bool CanWrite => false;
+ public override long Length => throw new NotSupportedException();
+ public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
+ public override void Flush() { }
+ public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+ public override void SetLength(long value) => throw new NotSupportedException();
+ public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+}
diff --git a/src/Filee.Engines/Archives/SevenZipCoders.cs b/src/Filee.Engines/Archives/SevenZipCoders.cs
index 4b80c5b05..c8fc638b3 100644
--- a/src/Filee.Engines/Archives/SevenZipCoders.cs
+++ b/src/Filee.Engines/Archives/SevenZipCoders.cs
@@ -195,7 +195,7 @@ protected override void Dispose(bool disposing)
///
/// A window of a stream others read too (the packed streams of one 7z folder side by side): every read seeks to
-/// where this window stopped. Disposing it leaves the stream open.
+/// where this window stopped, holding the stream's lock. Disposing it leaves the stream open.
///
internal sealed class SharedWindowStream(Stream shared, long offset, long length) : ReadOnlyDecoderStream
{
@@ -215,9 +215,14 @@ public override int Read(Span buffer)
return 0;
if (buffer.Length > _end - _position)
buffer = buffer[..(int)(_end - _position)];
- if (shared.Position != _position)
- shared.Position = _position;
- var read = shared.Read(buffer);
+ int read;
+ // The coders of one folder may read their packed streams on threads of their own (ReadAheadStream).
+ lock (shared)
+ {
+ if (shared.Position != _position)
+ shared.Position = _position;
+ read = shared.Read(buffer);
+ }
if (read <= 0)
return 0;
_position += read;
diff --git a/src/Filee.Engines/Archives/SevenZipReader.cs b/src/Filee.Engines/Archives/SevenZipReader.cs
index 7468a5252..b7191a07d 100644
--- a/src/Filee.Engines/Archives/SevenZipReader.cs
+++ b/src/Filee.Engines/Archives/SevenZipReader.cs
@@ -725,7 +725,28 @@ public static Stream Open(Stream stream, SevenZipListing listing, int index, str
firstOut[i] = firstOut[i - 1] + folder.Coders[i - 1].OutStreams;
}
var opened = new bool[folder.Coders.Length];
- return OpenCoder(Array.IndexOf(firstOut, folder.MainOut));
+ // Each coder that feeds another decodes ahead on a thread of its own, and so does the last one: the chain's
+ // stages and the writing of the files run side by side. The threads are stopped with the folder's stream,
+ // or here when the folder doesn't open.
+ var ahead = new List();
+ try
+ {
+ var output = Ahead(OpenCoder(Array.IndexOf(firstOut, folder.MainOut)), folder.UnpackSize);
+ return ahead.Count == 0 ? output : new FolderOutput(output, ahead);
+ }
+ catch
+ {
+ FolderOutput.Stop(ahead);
+ throw;
+ }
+
+ Stream Ahead(Stream decoder, long size)
+ {
+ var stream = ReadAheadStream.Wrap(decoder, size, cancellationToken);
+ if (stream != decoder)
+ ahead.Add(stream);
+ return stream;
+ }
Stream OpenCoder(int coder)
{
@@ -740,7 +761,8 @@ Stream OpenCoder(int coder)
var pair = Array.FindIndex(folder.BindPairs, p => p.In == input);
if (pair >= 0)
{
- inputs[j] = OpenCoder(Array.IndexOf(firstOut, folder.BindPairs[pair].Out));
+ var bound = folder.BindPairs[pair].Out;
+ inputs[j] = Ahead(OpenCoder(Array.IndexOf(firstOut, bound)), folder.UnpackSizes[bound]);
continue;
}
var pack = folder.FirstPackStream + Array.IndexOf(folder.PackedStreams, input);
@@ -752,6 +774,31 @@ Stream OpenCoder(int coder)
}
}
+ ///
+ /// A folder's output with coders read through s. Disposing it stops them all, the
+ /// later (which read the earlier) first.
+ ///
+ private sealed class FolderOutput(Stream output, List ahead) : ReadOnlyDecoderStream
+ {
+ public override int Read(Span buffer) => output.Read(buffer);
+
+ public static void Stop(List ahead)
+ {
+ for (var i = ahead.Count - 1; i >= 0; i--)
+ ahead[i].Dispose();
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ output.Dispose();
+ Stop(ahead);
+ }
+ base.Dispose(disposing);
+ }
+ }
+
/// Reads the numbers, bit vectors and digests of a 7z header; running out of bytes is damage.
private sealed class HeaderReader(byte[] data, string name)
{
diff --git a/src/Filee.Engines/Archives/SevenZipWriter.cs b/src/Filee.Engines/Archives/SevenZipWriter.cs
index c02b0107e..e8af27427 100644
--- a/src/Filee.Engines/Archives/SevenZipWriter.cs
+++ b/src/Filee.Engines/Archives/SevenZipWriter.cs
@@ -80,12 +80,17 @@ public SevenZipWriter(Stream output, ArchiveLevel level, long expectedTotal = -1
}
///
- /// The most data one solid block takes at : 16 × the LZMA2 dictionary (64 MiB fast,
- /// 256 MiB normal, 1 GiB maximum). Beyond the dictionary a new block costs only a little ratio, and a reader that
- /// wants one file decodes at most this much before it. The store level puts every file into its own folder.
+ /// The most data one solid block takes at : 64 MiB at the fast level, else 16 × the
+ /// LZMA2 dictionary (256 MiB normal, 1 GiB maximum). Beyond the dictionary a new block costs only a little ratio,
+ /// and a reader that wants one file decodes at most this much before it. The store level puts every file into its
+ /// own folder.
///
- public static long SolidBlockSize(ArchiveLevel level) =>
- level == ArchiveLevel.Store ? 0 : 16L * Lzma2Settings.For(level).Dictionary;
+ public static long SolidBlockSize(ArchiveLevel level) => level switch
+ {
+ ArchiveLevel.Store => 0,
+ ArchiveLevel.Fast => 64L << 20,
+ _ => 16L * Lzma2Settings.For(level).Dictionary,
+ };
/// Entries added so far.
public int Count => _entries.Count;
diff --git a/tests/Filee.Engines.Tests/ArchiveThreadTests.cs b/tests/Filee.Engines.Tests/ArchiveThreadTests.cs
index d75db106b..ae44cbac8 100644
--- a/tests/Filee.Engines.Tests/ArchiveThreadTests.cs
+++ b/tests/Filee.Engines.Tests/ArchiveThreadTests.cs
@@ -1,7 +1,7 @@
-// Archives coded and decoded on several threads (Lzma2Writer blocks, Lzma2Parallel, XzReader blocks, Bzip2Decoder's
-// stream): the same bytes as on one thread, the same errors at the same place, 7-Zip reads what Filee writes (skipped
-// when it is missing), and cancellation stops the workers. Block sizes are made small so that little data gives many
-// blocks.
+// Archives coded and decoded on several threads (Lzma2Writer blocks, the match finder's search threads, Lzma2Parallel,
+// XzReader blocks, Bzip2Decoder's stream, ReadAheadStream): the same bytes as on one thread, the same errors at the
+// same place, 7-Zip reads what Filee writes (skipped when it is missing), and cancellation stops the workers. Block
+// sizes are made small so that little data gives many blocks.
using Filee.Core.Presets;
using Filee.Engines.Archives;
@@ -380,4 +380,211 @@ public void A_cancelled_bzip2_stream_stops()
}
});
}
+ // ───────── Match finder threads ─────────
+
+ [Fact]
+ public void The_match_finder_split_over_threads_gives_the_same_bytes()
+ {
+ // A dictionary smaller than the block, so tree slots are taken over a dictionary later (by another thread).
+ var data = Bzip2WriterTests.Mixed(1_400_000);
+ var settings = new Lzma2Settings(1 << 18, new LzmaEncoderSettings(Depth: 32, Nice: 32, Optimal: true, Tree: true), Store: false);
+ byte[] Coded(int searchThreads)
+ {
+ Lzma2Writer.TestThreads.Value = 1;
+ Lzma2Writer.TestBlockSize.Value = 1 << 20;
+ Lzma2Writer.TestSearchAhead.Value = searchThreads > 0;
+ Lzma2Writer.TestSearchThreads.Value = searchThreads;
+ try
+ {
+ using var output = new MemoryStream();
+ var writer = new Lzma2Writer(output, settings, data.Length, Ct);
+ for (var at = 0; at < data.Length; at += 65_536)
+ writer.Write(data.AsSpan(at, Math.Min(65_536, data.Length - at)));
+ writer.Finish();
+ return output.ToArray();
+ }
+ finally
+ {
+ Lzma2Writer.TestThreads.Value = null;
+ Lzma2Writer.TestBlockSize.Value = null;
+ Lzma2Writer.TestSearchAhead.Value = null;
+ Lzma2Writer.TestSearchThreads.Value = null;
+ }
+ }
+ var alone = Coded(0);
+ foreach (var threads in new[] { 1, 2, 4 })
+ Assert.Equal(alone, Coded(threads));
+ var reader = new Lzma2Reader(new ByteInput(new MemoryStream(alone)), 1 << 18, data.Length, Ct);
+ Assert.Equal(data, ReadAll(new Lzma2Stream(reader)));
+ }
+
+ [Fact]
+ public void A_distance_past_the_dictionary_is_damage_even_within_the_windows_guard()
+ {
+ // A 100-byte repeat from 4,100 bytes back, once 4,200 bytes are out: fine with an 8 KiB dictionary, beyond a
+ // 4 KiB one (the decoder's window has a few bytes more than the dictionary; they don't count).
+ var random = Noise(4_200, 9);
+ byte[] data = [.. random, .. random.AsSpan(100, 100), .. Noise(300, 10)];
+ var settings = new Lzma2Settings(1 << 13, new LzmaEncoderSettings(Depth: 32, Nice: 32, Optimal: true, Tree: true), Store: false);
+ using var output = new MemoryStream();
+ var writer = new Lzma2Writer(output, settings, data.Length, Ct);
+ writer.Write(data);
+ writer.Finish();
+ var coded = output.ToArray();
+ byte[] Decode(uint dictionary) =>
+ ReadAll(new Lzma2Stream(new Lzma2Reader(new ByteInput(new MemoryStream(coded)), dictionary, data.Length, Ct)));
+ Assert.Equal(data, Decode(1 << 13));
+ Assert.Contains("before the start of the window", Assert.Throws(() => Decode(1 << 12)).Message);
+ }
+
+ // ───────── Read-ahead ─────────
+
+ /// Counts up byte by byte, a thousand at a time, and fails past (-1: never ends).
+ private sealed class Counting(long failAt) : Stream
+ {
+ private long _position;
+ private int _reads;
+
+ public int Reads => Volatile.Read(ref _reads);
+
+ public bool Disposed { get; private set; }
+
+ public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
+
+ public override int Read(Span buffer)
+ {
+ Interlocked.Increment(ref _reads);
+ if (failAt >= 0 && _position >= failAt)
+ throw new InvalidDataException("the test data is damaged here");
+ var n = Math.Min(buffer.Length, 1000);
+ for (var i = 0; i < n; i++)
+ buffer[i] = (byte)(_position + i);
+ _position += n;
+ return n;
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ Disposed = true;
+ base.Dispose(disposing);
+ }
+
+ public override bool CanRead => true;
+ public override bool CanSeek => false;
+ public override bool CanWrite => false;
+ public override long Length => throw new NotSupportedException();
+ public override long Position { get => _position; set => throw new NotSupportedException(); }
+ public override void Flush() { }
+ public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+ public override void SetLength(long value) => throw new NotSupportedException();
+ public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+ }
+
+ private static Stream Ahead(Stream inner, CancellationToken cancellationToken)
+ {
+ ReadAheadStream.TestMode.Value = true;
+ try
+ {
+ var stream = ReadAheadStream.Wrap(inner, 10, cancellationToken);
+ Assert.NotSame(inner, stream);
+ return stream;
+ }
+ finally
+ {
+ ReadAheadStream.TestMode.Value = null;
+ }
+ }
+
+ [Fact]
+ public void Read_ahead_hands_out_every_byte_then_the_decoders_error()
+ {
+ var inner = new Counting(failAt: 3_456_000);
+ using var stream = Ahead(inner, Ct);
+ var buffer = new byte[70_001];
+ var random = new Random(3);
+ long total = 0;
+ var error = Assert.Throws(() =>
+ {
+ int read;
+ while ((read = stream.Read(buffer, 0, random.Next(1, buffer.Length))) > 0)
+ {
+ for (var i = 0; i < read; i++)
+ Assert.Equal((byte)(total + i), buffer[i]);
+ total += read;
+ }
+ });
+ Assert.Equal("the test data is damaged here", error.Message);
+ // Everything the decoder gave before it failed, nothing more.
+ Assert.Equal(inner.Position, total);
+ Assert.True(total >= 3_456_000);
+ }
+
+ [Fact]
+ public void Read_ahead_ends_where_the_decoder_ends_and_stops_with_the_stream()
+ {
+ var data = LzmaTests.Prose(2_500_000);
+ using (var stream = Ahead(new MemoryStream(data), Ct))
+ Assert.Equal(data, ReadAll(stream));
+
+ // Disposed half way: the worker stops, the decoder is disposed and read no more.
+ var inner = new Counting(failAt: -1);
+ var ahead = Ahead(inner, Ct);
+ var buffer = new byte[100_000];
+ ahead.ReadExactly(buffer);
+ ahead.Dispose();
+ Assert.True(inner.Disposed);
+ var reads = inner.Reads;
+ Thread.Sleep(50);
+ Assert.Equal(reads, inner.Reads);
+ }
+
+ [Fact]
+ public void A_cancelled_read_ahead_stops()
+ {
+ using var cancel = new CancellationTokenSource();
+ using var stream = Ahead(new Counting(failAt: -1), cancel.Token);
+ var buffer = new byte[1 << 16];
+ Assert.True(stream.Read(buffer) > 0);
+ cancel.Cancel();
+ Assert.ThrowsAny(() =>
+ {
+ while (stream.Read(buffer) > 0)
+ {
+ }
+ });
+ }
+
+ [Fact]
+ public async Task BCJ_and_BCJ2_folders_unpack_alike_with_their_coders_on_threads_or_not()
+ {
+ Assert.SkipUnless(SevenZipExe is not null, "7-Zip makes the archives this test reads (optional engine, not installed)");
+ var dir = fx.NewFolder();
+ var source = Directory.CreateDirectory(Path.Combine(dir, "code")).FullName;
+ for (var i = 0; i < 4; i++)
+ File.WriteAllBytes(Path.Combine(source, $"part{i}.bin"), SevenZipReaderTests.X86Like(400_000 + i * 54_321, i));
+ foreach (var filter in new[] { "BCJ", "BCJ2" })
+ {
+ var archive = Path.Combine(dir, filter + ".7z");
+ await Run7zAsync("a", "-t7z", "-mf=" + filter, "-mx5", archive, source);
+ foreach (var ahead in new[] { false, true })
+ {
+ var opened = ReadAheadStream.Opened;
+ ReadAheadStream.TestMode.Value = ahead;
+ var target = fx.NewFolder();
+ try
+ {
+ using var file = File.OpenRead(archive);
+ SevenZipReader.Extract(file, Path.GetFileName(archive), target, Ct);
+ }
+ finally
+ {
+ ReadAheadStream.TestMode.Value = null;
+ }
+ if (ahead)
+ Assert.True(ReadAheadStream.Opened > opened, "no coder was read ahead");
+ foreach (var part in Directory.GetFiles(source))
+ Assert.Equal(File.ReadAllBytes(part), File.ReadAllBytes(Path.Combine(target, "code", Path.GetFileName(part))));
+ }
+ }
+ }
}