diff --git a/src/Filee.Engines/Archives/ArReader.cs b/src/Filee.Engines/Archives/ArReader.cs
index 04dfc3a8b..5ea4774bb 100644
--- a/src/Filee.Engines/Archives/ArReader.cs
+++ b/src/Filee.Engines/Archives/ArReader.cs
@@ -28,7 +28,7 @@ public static void Extract(string archive, string folder, CancellationToken canc
var members = List(stream, name);
ArchiveSafety.CheckListing([.. members.Select(m => new ArchiveItem(m.Name, false, m.Size))], name, stream.Length, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
foreach (var member in members)
{
cancellationToken.ThrowIfCancellationRequested();
diff --git a/src/Filee.Engines/Archives/ArchiveFiles.cs b/src/Filee.Engines/Archives/ArchiveFiles.cs
index 7184def91..fed4c6ce7 100644
--- a/src/Filee.Engines/Archives/ArchiveFiles.cs
+++ b/src/Filee.Engines/Archives/ArchiveFiles.cs
@@ -1,6 +1,8 @@
// File system helpers for the archive engine: copying with a size cap, unique names, and moving an unpacked folder
// into place (with the "one top folder is not nested twice" rule of the Extract preset).
+using System.Buffers;
+
namespace Filee.Engines.Archives;
/// Copy, naming and folder-moving helpers.
@@ -12,20 +14,28 @@ internal static class ArchiveFiles
public static long CopyLimited(Stream from, Stream to, long maxBytes, CancellationToken cancellationToken,
Action? progress = null, UnpackBudget? budget = null, Func? tooLarge = null)
{
- var buffer = new byte[1 << 16];
- long total = 0;
- int read;
- while ((read = from.Read(buffer, 0, buffer.Length)) > 0)
+ // Rented: archives of many small files call this once per file.
+ var buffer = ArrayPool.Shared.Rent(1 << 16);
+ try
+ {
+ long total = 0;
+ int read;
+ while ((read = from.Read(buffer, 0, 1 << 16)) > 0)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ total += read;
+ if (total > maxBytes)
+ throw tooLarge?.Invoke() ?? new InvalidDataException("the data unpacks to more than the archive declares");
+ budget?.Add(read);
+ to.Write(buffer, 0, read);
+ progress?.Invoke(read);
+ }
+ return total;
+ }
+ finally
{
- cancellationToken.ThrowIfCancellationRequested();
- total += read;
- if (total > maxBytes)
- throw tooLarge?.Invoke() ?? new InvalidDataException("the data unpacks to more than the archive declares");
- budget?.Add(read);
- to.Write(buffer, 0, read);
- progress?.Invoke(read);
+ ArrayPool.Shared.Return(buffer);
}
- return total;
}
/// True when nothing is at .
@@ -144,9 +154,10 @@ internal sealed class UniquePaths
/// Path → the next number to try for it.
private readonly Dictionary _next = new(StringComparer.OrdinalIgnoreCase);
- public string Get(string path)
+ /// Paths handed out already that may not be on disk yet (files still being written).
+ public string Get(string path, IReadOnlySet? taken = null)
{
- if (ArchiveFiles.IsFree(path))
+ if (taken?.Contains(path) != true && ArchiveFiles.IsFree(path))
return path;
var dir = Path.GetDirectoryName(path)!;
var stem = Path.GetFileNameWithoutExtension(path);
@@ -154,7 +165,7 @@ public string Get(string path)
for (var n = _next.GetValueOrDefault(path, 1); ; n++)
{
var candidate = Path.Combine(dir, $"{stem}_{n}{ext}");
- if (ArchiveFiles.IsFree(candidate))
+ if (taken?.Contains(candidate) != true && ArchiveFiles.IsFree(candidate))
{
_next[path] = n + 1;
return candidate;
diff --git a/src/Filee.Engines/Archives/ArjReader.cs b/src/Filee.Engines/Archives/ArjReader.cs
index 9b3c90ef8..6b240443e 100644
--- a/src/Filee.Engines/Archives/ArjReader.cs
+++ b/src/Filee.Engines/Archives/ArjReader.cs
@@ -41,7 +41,7 @@ public static void Extract(string archive, string folder, CancellationToken canc
if (entries.FirstOrDefault(e => !e.IsFolder && e.Method > 4) is { } unknown)
throw new NotSupportedException($"the ARJ method {unknown.Method}");
- var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
foreach (var entry in entries)
{
cancellationToken.ThrowIfCancellationRequested();
diff --git a/src/Filee.Engines/Archives/Bzip2Decoder.cs b/src/Filee.Engines/Archives/Bzip2Decoder.cs
index 2fd0a15c1..fb973b8f4 100644
--- a/src/Filee.Engines/Archives/Bzip2Decoder.cs
+++ b/src/Filee.Engines/Archives/Bzip2Decoder.cs
@@ -1,8 +1,21 @@
// bzip2 decompression in managed code, written for ALZip archives: ALZ stores bzip2 blocks without the "BZh" stream
// header, marks each block with "DLZ\x01" (instead of the 48-bit block magic, the block CRC and the "randomised" bit)
-// and ends with "DLZ\x02". Standard bzip2 streams (used by EGG) are read too, and NSIS installers' framing: no stream
-// header, the byte 0x31 before each block (no block CRC, no "randomised" bit) and 0x17 at the end. The block format
-// itself (Huffman coded MTF/RLE2 symbols, Burrows-Wheeler transform, RLE1) is plain bzip2.
+// and ends with "DLZ\x02". Standard bzip2 streams (.bz2, .tar.bz2, 7z and EGG) are read too, and NSIS installers'
+// framing: no stream header, the byte 0x31 before each block (no block CRC, no "randomised" bit) and 0x17 at the end.
+// The block format itself (Huffman coded MTF/RLE2 symbols, Burrows-Wheeler transform, RLE1) is plain bzip2.
+//
+// Each block is decoded in two steps: the Huffman and move-to-front symbols into the transformed block (which must
+// be read in order: where a block ends is only known once it is decoded), then the inverse transform, the run-length
+// coding and the block CRC. Standard streams opened with OpenRead do the second step for several blocks at once on
+// worker threads and hand the blocks out in order; the push API (Decompress) does both in turn. Huffman codes of up
+// to 10 bits are looked up in a table, longer ones are found as the reference decoder does, so any table, even a
+// malformed one, decodes to the same symbols. "Randomised" blocks (bzip2 0.9.0, and encoders that still randomise
+// very repetitive blocks) are not decoded here: the stream reader then starts over with the caller's fallback decoder
+// and skips what it handed out already.
+
+using System.Buffers;
+using System.Collections.Concurrent;
+using System.Runtime.ExceptionServices;
namespace Filee.Engines.Archives;
@@ -18,26 +31,22 @@ internal sealed class Bzip2Decoder
private const int MaxCodeLength = 20;
private const int GroupSize = 50;
private const int MaxSelectors = 18002;
+ private const int FastBits = 10;
/// bzip2's block CRC table (see ), shared with Bzip2Writer.
internal static readonly uint[] BlockCrcTable = CreateBlockCrcTable();
private readonly Stream _input;
- private readonly Stream _output;
- private readonly long _maxOutput;
private readonly byte[] _inBuffer = new byte[64 * 1024];
private int _inPos;
private int _inLength;
private ulong _bits;
- private int _bitCount;
- private long _written;
- private int[] _tt = [];
+ private int _bitCount; // bits held in _bits (the oldest highest)
+ private int _padBits; // of those, zero bits past the end of the input
- private Bzip2Decoder(Stream input, Stream output, long maxOutput)
+ private Bzip2Decoder(Stream input)
{
_input = input;
- _output = output;
- _maxOutput = maxOutput;
}
/// Decompresses to .
@@ -46,12 +55,13 @@ private Bzip2Decoder(Stream input, Stream output, long maxOutput)
/// Number of bytes written.
public static long Decompress(Stream input, Stream output, bool alzFraming, long maxOutput, CancellationToken cancellationToken)
{
- var decoder = new Bzip2Decoder(input, output, maxOutput);
+ var decoder = new Bzip2Decoder(input);
+ var sink = new PushSink(output, maxOutput);
if (alzFraming)
- decoder.DecodeAlz(cancellationToken);
+ decoder.DecodeAlz(sink, cancellationToken);
else
- decoder.DecodeStandard(cancellationToken);
- return decoder._written;
+ decoder.DecodeStandard(block => sink.Block(block), cancellationToken);
+ return sink.Written;
}
/// Decompresses bzip2 data in NSIS's framing (see the file comment) to .
@@ -59,21 +69,36 @@ public static long Decompress(Stream input, Stream output, bool alzFraming, long
/// Number of bytes written.
public static long DecompressNsis(Stream input, Stream output, long maxOutput, CancellationToken cancellationToken)
{
- var decoder = new Bzip2Decoder(input, output, maxOutput);
+ var decoder = new Bzip2Decoder(input);
+ var sink = new PushSink(output, maxOutput);
// NSIS always writes 900k blocks.
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var marker = decoder.ReadBits(8);
if (marker == NsisEndMarker)
- return decoder._written;
+ return sink.Written;
if (marker != NsisBlockMarker)
throw new InvalidDataException("bzip2 block marker missing");
- decoder.DecodeBlock(9, expectedCrc: null);
+ sink.Block(decoder.ReadBlock(9, expectedCrc: null));
}
}
- private void DecodeAlz(CancellationToken cancellationToken)
+ ///
+ /// A stream of the data in the standard bzip2 stream(s) of , with blocks decoded on worker
+ /// threads. Disposing it leaves open.
+ ///
+ ///
+ /// A decoder of the same data from its start, for streams with randomised blocks; without one they are refused.
+ ///
+ public static Stream OpenRead(Stream input, CancellationToken cancellationToken, Func? fallback = null) =>
+ new ReadStream(input, cancellationToken, fallback);
+
+ private const string RandomisedMessage = "randomised bzip2 blocks (bzip2 0.9.0) are not supported";
+
+ private static bool IsRandomised(Exception ex) => ex is InvalidDataException && ex.Message == RandomisedMessage;
+
+ private void DecodeAlz(PushSink sink, CancellationToken cancellationToken)
{
// ALZip always writes 900k blocks and leaves out the stream header.
while (TryReadBits(32, out var marker))
@@ -83,11 +108,12 @@ private void DecodeAlz(CancellationToken cancellationToken)
return;
if (marker != AlzBlockMarker)
throw new InvalidDataException("bzip2 block marker missing");
- DecodeBlock(9, expectedCrc: null);
+ sink.Block(ReadBlock(9, expectedCrc: null));
}
}
- private void DecodeStandard(CancellationToken cancellationToken)
+ /// Reads standard streams, handing every block (not yet transformed back) to .
+ private void DecodeStandard(Action onBlock, CancellationToken cancellationToken)
{
// Several streams may follow each other (pbzip2, "cat a.bz2 b.bz2").
var first = true;
@@ -128,8 +154,8 @@ private void DecodeStandard(CancellationToken cancellationToken)
if (magic != BlockMagic)
throw new InvalidDataException("bzip2 block magic missing");
if (ReadBits(1) != 0)
- throw new InvalidDataException("randomised bzip2 blocks (bzip2 0.9.0) are not supported");
- DecodeBlock(level, crc);
+ throw new InvalidDataException(RandomisedMessage);
+ onBlock(ReadBlock(level, crc));
combined = ((combined << 1) | (combined >> 31)) ^ crc;
}
// The next stream starts on a byte boundary.
@@ -137,14 +163,34 @@ private void DecodeStandard(CancellationToken cancellationToken)
}
}
- /// Decodes one block, starting at origPtr (after magic, CRC and randomised bit).
- private void DecodeBlock(int level, uint? expectedCrc)
+ /// A block after its Huffman and move-to-front step: the transformed bytes (in the low 8 bits of Tt).
+ private sealed class Block(int[] tt, int origPtr, uint? expectedCrc)
+ {
+ public int[] Tt { get; } = tt;
+ public int Length { get; set; }
+ public int OrigPtr { get; } = origPtr;
+ public uint? ExpectedCrc { get; } = expectedCrc;
+ public int[] Counts { get; } = new int[256];
+ }
+
+ /// Decodes the symbols of one block, starting at origPtr (after magic, CRC and randomised bit).
+ private Block ReadBlock(int level, uint? expectedCrc)
{
var blockMax = level * 100_000;
- if (_tt.Length < blockMax)
- _tt = new int[blockMax];
- var tt = _tt;
+ var tt = ArrayPool.Shared.Rent(blockMax);
+ try
+ {
+ return ReadBlock(tt, blockMax, expectedCrc);
+ }
+ catch
+ {
+ ArrayPool.Shared.Return(tt);
+ throw;
+ }
+ }
+ private Block ReadBlock(int[] tt, int blockMax, uint? expectedCrc)
+ {
var origPtr = (int)ReadBits(24);
// Symbols used in this block, as a 16x16 bitmap.
@@ -188,11 +234,8 @@ private void DecodeBlock(int level, uint? expectedCrc)
selectors[i] = value; // bzip2 1.0.8 reads but ignores selectors beyond the limit
}
- // Code lengths, delta coded, and the canonical Huffman decode tables built from them.
- var limit = new int[groupCount, MaxCodeLength + 2];
- var baseTable = new int[groupCount, MaxCodeLength + 2];
- var perm = new int[groupCount, 258];
- var minLengths = new int[groupCount];
+ // Code lengths, delta coded, and the decode tables built from them.
+ var tables = new HuffmanTable[groupCount];
var lengths = new int[alphaSize];
for (var t = 0; t < groupCount; t++)
{
@@ -209,20 +252,20 @@ private void DecodeBlock(int level, uint? expectedCrc)
}
lengths[i] = current;
}
- minLengths[t] = BuildDecodeTable(lengths, alphaSize, t, limit, baseTable, perm);
+ tables[t] = new HuffmanTable(lengths, alphaSize);
}
// MTF / RLE2 symbols → block bytes.
Span mtf = stackalloc byte[256];
for (var i = 0; i < 256; i++)
mtf[i] = (byte)i;
- Span counts = stackalloc int[256];
- counts.Clear();
+ var block = new Block(tt, origPtr, expectedCrc);
+ var counts = block.Counts;
var endOfBlock = inUseCount + 1;
var length = 0;
var groupIndex = -1;
var groupLeft = 0;
- var table = 0;
+ HuffmanTable table = tables[0];
var symbol = NextSymbol();
while (symbol != endOfBlock)
@@ -256,7 +299,15 @@ private void DecodeBlock(int level, uint? expectedCrc)
if (index >= inUseCount)
throw new InvalidDataException("invalid bzip2 symbol");
var front = mtf[index];
- mtf[..index].CopyTo(mtf[1..]);
+ if (index < 16)
+ {
+ for (var k = index; k > 0; k--)
+ mtf[k] = mtf[k - 1];
+ }
+ else
+ {
+ mtf[..index].CopyTo(mtf[1..]);
+ }
mtf[0] = front;
var b = seqToUnseq[front];
counts[b]++;
@@ -265,8 +316,140 @@ private void DecodeBlock(int level, uint? expectedCrc)
}
if (origPtr < 0 || origPtr >= Math.Max(length, 1))
throw new InvalidDataException("invalid bzip2 origin pointer");
+ block.Length = length;
+ return block;
+
+ int NextSymbol()
+ {
+ if (groupLeft == 0)
+ {
+ if (++groupIndex >= selectors.Length)
+ throw new InvalidDataException("bzip2 selectors exhausted");
+ table = tables[selectors[groupIndex]];
+ groupLeft = GroupSize;
+ }
+ groupLeft--;
+ var peek = PeekBits(MaxCodeLength);
+ var entry = table.Fast[peek >> (MaxCodeLength - FastBits)];
+ if (entry >= 0)
+ {
+ SkipBits(entry & 31);
+ return entry >> 5;
+ }
+ return table.Slow(this, peek);
+ }
+ }
+
+ ///
+ /// Canonical Huffman decode tables as in the reference bzip2 (limit / base / perm per code length), plus a table
+ /// of the codes of up to bits found the same way.
+ ///
+ private sealed class HuffmanTable
+ {
+ private readonly int[] _limit = new int[MaxCodeLength + 2];
+ private readonly int[] _base = new int[MaxCodeLength + 2];
+ private readonly int[] _perm = new int[258];
+ private readonly int _minLength;
+
+ /// Symbol << 5 | length for a code of up to FastBits bits, by its next FastBits bits; -1 for longer or invalid codes.
+ public readonly short[] Fast = new short[1 << FastBits];
+
+ public HuffmanTable(int[] lengths, int alphaSize)
+ {
+ var minLength = MaxCodeLength;
+ var maxLength = 0;
+ for (var i = 0; i < alphaSize; i++)
+ {
+ minLength = Math.Min(minLength, lengths[i]);
+ maxLength = Math.Max(maxLength, lengths[i]);
+ }
+
+ var p = 0;
+ for (var len = minLength; len <= maxLength; len++)
+ for (var symbol = 0; symbol < alphaSize; symbol++)
+ if (lengths[symbol] == len)
+ _perm[p++] = symbol;
+
+ Span countAtLength = stackalloc int[MaxCodeLength + 2];
+ countAtLength.Clear();
+ for (var i = 0; i < alphaSize; i++)
+ countAtLength[lengths[i] + 1]++;
+ for (var i = 1; i < MaxCodeLength + 2; i++)
+ countAtLength[i] += countAtLength[i - 1];
+
+ for (var i = 0; i < MaxCodeLength + 2; i++)
+ {
+ _limit[i] = -1; // lengths outside min..max never match, so decoding continues to longer codes
+ _base[i] = countAtLength[i];
+ }
- // Inverse Burrows-Wheeler transform: link every position to its successor.
+ var code = 0;
+ for (var len = minLength; len <= maxLength; len++)
+ {
+ code += countAtLength[len + 1] - countAtLength[len];
+ _limit[len] = code - 1;
+ code <<= 1;
+ }
+ for (var len = minLength + 1; len <= maxLength; len++)
+ _base[len] = ((_limit[len - 1] + 1) << 1) - countAtLength[len];
+ _minLength = minLength;
+
+ // The fast table: for every FastBits-bit prefix, the code the reference search finds within those bits.
+ for (var prefix = 0; prefix < Fast.Length; prefix++)
+ {
+ short entry = -1;
+ for (var len = minLength; len <= FastBits; len++)
+ {
+ var c = prefix >> (FastBits - len);
+ if (c <= _limit[len])
+ {
+ var slot = c - _base[len];
+ if (slot is >= 0 and < 258)
+ entry = (short)(_perm[slot] << 5 | len);
+ break;
+ }
+ }
+ Fast[prefix] = entry;
+ }
+ }
+
+ /// The reference search, for codes the fast table doesn't hold: holds the next 20 bits.
+ public int Slow(Bzip2Decoder decoder, uint peek)
+ {
+ var bitsUsed = _minLength;
+ var code = (int)(peek >> (MaxCodeLength - bitsUsed));
+ while (bitsUsed <= MaxCodeLength && code > _limit[bitsUsed])
+ {
+ bitsUsed++;
+ if (bitsUsed <= MaxCodeLength)
+ code = (int)(peek >> (MaxCodeLength - bitsUsed));
+ }
+ if (bitsUsed > MaxCodeLength)
+ {
+ // The reference decoder reads all 20 bits (and more) before it gives up.
+ decoder.SkipBits(MaxCodeLength);
+ throw new InvalidDataException("invalid bzip2 Huffman code");
+ }
+ decoder.SkipBits(bitsUsed);
+ var slot = code - _base[bitsUsed];
+ if (slot is < 0 or >= 258)
+ throw new InvalidDataException("invalid bzip2 Huffman code");
+ return _perm[slot];
+ }
+ }
+
+ ///
+ /// The second step of a block: the inverse Burrows-Wheeler transform, the run-length coding (4 equal bytes + a
+ /// count byte) and the block CRC. Writes into (grown as needed, rented) and returns the
+ /// byte count; past bytes it stops with an error.
+ ///
+ private static int Inverse(Block block, ref byte[] output, long limit)
+ {
+ var tt = block.Tt;
+ var length = block.Length;
+ var counts = block.Counts;
+
+ // Link every position to its successor.
Span starts = stackalloc int[256];
var sum = 0;
for (var i = 0; i < 256; i++)
@@ -280,13 +463,14 @@ private void DecodeBlock(int level, uint? expectedCrc)
tt[starts[b]++] |= i << 8;
}
- // Walk the chain and undo the initial run-length coding (4 equal bytes + a count byte).
+ // Walk the chain and undo the initial run-length coding.
+ var table = BlockCrcTable;
var crc = 0xFFFFFFFFu;
- var buffer = new byte[64 * 1024];
var used = 0;
- var position = length == 0 ? 0 : tt[origPtr] >> 8;
+ var position = length == 0 ? 0 : tt[block.OrigPtr] >> 8;
var previous = -1;
var repeat = 0;
+ var buffer = output;
for (var i = 0; i < length; i++)
{
var entry = tt[position];
@@ -294,107 +478,241 @@ private void DecodeBlock(int level, uint? expectedCrc)
position = entry >> 8;
if (repeat == 4)
{
+ if (used + b > buffer.Length)
+ buffer = Grow(buffer, used + b, used, limit);
for (var k = 0; k < b; k++)
- Emit((byte)previous);
+ {
+ crc = (crc << 8) ^ table[(crc >> 24) ^ (uint)previous];
+ buffer[used++] = (byte)previous;
+ }
repeat = 0;
previous = -1;
continue;
}
repeat = b == previous ? repeat + 1 : 1;
previous = b;
- Emit((byte)b);
+ if (used == buffer.Length)
+ buffer = Grow(buffer, used + 1, used, limit);
+ crc = (crc << 8) ^ table[(crc >> 24) ^ (uint)b];
+ buffer[used++] = (byte)b;
}
- Flush();
+ output = buffer;
+ if (used > limit)
+ throw new InvalidDataException("the data unpacks to more than the archive declares");
+ if (block.ExpectedCrc is { } expected && ~crc != expected)
+ throw new BlockCrcException(used);
+ return used;
+ }
- if (expectedCrc is { } expected && ~crc != expected)
- throw new InvalidDataException("bzip2 block checksum mismatch");
+ private static byte[] Grow(byte[] buffer, int needed, int used, long limit)
+ {
+ if (needed > limit)
+ throw new InvalidDataException("the data unpacks to more than the archive declares");
+ var larger = ArrayPool.Shared.Rent(Math.Max(needed, buffer.Length * 2));
+ buffer.AsSpan(0, used).CopyTo(larger);
+ if (buffer.Length > 0)
+ ArrayPool.Shared.Return(buffer);
+ return larger;
+ }
- void Emit(byte value)
+ /// A block whose bytes came out (all of them, ) but whose CRC doesn't match.
+ private sealed class BlockCrcException(int count) : Exception("bzip2 block checksum mismatch")
+ {
+ public int Count { get; } = count;
+ }
+
+ /// Writes blocks to a stream, as they are decoded.
+ private sealed class PushSink(Stream output, long maxOutput)
+ {
+ private byte[] _buffer = [];
+
+ public long Written { get; private set; }
+
+ public void Block(Block block)
{
- crc = (crc << 8) ^ BlockCrcTable[(crc >> 24) ^ value];
- buffer[used++] = value;
- if (used == buffer.Length)
- Flush();
+ try
+ {
+ if (_buffer.Length < block.Length + (block.Length >> 3))
+ {
+ if (_buffer.Length > 0)
+ ArrayPool.Shared.Return(_buffer);
+ _buffer = ArrayPool.Shared.Rent(Math.Max(1 << 16, block.Length + (block.Length >> 3)));
+ }
+ int count;
+ try
+ {
+ count = Inverse(block, ref _buffer, maxOutput - Written);
+ }
+ catch (BlockCrcException ex)
+ {
+ // The bytes go out before the mismatch is reported, as they always did.
+ Write(ex.Count);
+ throw new InvalidDataException(ex.Message);
+ }
+ Write(count);
+ }
+ finally
+ {
+ ArrayPool.Shared.Return(block.Tt);
+ }
}
- void Flush()
+ private void Write(int count)
{
- if (used == 0)
- return;
- _written += used;
- if (_written > _maxOutput)
- throw new InvalidDataException("the data unpacks to more than the archive declares");
- _output.Write(buffer, 0, used);
- used = 0;
+ Written += count;
+ output.Write(_buffer, 0, count);
}
+ }
- int NextSymbol()
+ /// Standard bzip2 as a stream: blocks are read in turn and transformed back on worker threads.
+ private sealed class ReadStream : ReadOnlyDecoderStream
+ {
+ private readonly BlockingCollection> _blocks;
+ private readonly CancellationTokenSource _stop;
+ private readonly CancellationToken _cancellationToken;
+ private readonly Task _producer;
+ private readonly Func? _fallback;
+ private Decoded? _current;
+ private int _position;
+ private long _handedOut;
+ private Stream? _other; // the fallback decoder, once a randomised block came
+
+ public ReadStream(Stream input, CancellationToken cancellationToken, Func? fallback)
{
- if (groupLeft == 0)
+ _cancellationToken = cancellationToken;
+ _fallback = fallback;
+ _stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ _blocks = new BlockingCollection>(Math.Max(2, Environment.ProcessorCount));
+ var token = _stop.Token;
+ _producer = Task.Factory.StartNew(() => Produce(input, token), CancellationToken.None, TaskCreationOptions.LongRunning,
+ TaskScheduler.Default);
+ }
+
+ private void Produce(Stream input, CancellationToken token)
+ {
+ try
{
- if (++groupIndex >= selectors.Length)
- throw new InvalidDataException("bzip2 selectors exhausted");
- table = selectors[groupIndex];
- groupLeft = GroupSize;
+ new Bzip2Decoder(input).DecodeStandard(block => _blocks.Add(Task.Run(() => Decode(block), CancellationToken.None), token), token);
}
- groupLeft--;
- var bitsUsed = minLengths[table];
- var code = (int)ReadBits(bitsUsed);
- while (bitsUsed <= MaxCodeLength && code > limit[table, bitsUsed])
+ catch (Exception ex) when (!token.IsCancellationRequested)
{
- bitsUsed++;
- code = (code << 1) | (int)ReadBits(1);
+ // After the blocks before it; waits for room like they do (never dropped).
+ try
+ {
+ _blocks.Add(Task.FromException(ex), token);
+ }
+ catch (OperationCanceledException)
+ {
+ // The reader is gone.
+ }
+ }
+ catch (OperationCanceledException)
+ {
+ // Stopped: the reader is gone or the work was cancelled (the reader sees that itself).
+ }
+ finally
+ {
+ _blocks.CompleteAdding();
}
- if (bitsUsed > MaxCodeLength)
- throw new InvalidDataException("invalid bzip2 Huffman code");
- var slot = code - baseTable[table, bitsUsed];
- if (slot is < 0 or >= 258)
- throw new InvalidDataException("invalid bzip2 Huffman code");
- return perm[table, slot];
}
- }
- /// Canonical Huffman decode tables as in the reference bzip2 (limit / base / perm per code length).
- /// The shortest code length.
- private static int BuildDecodeTable(int[] lengths, int alphaSize, int t, int[,] limit, int[,] baseTable, int[,] perm)
- {
- var minLength = MaxCodeLength;
- var maxLength = 0;
- for (var i = 0; i < alphaSize; i++)
+ private static Decoded Decode(Block block)
{
- minLength = Math.Min(minLength, lengths[i]);
- maxLength = Math.Max(maxLength, lengths[i]);
+ var output = ArrayPool.Shared.Rent(Math.Max(1 << 16, block.Length + (block.Length >> 3)));
+ try
+ {
+ var count = Inverse(block, ref output, long.MaxValue);
+ return new Decoded(output, count, null);
+ }
+ catch (BlockCrcException ex)
+ {
+ return new Decoded(output, ex.Count, ExceptionDispatchInfo.Capture(new InvalidDataException(ex.Message)));
+ }
+ catch (Exception ex)
+ {
+ return new Decoded(output, 0, ExceptionDispatchInfo.Capture(ex));
+ }
+ finally
+ {
+ ArrayPool.Shared.Return(block.Tt);
+ }
}
- var p = 0;
- for (var len = minLength; len <= maxLength; len++)
- for (var symbol = 0; symbol < alphaSize; symbol++)
- if (lengths[symbol] == len)
- perm[t, p++] = symbol;
-
- Span countAtLength = stackalloc int[MaxCodeLength + 2];
- countAtLength.Clear();
- for (var i = 0; i < alphaSize; i++)
- countAtLength[lengths[i] + 1]++;
- for (var i = 1; i < MaxCodeLength + 2; i++)
- countAtLength[i] += countAtLength[i - 1];
+ public override int Read(Span buffer)
+ {
+ if (buffer.IsEmpty)
+ return 0;
+ while (true)
+ {
+ if (_other is not null)
+ return _other.Read(buffer);
+ if (_current is { } current)
+ {
+ if (_position < current.Count)
+ {
+ var count = Math.Min(buffer.Length, current.Count - _position);
+ current.Output.AsSpan(_position, count).CopyTo(buffer);
+ _position += count;
+ _handedOut += count;
+ return count;
+ }
+ current.Error?.Throw();
+ ArrayPool.Shared.Return(current.Output);
+ _current = null;
+ }
+ _cancellationToken.ThrowIfCancellationRequested();
+ if (!_blocks.TryTake(out var next, Timeout.Infinite, _cancellationToken))
+ return 0;
+ try
+ {
+ _current = next.GetAwaiter().GetResult();
+ }
+ catch (InvalidDataException ex) when (IsRandomised(ex) && _fallback is not null)
+ {
+ StartOver();
+ continue;
+ }
+ _position = 0;
+ }
+ }
- for (var i = 0; i < MaxCodeLength + 2; i++)
+ /// A randomised block: the fallback decoder from the start, past what was handed out already.
+ private void StartOver()
{
- limit[t, i] = -1; // lengths outside min..max never match, so decoding continues to longer codes
- baseTable[t, i] = countAtLength[i];
+ _producer.Wait(_cancellationToken); // it stopped at the block; the input is free
+ var other = _fallback!();
+ var skip = new byte[1 << 16];
+ for (var left = _handedOut; left > 0;)
+ {
+ var read = other.Read(skip, 0, (int)Math.Min(skip.Length, left));
+ if (read <= 0)
+ throw new InvalidDataException("bzip2 data ends unexpectedly");
+ left -= read;
+ }
+ _other = other;
}
- var code = 0;
- for (var len = minLength; len <= maxLength; len++)
+ protected override void Dispose(bool disposing)
{
- code += countAtLength[len + 1] - countAtLength[len];
- limit[t, len] = code - 1;
- code <<= 1;
+ if (disposing)
+ {
+ _stop.Cancel();
+ try
+ {
+ _producer.Wait();
+ }
+ catch (AggregateException)
+ {
+ // The producer catches everything it throws; nothing to report on the way out.
+ }
+ _stop.Dispose();
+ _blocks.Dispose();
+ _other?.Dispose();
+ }
+ base.Dispose(disposing);
}
- for (var len = minLength + 1; len <= maxLength; len++)
- baseTable[t, len] = ((limit[t, len - 1] + 1) << 1) - countAtLength[len];
- return minLength;
+
+ private sealed record Decoded(byte[] Output, int Count, ExceptionDispatchInfo? Error);
}
private uint ReadBits(int count)
@@ -407,24 +725,55 @@ private uint ReadBits(int count)
/// Reads up to 32 bits, most significant first; false when the input is exhausted.
private bool TryReadBits(int count, out uint value)
{
- while (_bitCount < count)
+ if (_bitCount - _padBits < count)
+ Refill();
+ if (_bitCount - _padBits < count)
{
- if (_inPos == _inLength)
+ value = 0;
+ return false;
+ }
+ _bitCount -= count;
+ value = (uint)((_bits >> _bitCount) & ((1UL << count) - 1));
+ return true;
+ }
+
+ /// The next bits (up to 32) without using them; zeros past the end of the input.
+ private uint PeekBits(int count)
+ {
+ if (_bitCount < count)
+ Refill();
+ return (uint)((_bits >> (_bitCount - count)) & ((1UL << count) - 1));
+ }
+
+ /// Uses bits seen with ; past the end of the input the data is cut short.
+ private void SkipBits(int count)
+ {
+ _bitCount -= count;
+ if (_bitCount < _padBits)
+ throw new InvalidDataException("bzip2 data ends unexpectedly");
+ }
+
+ /// Tops the bit buffer up to more than 56 bits, with zero bits once the input has ended.
+ private void Refill()
+ {
+ while (_bitCount <= 56)
+ {
+ if (_inPos == _inLength && _padBits == 0)
{
_inLength = _input.Read(_inBuffer, 0, _inBuffer.Length);
_inPos = 0;
- if (_inLength == 0)
- {
- value = 0;
- return false;
- }
}
- _bits = (_bits << 8) | _inBuffer[_inPos++];
+ if (_inPos < _inLength)
+ {
+ _bits = (_bits << 8) | _inBuffer[_inPos++];
+ }
+ else
+ {
+ _bits <<= 8;
+ _padBits += 8;
+ }
_bitCount += 8;
}
- _bitCount -= count;
- value = (uint)((_bits >> _bitCount) & ((1UL << count) - 1));
- return true;
}
/// bzip2's block CRC: CRC-32 with the polynomial 0x04C11DB7, most significant bit first.
diff --git a/src/Filee.Engines/Archives/CompressedFile.cs b/src/Filee.Engines/Archives/CompressedFile.cs
index e35e050e1..95319fe87 100644
--- a/src/Filee.Engines/Archives/CompressedFile.cs
+++ b/src/Filee.Engines/Archives/CompressedFile.cs
@@ -1,7 +1,9 @@
// Unpacks a compressed single file (.gz, .bz2, .xz, .lzma, .Z, .zst and the outer layer of .tar.gz, .tar.bz2, .tar.xz
-// and .tar.Z) in-process: gzip with .NET's own decoder, bzip2 and zstd with SharpCompress (MIT), xz, LZMA and Unix
-// compress with Filee's own decoders (XzReader, LzmaRawReader, UnixCompressStream). None of these formats declares a size it must keep (gzip's is the
-// size modulo 4 GiB), so what comes out is counted by UnpackBudget against bombs and a full drive.
+// and .tar.Z) in-process: gzip with .NET's own decoder, zstd with SharpCompress (MIT), bzip2, xz, LZMA and Unix
+// compress with Filee's own decoders (Bzip2Decoder, XzReader, LzmaRawReader, UnixCompressStream; bzip2 with
+// randomised blocks from old encoders goes back to SharpCompress). None of these
+// formats declares a size it must keep (gzip's is the size modulo 4 GiB), so what comes out is counted by
+// UnpackBudget against bombs and a full drive.
using System.IO.Compression;
using Filee.Engines.Archives.Lzma;
@@ -71,7 +73,15 @@ internal static Stream Open(string format, Stream input, string name, ref long d
return new GZipStream(input, CompressionMode.Decompress, leaveOpen: true);
case "bz2":
case "tbz2":
- return new BZip2Stream(new NonClosingStream(input), SharpCompressMode.Decompress, decompressConcatenated: true);
+ {
+ // SharpCompress reads the randomised blocks of old encoders, which Filee's decoder leaves out.
+ var start = input.CanSeek ? input.Position : -1;
+ return Bzip2Decoder.OpenRead(input, CancellationToken.None, start < 0 ? null : () =>
+ {
+ input.Position = start;
+ return new BZip2Stream(new NonClosingStream(input), SharpCompressMode.Decompress, decompressConcatenated: true);
+ });
+ }
case "xz":
case "txz":
return new XzReader(input, name);
diff --git a/src/Filee.Engines/Archives/CpioReader.cs b/src/Filee.Engines/Archives/CpioReader.cs
index 8d1e64b7d..3ba59dc26 100644
--- a/src/Filee.Engines/Archives/CpioReader.cs
+++ b/src/Filee.Engines/Archives/CpioReader.cs
@@ -49,7 +49,7 @@ public static void Extract(string archive, string folder, CancellationToken canc
ArchiveSafety.CheckListing([.. entries.Select(e => new ArchiveItem(e.Name, e.Type == 0x4000, SizeOf(e), IsLink: e.Type is not (0x4000 or 0x8000)))],
name, stream.Length, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
// Hard links of a newc group without data yet: (device, inode) → names waiting for it.
var waiting = new Dictionary<(long, long), List>();
foreach (var entry in entries)
diff --git a/src/Filee.Engines/Archives/DiskImageReader.cs b/src/Filee.Engines/Archives/DiskImageReader.cs
index 6342d0cc4..cac84c4ac 100644
--- a/src/Filee.Engines/Archives/DiskImageReader.cs
+++ b/src/Filee.Engines/Archives/DiskImageReader.cs
@@ -198,7 +198,7 @@ private static void ExtractVolumes(List<(DiskVolume Volume, Kind Kind)> volumes,
ArchiveSafety.CheckListing([.. nodes.Select(n => new ArchiveItem(n.Path, n.IsFolder, n.IsFolder || n.IsLink ? 0 : n.Size, IsLink: n.IsLink))],
name, archiveLength, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(name, archiveLength, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, archiveLength, folder), cancellationToken);
foreach (var node in nodes)
{
cancellationToken.ThrowIfCancellationRequested();
diff --git a/src/Filee.Engines/Archives/EntryWriter.cs b/src/Filee.Engines/Archives/EntryWriter.cs
index 6f6d41011..e98c63e2b 100644
--- a/src/Filee.Engines/Archives/EntryWriter.cs
+++ b/src/Filee.Engines/Archives/EntryWriter.cs
@@ -2,18 +2,45 @@
// clashing names are numbered as 7-Zip numbers them, sizes and CRC-32s are checked, hard links become copies of the
// file they name, links are never created, and folder times are set last (writing files into a folder changes its
// time).
+//
+// Small files are written on worker threads: creating a file costs the system (and a virus scanner looking at every
+// new file) far more than the few KB in it, and archives of thousands of small files spend most of their time there.
+// The reader's thread still reads each one whole, counts it against the budget and checks its size and CRC, so every
+// error comes where it always did; it only hands the bytes over. Names are reserved as they are handed out, so the
+// numbering of clashing names and "a file of that name came first" are decided as if every file were already there.
+// Finish waits for the writes and reports the first one that failed; Dispose waits for them on the way out of an
+// error.
+
+using System.Buffers;
+using System.Runtime.ExceptionServices;
namespace Filee.Engines.Archives;
/// The output side of Filee's own archive readers and the SharpCompress ones.
-internal sealed class EntryWriter(string root, UnpackBudget budget, CancellationToken cancellationToken)
+internal sealed class EntryWriter(string root, UnpackBudget budget, CancellationToken cancellationToken) : IDisposable
{
+ /// Files up to this size are written on worker threads.
+ private const int SmallFile = 1 << 20;
+
+ /// Tests turn the worker threads off.
+ internal static readonly AsyncLocal TestOneThread = new();
+
private readonly List<(string Path, DateTime? Modified)> _folders = [];
private readonly UniquePaths _unique = new();
+ private string? _lastFolder; // the folder the last file went into, known to exist
/// Entry path (as spells it) → the file written for it, for hard links.
private readonly Dictionary _files = new(StringComparer.Ordinal);
+ /// Every file path handed out, written or still being written.
+ private readonly HashSet _reserved = new(StringComparer.OrdinalIgnoreCase);
+
+ private readonly List _pending = [];
+ private readonly SemaphoreSlim _slots = new(Math.Clamp(Environment.ProcessorCount * 2, 2, 32));
+ private readonly bool _threads = !(TestOneThread.Value ?? Environment.ProcessorCount == 1);
+ private readonly string _root = Path.GetFullPath(root);
+ private ExceptionDispatchInfo? _failed;
+
private string ArchiveName => budget.ArchiveName;
/// Creates a folder entry ("./" and other empty names are the root itself and are left out).
@@ -22,7 +49,7 @@ public void AddFolder(string name, DateTime? modified)
if (Key(name).Length == 0)
return;
var path = ArchiveSafety.ResolveEntryPath(root, name, ArchiveName);
- if (File.Exists(path))
+ if (_reserved.Contains(path) || File.Exists(path))
return; // a file of that name came first; what goes into the folder fails with a clear error later
Directory.CreateDirectory(path);
_folders.Add((path, modified));
@@ -30,25 +57,103 @@ public void AddFolder(string name, DateTime? modified)
///
/// Writes one file from , which must hold exactly bytes (-1: not
- /// known) with the CRC-32 (null: none to check). Returns the file written.
+ /// known) with the CRC-32 (null: none to check). Returns the file written (small files may
+ /// still be on their way until ).
///
public string AddFile(string name, Stream data, long size, uint? crc, DateTime? modified)
{
+ _failed?.Throw();
var path = Target(name);
+ if (_threads && size is > 0 and <= SmallFile)
+ WriteLater(name, path, data, (int)size, crc, modified);
+ else
+ WriteNow(name, path, data, size, crc, modified);
+ _files[Key(name)] = path;
+ return path;
+ }
+
+ private void WriteNow(string name, string path, Stream data, long size, uint? crc, DateTime? modified)
+ {
var checksum = crc is null ? null : new Crc32();
- long written;
- using (var file = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16))
+ using var file = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16);
+ Stream output = checksum is null ? file : new CrcStream(new NonClosingStream(file), checksum);
+ var written = ArchiveFiles.CopyLimited(data, output, size >= 0 ? size : long.MaxValue, cancellationToken, budget: budget);
+ Check(name, size, written, crc, checksum);
+ // The time through the open file (after the last write reached it): no second open per file.
+ file.Flush();
+ SetTime(file, modified);
+ }
+
+ /// Reads the file here (budget, size and CRC checked as it is read) and writes it on a worker thread.
+ private void WriteLater(string name, string path, Stream data, int size, uint? crc, DateTime? modified)
+ {
+ var buffer = ArrayPool.Shared.Rent(size + 1);
+ try
+ {
+ var checksum = crc is null ? null : new Crc32();
+ using var memory = new MemoryStream(buffer, 0, buffer.Length, writable: true);
+ Stream output = checksum is null ? memory : new CrcStream(new NonClosingStream(memory), checksum);
+ var written = ArchiveFiles.CopyLimited(data, output, size, cancellationToken, budget: budget);
+ Check(name, size, written, crc, checksum);
+ }
+ catch
{
- Stream output = checksum is null ? file : new CrcStream(new NonClosingStream(file), checksum);
- written = ArchiveFiles.CopyLimited(data, output, size >= 0 ? size : long.MaxValue, cancellationToken, budget: budget);
+ ArrayPool.Shared.Return(buffer);
+ throw;
}
+ _slots.Wait(cancellationToken);
+ _pending.Add(Task.Run(() =>
+ {
+ try
+ {
+ using var file = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1);
+ file.Write(buffer, 0, size);
+ SetTime(file, modified);
+ }
+ finally
+ {
+ ArrayPool.Shared.Return(buffer);
+ _slots.Release();
+ }
+ }, CancellationToken.None));
+ if (_pending.Count >= 256)
+ Collect();
+ }
+
+ private void Check(string name, long size, long written, uint? crc, Crc32? checksum)
+ {
if (size >= 0 && written != size)
throw ArchiveSafety.Damaged(ArchiveName, $"{name} is shorter than the archive says");
if (crc is { } expected && checksum!.Value != expected)
throw ArchiveSafety.Damaged(ArchiveName, $"checksum error in {name}");
- SetTime(path, modified);
- _files[Key(name)] = path;
- return path;
+ }
+
+ /// Forgets the writes that are done; the first one that failed is kept, to be thrown.
+ private void Collect()
+ {
+ _pending.RemoveAll(task =>
+ {
+ if (!task.IsCompleted)
+ return false;
+ if (task.Exception is { } error && _failed is null)
+ _failed = ExceptionDispatchInfo.Capture(error.InnerException ?? error);
+ return true;
+ });
+ }
+
+ /// Waits for every write on the worker threads; throws the first that failed.
+ private void Drain()
+ {
+ try
+ {
+ Task.WaitAll(_pending);
+ }
+ catch (AggregateException)
+ {
+ // Collected below.
+ }
+ Collect();
+ _failed?.Throw();
}
///
@@ -59,18 +164,42 @@ public void AddHardLink(string name, string target, DateTime? modified)
{
if (!_files.TryGetValue(Key(target), out var source))
return;
+ Drain(); // the file it copies may still be on its way
using var data = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16);
AddFile(name, data, data.Length, null, modified);
}
- /// Sets the folder times; call after the last entry.
+ /// A copy, under , of the file that AddFile returned.
+ public void AddCopy(string name, string written, DateTime? modified)
+ {
+ Drain(); // it may still be on its way
+ using var data = new FileStream(written, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16);
+ AddFile(name, data, data.Length, null, modified);
+ }
+
+ /// Waits for the files still being written and sets the folder times; call after the last entry.
public void Finish()
{
+ Drain();
foreach (var (path, modified) in _folders)
if (modified is { } time)
Directory.SetLastWriteTime(path, time);
}
+ /// Waits for the files still being written (their errors are dropped: something failed already, or Finish reported them).
+ public void Dispose()
+ {
+ try
+ {
+ Drain();
+ }
+ catch (Exception)
+ {
+ // On the way out of another error, or reported by Finish already.
+ }
+ _slots.Dispose();
+ }
+
/// The entry path with "/" separators and without "." and empty parts.
public static string Key(string name) =>
string.Join('/', name.Split(['/', '\\'], StringSplitOptions.RemoveEmptyEntries).Where(part => part != "."));
@@ -80,17 +209,33 @@ private string Target(string name)
if (Key(name).Length == 0)
throw ArchiveSafety.Damaged(ArchiveName, "an entry has no name");
var path = ArchiveSafety.ResolveEntryPath(root, name, ArchiveName);
- Directory.CreateDirectory(Path.GetDirectoryName(path)!);
- return _unique.Get(path);
+ var folder = Path.GetDirectoryName(path)!;
+ if (folder != _lastFolder)
+ {
+ // A file still being written where a folder must go: let it land, so making the folder fails as it would.
+ for (var up = folder; up.Length > _root.Length && _pending.Count > 0; up = Path.GetDirectoryName(up) ?? "")
+ {
+ if (_reserved.Contains(up))
+ {
+ Drain();
+ break;
+ }
+ }
+ Directory.CreateDirectory(folder);
+ _lastFolder = folder;
+ }
+ var unique = _unique.Get(path, _reserved);
+ _reserved.Add(unique);
+ return unique;
}
- private static void SetTime(string path, DateTime? modified)
+ private static void SetTime(FileStream file, DateTime? modified)
{
if (modified is not { } time || time.Year < 1601 || time.Year > 9999)
return;
try
{
- File.SetLastWriteTime(path, time);
+ File.SetLastWriteTime(file.SafeFileHandle, time);
}
catch (ArgumentOutOfRangeException) { }
}
diff --git a/src/Filee.Engines/Archives/IsoReader.cs b/src/Filee.Engines/Archives/IsoReader.cs
index 3d9f91b28..34a7ba1c7 100644
--- a/src/Filee.Engines/Archives/IsoReader.cs
+++ b/src/Filee.Engines/Archives/IsoReader.cs
@@ -71,7 +71,7 @@ public static void Extract(string image, string folder, CancellationToken cancel
ArchiveSafety.CheckListing([.. nodes.Select(n => new ArchiveItem(n.Path, n.IsFolder, n.IsFolder || n.IsLink ? 0 : n.Size, IsLink: n.IsLink))],
name, stream.Length, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
foreach (var node in nodes)
{
cancellationToken.ThrowIfCancellationRequested();
diff --git a/src/Filee.Engines/Archives/LhaReader.cs b/src/Filee.Engines/Archives/LhaReader.cs
index f11c01aa7..70ac5ea16 100644
--- a/src/Filee.Engines/Archives/LhaReader.cs
+++ b/src/Filee.Engines/Archives/LhaReader.cs
@@ -49,7 +49,7 @@ public static void Extract(string archive, string folder, CancellationToken canc
if (entries.FirstOrDefault(e => !e.IsFolder && !e.IsLink && DictionaryBits(e.Method) is null) is { } unknown)
throw new NotSupportedException($"the LHA method {unknown.Method}");
- var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
foreach (var entry in entries)
{
cancellationToken.ThrowIfCancellationRequested();
diff --git a/src/Filee.Engines/Archives/LibraryArchives.cs b/src/Filee.Engines/Archives/LibraryArchives.cs
index 6f741374f..67baa6484 100644
--- a/src/Filee.Engines/Archives/LibraryArchives.cs
+++ b/src/Filee.Engines/Archives/LibraryArchives.cs
@@ -139,7 +139,7 @@ private static void Extract(IArchive archive, string name, long length, string f
ArchiveSafety.CheckListing([.. entries.Select(e => new ArchiveItem(e.Key ?? "", e.IsDirectory, e.IsDirectory ? 0 : e.Size, Encrypted(archive, e), IsLink(e)))],
name, length, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(name, length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, length, folder), cancellationToken);
var checksCrc = archive.Type is ArchiveType.Zip or ArchiveType.SevenZip;
if (archive.IsSolid || archive.Type == ArchiveType.SevenZip)
{
diff --git a/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs b/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs
new file mode 100644
index 000000000..63c1477f8
--- /dev/null
+++ b/src/Filee.Engines/Archives/Lzma/Lzma2Parallel.cs
@@ -0,0 +1,327 @@
+// LZMA2 decoding on several threads. A chunk that resets the dictionary (0x01, or an LZMA chunk with control 0xE0 and
+// up) starts data that needs nothing from before it: the state, the properties and the window all start over. So the
+// chunks from one such reset to the next (a segment) can be decoded side by side, each by its own Lzma2Reader, and
+// handed out in order. 7-Zip writes a reset at every block when it compresses with several threads, xz and Filee's
+// own writer start every block with one.
+//
+// - 7z: the packed stream of an LZMA2 coder lies in the archive, so the chunk headers are read first (they say how
+// long every chunk is) to find the segments; a stream with a single segment, or anything the headers don't explain,
+// is decoded the usual way, which also gives the usual errors.
+// - .xz: every block is its own LZMA2 data, and a block header that names both sizes lets the reader take the block
+// whole and go on to the next (XzReader).
+// What a segment decodes to is held in memory until it is read, so the threads and the size of a segment are
+// bounded: at most one segment per processor, all of them in an eighth of the memory, none past 512 MiB. A segment
+// that fails on its thread is decoded again, from its start, on the reader's thread as it is read: so damaged data
+// gives the very bytes and the very error, at the same place, as the one-thread decoder.
+
+using System.Runtime.ExceptionServices;
+
+namespace Filee.Engines.Archives.Lzma;
+
+/// Decodes LZMA2 segments on worker threads.
+internal static class Lzma2Parallel
+{
+ /// The largest segment decoded on its own thread.
+ public const int MaxSegment = 512 << 20;
+
+ /// Segments smaller than this are joined with the next (fewer, larger tasks).
+ private const int MinSegment = 4 << 20;
+
+ /// Tests turn the threads off (false) or on for any data (true).
+ internal static readonly AsyncLocal TestMode = new();
+
+ /// True when decoding on several threads is allowed at all.
+ public static bool Enabled => TestMode.Value ?? Environment.ProcessorCount > 1;
+
+ /// True when tests ask for threads even where they would not pay off.
+ public static bool Forced => TestMode.Value == true;
+
+ /// 7z coders decoded on several threads so far (for tests).
+ internal static int Opened;
+
+ /// Memory the decoded segments of one stream may hold at once: an eighth of what the machine has.
+ public static long Budget
+ {
+ get
+ {
+ try
+ {
+ var total = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes;
+ return total > 0 ? total / 8 : 256L << 20;
+ }
+ catch (Exception ex) when (ex is InvalidOperationException or PlatformNotSupportedException)
+ {
+ return 256L << 20;
+ }
+ }
+ }
+
+ ///
+ /// Segments that fit in the memory budget at once, at most one per processor (the one being read counts too), for
+ /// segments of up to bytes from bytes.
+ ///
+ public static int Workers(long unpacked, long packed, uint dictionary)
+ {
+ var each = unpacked + packed + Math.Min(dictionary, unpacked);
+ var byMemory = Budget / Math.Max(1, each) - 1;
+ var workers = (int)Math.Min(Environment.ProcessorCount, byMemory);
+ return Forced ? Math.Max(2, workers) : workers;
+ }
+
+ ///
+ /// Opens the LZMA2 data of a 7z coder in for decoding on several threads; null when it
+ /// has fewer than two segments or doesn't read as expected (the caller then decodes it the usual way).
+ ///
+ public static Stream? TryOpen(PackedWindow packed, uint dictionary, long size, CancellationToken cancellationToken)
+ {
+ if (!Enabled || size <= 0)
+ return null;
+ var segments = Scan(packed, size);
+ if (segments is null || segments.Count < 2)
+ return null;
+ long largest = 0, largestPacked = 0, totalPacked = 0;
+ foreach (var segment in segments)
+ {
+ largest = Math.Max(largest, segment.Unpacked);
+ largestPacked = Math.Max(largestPacked, segment.Length);
+ totalPacked += segment.Length;
+ }
+ // Data that hardly shrank is mostly stored chunks: copying them is quicker here than handing them to threads.
+ if (!Forced && totalPacked > size / 10 * 9)
+ return null;
+ var workers = Math.Min(Workers(largest, largestPacked, dictionary), segments.Count);
+ if (workers < 2)
+ return null;
+ Interlocked.Increment(ref Opened);
+ return new SegmentStream(packed, segments, dictionary, workers, cancellationToken);
+ }
+
+ /// One run of chunks from a dictionary reset, at in the packed stream.
+ internal readonly record struct Segment(long Offset, int Length, int Unpacked);
+
+ ///
+ /// The segments of the LZMA2 data, from its chunk headers; null when the data does not end with the end byte
+ /// within the packed stream, doesn't start with a dictionary reset, has an unknown chunk type, a segment past
+ /// , or doesn't unpack to bytes.
+ ///
+ private static List? Scan(PackedWindow packed, long size)
+ {
+ var segments = new List();
+ var minSegment = Forced ? 1 : MinSegment;
+ Span header = stackalloc byte[6];
+ long at = 0;
+ long start = 0;
+ long unpacked = 0;
+ long total = 0;
+ var first = true;
+ while (true)
+ {
+ if (at >= packed.Length || packed.ReadAt(at, header[..1]) < 1)
+ return null;
+ var control = header[0];
+ if (control == 0)
+ break;
+ var reset = control == 1 || control >= 0xE0;
+ if (first && !reset)
+ return null;
+ if (reset && !first && unpacked >= minSegment)
+ {
+ segments.Add(new Segment(start, (int)(at - start), (int)unpacked));
+ start = at;
+ unpacked = 0;
+ }
+ first = false;
+ int chunkUnpacked, chunkLength;
+ if (control < 0x80)
+ {
+ if (control > 2 || packed.ReadAt(at + 1, header[..2]) < 2)
+ return null;
+ chunkUnpacked = (header[0] << 8 | header[1]) + 1;
+ chunkLength = 3 + chunkUnpacked;
+ }
+ else
+ {
+ if (packed.ReadAt(at + 1, header[..4]) < 4)
+ return null;
+ chunkUnpacked = ((control & 0x1F) << 16 | header[0] << 8 | header[1]) + 1;
+ chunkLength = 5 + (header[2] << 8 | header[3]) + 1 + (control >= 0xC0 ? 1 : 0);
+ }
+ unpacked += chunkUnpacked;
+ total += chunkUnpacked;
+ at += chunkLength;
+ if (unpacked > MaxSegment || at - start > MaxSegment || total > size)
+ return null;
+ }
+ if (total != size || unpacked == 0)
+ return null;
+ segments.Add(new Segment(start, (int)(at - start), (int)unpacked));
+ return segments;
+ }
+
+ /// What a worker decoded: the output, how much of it is good, and the error that stopped it, if any.
+ internal sealed class Decoded(byte[] output)
+ {
+ public byte[] Output { get; } = output;
+ public int Count { get; set; }
+ public ExceptionDispatchInfo? Error { get; set; }
+ }
+
+ ///
+ /// Decodes one segment: bytes of chunks followed by the end byte (the caller puts it at
+ /// packed[length]), expected to give bytes.
+ ///
+ public static Decoded DecodeSegment(byte[] packed, int length, uint dictionary, int unpacked, CancellationToken cancellationToken)
+ {
+ var result = new Decoded(GC.AllocateUninitializedArray(unpacked));
+ try
+ {
+ var reader = new Lzma2Reader(new ByteInput(new MemoryStream(packed, 0, length + 1, writable: false)), dictionary, unpacked,
+ cancellationToken);
+ var output = result.Output;
+ while (result.Count < unpacked)
+ {
+ var read = reader.Read(output.AsSpan(result.Count));
+ if (read == 0)
+ throw new InvalidDataException("the LZMA2 chunk is damaged");
+ result.Count += read;
+ }
+ }
+ catch (Exception ex)
+ {
+ result.Error = ExceptionDispatchInfo.Capture(ex);
+ }
+ return result;
+ }
+
+ /// The segments of a 7z LZMA2 coder, decoded ahead on worker threads and read in order.
+ private sealed class SegmentStream : Stream
+ {
+ private readonly PackedWindow _packed;
+ private readonly List _segments;
+ private readonly uint _dictionary;
+ private readonly int _workers;
+ private readonly CancellationTokenSource _cancel;
+ private readonly Queue<(Segment Segment, Task Work)> _ahead = new();
+ private int _next;
+ private Decoded? _current;
+ private int _position;
+ private Lzma2Reader? _replay; // a failed segment, decoded again here
+
+ public SegmentStream(PackedWindow packed, List segments, uint dictionary, int workers, CancellationToken cancellationToken)
+ {
+ _packed = packed;
+ _segments = segments;
+ _dictionary = dictionary;
+ _workers = workers;
+ _cancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ }
+
+ public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
+
+ public override int Read(Span buffer)
+ {
+ if (buffer.IsEmpty)
+ return 0;
+ while (true)
+ {
+ if (_replay is not null)
+ {
+ var read = _replay.Read(buffer);
+ if (read > 0)
+ return read;
+ _replay = null;
+ continue;
+ }
+ if (_current is { } current)
+ {
+ if (_position < current.Count)
+ {
+ var count = Math.Min(buffer.Length, current.Count - _position);
+ current.Output.AsSpan(_position, count).CopyTo(buffer);
+ _position += count;
+ return count;
+ }
+ _current = null;
+ }
+ _cancel.Token.ThrowIfCancellationRequested();
+ Start();
+ if (!_ahead.TryDequeue(out var next))
+ return 0;
+ var decoded = next.Work.GetAwaiter().GetResult();
+ if (decoded.Error is { } error)
+ {
+ if (error.SourceException is OperationCanceledException)
+ error.Throw();
+ _replay = Open(next.Segment, _cancel.Token);
+ continue;
+ }
+ _current = decoded;
+ _position = 0;
+ }
+ }
+
+ /// Reads the packed bytes of the next segments (on this thread: the archive stream is shared) and starts their decoders.
+ private void Start()
+ {
+ while (_ahead.Count < _workers && _next < _segments.Count)
+ {
+ var segment = _segments[_next++];
+ var packed = ReadPacked(segment);
+ var dictionary = _dictionary;
+ var token = _cancel.Token;
+ _ahead.Enqueue((segment, Task.Run(() => DecodeSegment(packed, segment.Length, dictionary, segment.Unpacked, token), CancellationToken.None)));
+ }
+ }
+
+ /// A segment's chunks and the end byte after them, so its reader ends cleanly.
+ private byte[] ReadPacked(Segment segment)
+ {
+ var packed = GC.AllocateUninitializedArray(segment.Length + 1);
+ if (_packed.ReadAt(segment.Offset, packed.AsSpan(0, segment.Length)) < segment.Length)
+ throw new InvalidDataException("the LZMA2 data ends too early");
+ packed[segment.Length] = 0x00;
+ return packed;
+ }
+
+ /// A reader of one segment, on this thread.
+ private Lzma2Reader Open(Segment segment, CancellationToken cancellationToken) =>
+ new(new ByteInput(new MemoryStream(ReadPacked(segment), writable: false)), _dictionary, segment.Unpacked, cancellationToken);
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ _cancel.Cancel();
+ _cancel.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+
+ public override bool CanRead => true;
+ public override bool CanSeek => false;
+ public override bool CanWrite => false;
+ public override long Length => throw new NotSupportedException();
+ public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
+ public override void Flush() { }
+ public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+ public override void SetLength(long value) => throw new NotSupportedException();
+ public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+ }
+}
+
+/// A packed stream that lies at a known place in a seekable stream others read too (a 7z pack stream).
+internal sealed class PackedWindow(Stream shared, long offset, long length)
+{
+ public long Length => length;
+
+ /// Reads up to 's length at within the window; returns the bytes read.
+ public int ReadAt(long at, Span target)
+ {
+ if (at < 0 || at >= length)
+ return 0;
+ if (target.Length > length - at)
+ target = target[..(int)(length - at)];
+ shared.Position = offset + at;
+ return shared.ReadAtLeast(target, target.Length, throwOnEndOfStream: false);
+ }
+}
diff --git a/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs b/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs
index 9f93561b2..d919df87e 100644
--- a/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs
+++ b/src/Filee.Engines/Archives/Lzma/Lzma2Writer.cs
@@ -1,9 +1,14 @@
-// LZMA2 encoding (see Lzma2Reader for the chunk layout). Input collects in the match finder's window; whenever enough
-// is waiting, one LZMA chunk is coded: up to 2 MiB of input, stopped early when its output nears 64 KiB. A chunk that
-// came out no smaller than its input is written as stored chunks instead, and the next LZMA chunk resets the
-// state (the decoder never saw the probabilities the discarded attempt left behind).
-// The first chunk resets the dictionary and sets the properties; after a first stored chunk (0x01) the decoder needs
-// the properties again, so the next LZMA chunk carries them (0xC0).
+// LZMA2 encoding (see Lzma2Reader for the chunk layout). The input is cut into blocks of a fixed size (one to four
+// dictionaries by level, see Lzma2Settings.BlockSize) and each block is coded on its own, starting with a dictionary
+// reset: blocks are coded on several threads at once, and since where a block ends depends only on the input and the
+// level, the output is the same bytes whatever the number of threads. A reader can decode the blocks side by side
+// too (Lzma2Parallel). Within a block, one LZMA chunk is coded at a time: up to 2 MiB of input, stopped early when
+// its output nears 64 KiB. A chunk that came out no smaller than its input is written as stored chunks instead, and
+// the next LZMA chunk resets the state (the decoder never saw the probabilities the discarded attempt left behind).
+// A block's first chunk resets the dictionary and sets the properties; after a first stored chunk (0x01) the decoder
+// needs the properties again, so the next LZMA chunk carries them (0xC0).
+// With processors to spare (fewer blocks than half of them), each block's binary-tree match finder searches ahead on
+// a thread of its own (MatchFinder.StartAhead), which gives the same bytes too.
using System.Numerics;
using Filee.Core.Presets;
@@ -15,8 +20,9 @@ internal readonly record struct Lzma2Settings(int Dictionary, LzmaEncoderSetting
{
///
/// The settings for , with the dictionary cut to the input when its size is known. Memory
- /// for the encoder is about 8 × the dictionary with the hash chain (fast: 4 MiB → ~32 MB) and 10–11 × with the binary
- /// tree (normal: 16 MiB → ~180 MB, maximum: 64 MiB → ~660 MB, as xz -9); unpacking needs only the dictionary.
+ /// for one block's encoder is about 8 × the dictionary with the hash chain (fast: 4 MiB → ~32 MB) and 10–11 × with
+ /// the binary tree (normal: 16 MiB → ~180 MB, maximum: 64 MiB → ~660 MB, as xz -9), plus the block itself while it
+ /// is coded (); unpacking needs only the dictionary.
///
public static Lzma2Settings For(ArchiveLevel level, long expectedSize = -1)
{
@@ -34,45 +40,64 @@ public static Lzma2Settings For(ArchiveLevel level, long expectedSize = -1)
}
return settings;
}
+
+ ///
+ /// The input coded as one block, from one dictionary reset to the next, at least 1 MiB and at most 256 MiB: one
+ /// dictionary at the fast level (many blocks for many threads; about 0.2 % larger output), two at normal (about
+ /// 0.15 %), four at maximum. Matches never reach into the block before, which costs that little ratio.
+ ///
+ public int BlockSize => (int)Math.Clamp((long)Dictionary * (!Encoder.Optimal ? 1 : Encoder.Depth >= 64 ? 4 : 2), 1 << 20, 1 << 28);
}
internal sealed class Lzma2Writer
{
- private const int StoredChunk = 1 << 16;
+ /// Tests set the number of threads (normally from the processors, the blocks expected and memory).
+ internal static readonly AsyncLocal TestThreads = new();
- // Stop a chunk while a worst-case symbol (about 25 bytes) still fits into the 64 KiB a chunk may hold.
- private const int PackedLimit = Lzma2Reader.MaxChunkPacked - 64;
+ /// Tests set a smaller block size, to get many blocks from little input.
+ internal static readonly AsyncLocal TestBlockSize = new();
- // Bytes kept unread at the end of the window until the input is finished, so hashes can read ahead.
- private const int LookAhead = LzmaModel.MatchMaxLength + 4;
+ /// Tests turn the match finder's own thread on or off (normally on with processors to spare).
+ internal static readonly AsyncLocal TestSearchAhead = new();
private readonly Stream _output;
private readonly Lzma2Settings _settings;
- private readonly MatchFinder? _mf;
- private readonly LzmaEncoder? _encoder;
- private readonly RangeEncoder? _rc;
- private readonly byte[] _stored = new byte[StoredChunk];
- private readonly byte[] _header = new byte[6];
+ private readonly Action? _onBlock;
private readonly CancellationToken _cancellationToken;
- private int _storedCount;
- private bool _first = true;
- private bool _needProperties;
- private bool _needStateReset;
+ private readonly int _blockSize;
+ private readonly int _firstCapacity;
+ private readonly int _threads;
+ private readonly bool _searchAhead;
+ private readonly bool _check;
+ private readonly Stack _free = new();
+ private readonly Queue<(Lzma2BlockEncoder Encoder, Task Work)> _pending = new();
+ private int _created;
+ private Lzma2BlockEncoder? _current;
+ private bool _finished;
- /// The input size if known (-1 otherwise); only sizes the window's room for input.
- public Lzma2Writer(Stream output, Lzma2Settings settings, long expectedSize, CancellationToken cancellationToken)
+ ///
+ /// The input size if known (-1 otherwise); it sizes the first block's buffer and how many threads are worth
+ /// starting, never where blocks end.
+ ///
+ ///
+ /// Takes each coded block (in order) instead of the output: then nothing is written to ,
+ /// the end byte neither, and each block's input gets a CRC-64 (for .xz blocks).
+ ///
+ public Lzma2Writer(Stream output, Lzma2Settings settings, long expectedSize, CancellationToken cancellationToken,
+ Action? onBlock = null)
{
_output = output;
_settings = settings;
+ _onBlock = onBlock;
+ _check = onBlock is not null;
_cancellationToken = cancellationToken;
- if (settings.Store)
- return;
- var ahead = Math.Max(Lzma2Reader.MaxChunkUnpacked * 2, settings.Dictionary / 4);
- if (expectedSize >= 0)
- ahead = (int)Math.Min(ahead, expectedSize + LookAhead);
- _mf = new MatchFinder(settings.Dictionary, ahead, settings.Encoder.Depth, settings.Encoder.Nice, settings.Encoder.Tree);
- _encoder = new LzmaEncoder(_mf, LzmaProperties.Default, settings.Encoder);
- _rc = new RangeEncoder(Lzma2Reader.MaxChunkPacked + 1024);
+ _blockSize = TestBlockSize.Value ?? settings.BlockSize;
+ _firstCapacity = expectedSize >= 0
+ ? (int)Math.Min(_blockSize, Math.Max(expectedSize, 1) + LzmaModel.MatchMaxLength + 4)
+ : _blockSize;
+ _threads = TestThreads.Value ?? Threads(settings, _blockSize, expectedSize);
+ _searchAhead = settings.Encoder is { Tree: true, Optimal: true }
+ && (TestSearchAhead.Value ?? 2 * _threads <= Environment.ProcessorCount);
}
/// The dictionary size the decoder needs (for the filter properties).
@@ -81,59 +106,241 @@ public Lzma2Writer(Stream output, Lzma2Settings settings, long expectedSize, Can
/// Bytes written to the output so far.
public long Written { get; private set; }
+ ///
+ /// Threads worth starting: one per block expected, at most one per processor, and no more block encoders than an
+ /// eighth of the memory holds.
+ ///
+ private static int Threads(Lzma2Settings settings, int blockSize, long expectedSize)
+ {
+ var blocks = expectedSize < 0 ? int.MaxValue : (int)Math.Min(int.MaxValue, (expectedSize + blockSize - 1) / blockSize);
+ var perBlock = Lzma2BlockEncoder.MemoryFor(settings, blockSize);
+ long memory;
+ try
+ {
+ memory = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes;
+ }
+ catch (Exception ex) when (ex is InvalidOperationException or PlatformNotSupportedException)
+ {
+ memory = 0;
+ }
+ var byMemory = memory > 0 ? (int)Math.Min(int.MaxValue, memory / 8 / perBlock) : 1;
+ return Math.Max(1, Math.Min(Math.Min(Environment.ProcessorCount, blocks), byMemory));
+ }
+
public void Write(ReadOnlySpan data)
{
+ ObjectDisposedException.ThrowIf(_finished, this);
while (!data.IsEmpty)
{
- if (_mf is null)
- {
- var count = Math.Min(data.Length, StoredChunk - _storedCount);
- data[..count].CopyTo(_stored.AsSpan(_storedCount));
- _storedCount += count;
- data = data[count..];
- if (_storedCount == StoredChunk)
- FlushStored();
- continue;
- }
- data = data[_mf.Append(data)..];
- if (_mf.Filled == _mf.Buffer.Length)
+ _current ??= Acquire();
+ var room = _blockSize - _current.Filled;
+ if (_current.Room == 0)
+ _current.Grow(_blockSize);
+ var count = _current.Append(data[..Math.Min(data.Length, room)]);
+ data = data[count..];
+ if (_current.Filled == _blockSize)
{
- EncodeAvailable(final: false);
- _encoder!.Shifted(_mf.Slide());
+ Dispatch(_current);
+ _current = null;
}
}
}
- /// Codes what is left and writes the end byte.
+ /// Codes what is left and writes the end byte (not with a block callback).
public void Finish()
{
- if (_mf is null)
- FlushStored();
- else
- EncodeAvailable(final: true);
- _output.WriteByte(0x00);
- Written++;
+ if (_finished)
+ return;
+ if (_current is { Filled: > 0 })
+ Dispatch(_current);
+ _current = null;
+ while (_pending.Count > 0)
+ _free.Push(Complete(_pending.Dequeue()));
+ _finished = true;
+ if (_onBlock is null)
+ {
+ _output.WriteByte(0x00);
+ Written++;
+ }
}
- private void FlushStored()
+ /// A block encoder to fill: an idle one, a new one while there are threads to spare, or the oldest busy one once done.
+ private Lzma2BlockEncoder Acquire()
{
- if (_storedCount == 0)
+ if (_free.TryPop(out var idle))
+ {
+ idle.Reset();
+ return idle;
+ }
+ if (_created < _threads)
+ {
+ _created++;
+ return new Lzma2BlockEncoder(_settings, _created == 1 ? _firstCapacity : _blockSize, _check);
+ }
+ var done = Complete(_pending.Dequeue());
+ done.Reset();
+ return done;
+ }
+
+ private void Dispatch(Lzma2BlockEncoder encoder)
+ {
+ _cancellationToken.ThrowIfCancellationRequested();
+ if (_threads == 1)
+ {
+ encoder.Encode(_cancellationToken, _searchAhead);
+ Emit(encoder);
+ _free.Push(encoder);
return;
- WriteStored(_stored.AsSpan(0, _storedCount));
+ }
+ var (token, ahead) = (_cancellationToken, _searchAhead);
+ _pending.Enqueue((encoder, Task.Run(() => encoder.Encode(token, ahead), CancellationToken.None)));
+ }
+
+ /// Waits for a block, writes it out and hands its encoder back.
+ private Lzma2BlockEncoder Complete((Lzma2BlockEncoder Encoder, Task Work) pending)
+ {
+ pending.Work.GetAwaiter().GetResult();
+ Emit(pending.Encoder);
+ return pending.Encoder;
+ }
+
+ private void Emit(Lzma2BlockEncoder encoder)
+ {
+ if (_onBlock is not null)
+ {
+ _onBlock(new Lzma2Block(encoder.Output, encoder.Filled, encoder.Check));
+ return;
+ }
+ _output.Write(encoder.Output);
+ Written += encoder.Output.Length;
+ }
+}
+
+/// One coded LZMA2 block: its chunks (without the end byte), its input size and that input's CRC-64.
+internal readonly ref struct Lzma2Block(ReadOnlySpan chunks, int unpacked, ulong check)
+{
+ public ReadOnlySpan Chunks { get; } = chunks;
+ public int Unpacked { get; } = unpacked;
+ public ulong Check { get; } = check;
+}
+
+/// Codes one LZMA2 block (all of it in its buffer) into chunks that start with a dictionary reset.
+internal sealed class Lzma2BlockEncoder
+{
+ private const int StoredChunk = 1 << 16;
+
+ // Stop a chunk while a worst-case symbol (about 25 bytes) still fits into the 64 KiB a chunk may hold.
+ private const int PackedLimit = Lzma2Reader.MaxChunkPacked - 64;
+
+ private readonly Lzma2Settings _settings;
+ private readonly bool _check;
+ private readonly MatchFinder? _mf;
+ private readonly LzmaEncoder? _encoder;
+ private readonly RangeEncoder? _rc;
+ private byte[] _stored = []; // the input at the store level
+ private int _storedCount;
+ private byte[] _out;
+ private int _outCount;
+ private bool _first;
+ private bool _needProperties;
+ private bool _needStateReset;
+
+ public Lzma2BlockEncoder(Lzma2Settings settings, int capacity, bool check)
+ {
+ _settings = settings;
+ _check = check;
+ _out = new byte[Math.Max(1 << 16, capacity / 4)];
+ if (settings.Store)
+ {
+ _stored = new byte[capacity];
+ return;
+ }
+ _mf = new MatchFinder(settings.Dictionary, Math.Max(0, capacity - settings.Dictionary), settings.Encoder.Depth,
+ settings.Encoder.Nice, settings.Encoder.Tree);
+ _encoder = new LzmaEncoder(_mf, LzmaProperties.Default, settings.Encoder);
+ _rc = new RangeEncoder(Lzma2Reader.MaxChunkPacked + 1024);
+ }
+
+ /// About the memory one block encoder takes: its buffer, the match finder's tables and its output.
+ public static long MemoryFor(Lzma2Settings settings, int blockSize) =>
+ settings.Store ? 2L * blockSize : blockSize * 3L / 2 + MatchFinder.TableBytes(settings.Dictionary, settings.Encoder.Tree);
+
+ /// Input bytes in the block.
+ public int Filled => _mf?.Filled ?? _storedCount;
+
+ /// Room left in the buffer.
+ public int Room => _mf is not null ? _mf.Buffer.Length - _mf.Filled : _stored.Length - _storedCount;
+
+ /// The coded chunks, after .
+ public ReadOnlySpan Output => _out.AsSpan(0, _outCount);
+
+ /// CRC-64 of the input, after (only when asked for).
+ public ulong Check { get; private set; }
+
+ public int Append(ReadOnlySpan data)
+ {
+ if (_mf is not null)
+ return _mf.Append(data);
+ var count = Math.Min(data.Length, _stored.Length - _storedCount);
+ data[..count].CopyTo(_stored.AsSpan(_storedCount));
+ _storedCount += count;
+ return count;
+ }
+
+ /// Makes room for up to bytes of input (the input was larger than expected).
+ public void Grow(int capacity)
+ {
+ if (_mf is not null)
+ {
+ _mf.Grow(capacity);
+ return;
+ }
+ if (_stored.Length < capacity)
+ Array.Resize(ref _stored, Math.Min(capacity, Math.Max(_stored.Length * 2, 1 << 20)));
+ }
+
+ /// Empties the block for the next one, keeping the buffers.
+ public void Reset()
+ {
_storedCount = 0;
+ _outCount = 0;
+ if (_mf is null)
+ return;
+ _mf.Reset();
+ _encoder!.Restart();
}
- private void EncodeAvailable(bool final)
+ /// Codes the whole block.
+ /// Lets the match finder search on a thread of its own (binary tree, large blocks).
+ public void Encode(CancellationToken cancellationToken, bool searchAhead = false)
{
- var mf = _mf!;
- var limit = final ? mf.Filled : mf.Filled - LookAhead;
- while (mf.Position < limit)
- EncodeChunk(Math.Min(limit, mf.Position + Lzma2Reader.MaxChunkUnpacked));
+ _first = true;
+ _needProperties = false;
+ _needStateReset = false;
+ _outCount = 0;
+ if (_check)
+ Check = Crc64.Compute(_mf is not null ? _mf.Buffer.AsSpan(0, _mf.Filled) : _stored.AsSpan(0, _storedCount));
+ if (_mf is null)
+ {
+ WriteStored(_stored.AsSpan(0, _storedCount));
+ return;
+ }
+ if (searchAhead && _settings.Encoder.Tree && _mf.Filled >= 1 << 18)
+ _mf.StartAhead(cancellationToken);
+ try
+ {
+ while (_mf.Position < _mf.Filled)
+ EncodeChunk(Math.Min(_mf.Filled, _mf.Position + Lzma2Reader.MaxChunkUnpacked), cancellationToken);
+ }
+ finally
+ {
+ _mf.StopAhead();
+ }
}
- private void EncodeChunk(int end)
+ private void EncodeChunk(int end, CancellationToken cancellationToken)
{
- _cancellationToken.ThrowIfCancellationRequested();
+ cancellationToken.ThrowIfCancellationRequested();
var mf = _mf!;
var rc = _rc!;
var start = mf.Position;
@@ -163,17 +370,17 @@ private void EncodeChunk(int end)
control = 0x80;
var u = unpacked - 1;
var p = packed - 1;
- _header[0] = (byte)(control | (u >> 16));
- _header[1] = (byte)(u >> 8);
- _header[2] = (byte)u;
- _header[3] = (byte)(p >> 8);
- _header[4] = (byte)p;
+ Span header = stackalloc byte[6];
+ header[0] = (byte)(control | (u >> 16));
+ header[1] = (byte)(u >> 8);
+ header[2] = (byte)u;
+ header[3] = (byte)(p >> 8);
+ header[4] = (byte)p;
var headerLength = 5;
if (control >= 0xC0)
- _header[headerLength++] = LzmaProperties.Default.ToByte();
- _output.Write(_header, 0, headerLength);
- _output.Write(rc.Output);
- Written += headerLength + packed;
+ header[headerLength++] = LzmaProperties.Default.ToByte();
+ Put(header[..headerLength]);
+ Put(rc.Output);
_first = false;
_needProperties = false;
_needStateReset = false;
@@ -181,16 +388,16 @@ private void EncodeChunk(int end)
private void WriteStored(ReadOnlySpan data)
{
+ Span header = stackalloc byte[3];
while (!data.IsEmpty)
{
var count = Math.Min(data.Length, StoredChunk);
var u = count - 1;
- _header[0] = (byte)(_first ? 0x01 : 0x02);
- _header[1] = (byte)(u >> 8);
- _header[2] = (byte)u;
- _output.Write(_header, 0, 3);
- _output.Write(data[..count]);
- Written += 3 + count;
+ header[0] = (byte)(_first ? 0x01 : 0x02);
+ header[1] = (byte)(u >> 8);
+ header[2] = (byte)u;
+ Put(header);
+ Put(data[..count]);
if (_first)
_needProperties = true;
_first = false;
@@ -198,4 +405,12 @@ private void WriteStored(ReadOnlySpan data)
data = data[count..];
}
}
+
+ private void Put(ReadOnlySpan data)
+ {
+ if (_outCount + data.Length > _out.Length)
+ Array.Resize(ref _out, Math.Max(_outCount + data.Length, _out.Length * 2));
+ data.CopyTo(_out.AsSpan(_outCount));
+ _outCount += data.Length;
+ }
}
diff --git a/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs b/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs
index 7282be7a1..2859f385d 100644
--- a/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs
+++ b/src/Filee.Engines/Archives/Lzma/LzmaEncoder.cs
@@ -85,6 +85,14 @@ public override void ResetState()
_pricedAt = -1;
}
+ /// Starts over for a new block (after a dictionary reset): state, position and the cached search.
+ public void Restart()
+ {
+ ResetState();
+ _total = 0;
+ _cachedAt = -1;
+ }
+
/// The window moved back by bytes (a slide); the cached search moves with it.
public void Shifted(int shift)
{
@@ -113,7 +121,7 @@ private void EncodeOne(RangeEncoder rc, int end)
var from = pos - (int)distance - 1;
if (buffer[from] != buffer[pos] || buffer[from + 1] != buffer[pos + 1])
continue;
- var length = buffer.AsSpan(pos, available).CommonPrefixLength(buffer.AsSpan(from, available));
+ var length = MatchFinder.Common(buffer, pos, from, available);
if (length > repLength)
{
repLength = length;
@@ -211,7 +219,7 @@ private void EncodeOptimal(RangeEncoder rc, int end, long packedLimit)
var from = cur - (int)distance - 1;
if (buffer[from] != buffer[cur] || buffer[from + 1] != buffer[cur + 1])
continue;
- var length = buffer.AsSpan(cur, available).CommonPrefixLength(buffer.AsSpan(from, available));
+ var length = MatchFinder.Common(buffer, cur, from, available);
_repLengths[r] = length;
if (length > bestRep)
{
diff --git a/src/Filee.Engines/Archives/Lzma/MatchFinder.cs b/src/Filee.Engines/Archives/Lzma/MatchFinder.cs
index f679a00e0..37522d4ca 100644
--- a/src/Filee.Engines/Archives/Lzma/MatchFinder.cs
+++ b/src/Filee.Engines/Archives/Lzma/MatchFinder.cs
@@ -11,10 +11,18 @@
// The window is one array: the dictionary behind the cursor plus room ahead for new input, slid back when the room
// is used up. Positions are stored as buffer index + an offset that grows with each slide, so the tables never need
// rewriting when the window moves; only every ~1 GiB of input are they rebased (Normalize). 0 marks an empty slot.
+//
+// Searching ahead (binary tree, for the price-based parse, with the whole input in the buffer): a thread of its own
+// searches every position in order, as the parse would search it or step over it (both walk the tree the same way),
+// and hands the matches over in batches; the parse takes each position's list cut to the length it allows there.
+// The lists are exactly those a search in the parse would find, so the output is the same bytes either way.
using System.Buffers.Binary;
+using System.Collections.Concurrent;
using System.Numerics;
using System.Runtime.CompilerServices;
+using System.Runtime.ExceptionServices;
+using System.Runtime.InteropServices;
namespace Filee.Engines.Archives.Lzma;
@@ -38,6 +46,7 @@ internal sealed class MatchFinder
private readonly int _nice;
private int _offset = 1;
private int _nextInsert;
+ private Ahead? _ahead;
/// The window, a power of two (4 KiB … 256 MiB, so stored positions stay within an int).
/// Room for input beyond the window.
@@ -60,8 +69,38 @@ public MatchFinder(int dictionary, int ahead, int depth, int nice, bool tree)
Buffer = new byte[dictionary + Math.Max(ahead, MinAhead)];
}
+ /// Bytes of the hash tables and links for a dictionary (the buffer not counted).
+ public static long TableBytes(int dictionary, bool tree)
+ {
+ var hashBits = Math.Clamp(BitOperations.Log2((uint)dictionary) - 1, 16, 24);
+ return 4L * ((tree ? 2L : 1L) * dictionary + (1 << hashBits) + (1 << Hash3Bits) + (1 << 16));
+ }
+
+ /// Forgets the input and every position entered, keeping the buffer and the tables' memory.
+ public void Reset()
+ {
+ Position = 0;
+ Filled = 0;
+ _nextInsert = 0;
+ _offset = 1;
+ Array.Clear(_links);
+ Array.Clear(_head4);
+ Array.Clear(_head3);
+ Array.Clear(_head2);
+ }
+
+ /// Makes the buffer hold up to bytes (at least doubling it), keeping its content.
+ public void Grow(int capacity)
+ {
+ if (Buffer.Length >= capacity)
+ return;
+ var buffer = Buffer;
+ Array.Resize(ref buffer, (int)Math.Min(capacity, Math.Max(2L * buffer.Length, 1 << 20)));
+ Buffer = buffer;
+ }
+
/// The window: history before , input not yet encoded after it.
- public byte[] Buffer { get; }
+ public byte[] Buffer { get; private set; }
/// Index of the next byte to encode.
public int Position { get; set; }
@@ -127,7 +166,26 @@ static void Rebase(int[] table, int delta)
/// bytes, each longer than the one before; returns how many.
///
public int FindAll(int pos, int maxLength, int[] lengths, uint[] distances) =>
- Search(pos, maxLength, lengths, distances, out _);
+ _ahead is { } ahead ? ahead.Take(pos, maxLength, lengths, distances) : Search(pos, maxLength, lengths, distances, out _);
+
+ ///
+ /// Starts searching every position from to on a thread of its own
+ /// (binary tree only); then takes its results and does nothing. Call
+ /// when done.
+ ///
+ public void StartAhead(CancellationToken cancellationToken)
+ {
+ if (!_tree)
+ throw new InvalidOperationException("only the binary tree searches ahead");
+ _ahead = new Ahead(this, Position, Filled, cancellationToken);
+ }
+
+ /// Stops the search thread (if any) and waits for it.
+ public void StopAhead()
+ {
+ _ahead?.Stop();
+ _ahead = null;
+ }
/// The matches found by one search, longest last.
private ref struct Found(int maxLength, int[]? lengths, uint[]? distances)
@@ -183,7 +241,7 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou
{
var at = near - _offset;
if (buffer[at] == buffer[pos])
- found.Offer(buffer.AsSpan(pos, maxLength).CommonPrefixLength(buffer.AsSpan(at, maxLength)), stored - near);
+ found.Offer(Common(buffer, pos, at, maxLength), stored - near);
}
}
}
@@ -195,7 +253,7 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou
if (found.Best >= _nice && found.Best < maxLength)
{
var at = pos - (int)found.Distance - 1;
- var longer = buffer.AsSpan(pos, maxLength).CommonPrefixLength(buffer.AsSpan(at, maxLength));
+ var longer = Common(buffer, pos, at, maxLength);
found.Best = longer;
if (lengths is not null)
lengths[found.Count - 1] = longer;
@@ -213,7 +271,7 @@ private int Search(int pos, int maxLength, int[]? lengths, uint[]? distances, ou
candidate = _links[candidate & _mask];
if (buffer[at + found.Best] != buffer[pos + found.Best] || buffer[at] != buffer[pos])
continue;
- found.Offer(buffer.AsSpan(pos, maxLength).CommonPrefixLength(buffer.AsSpan(at, maxLength)), gap);
+ found.Offer(Common(buffer, pos, at, maxLength), gap);
}
}
@@ -247,9 +305,14 @@ private void TreeWalk(int pos, int stored, int candidate, ref Found found)
var at = candidate - _offset;
var node = (candidate & _mask) << 1;
var length = Math.Min(leftLength, rightLength);
- if (length < limit)
- length += buffer.AsSpan(pos + length, limit - length).CommonPrefixLength(buffer.AsSpan(at + length, limit - length));
- found.Offer(length, gap);
+ // Most candidates differ at the first byte not known to match: only then is the rest compared.
+ if (length < limit && buffer[at + length] == buffer[pos + length])
+ {
+ length++;
+ if (length < limit)
+ length += Common(buffer, pos + length, at + length, limit - length);
+ found.Offer(length, gap);
+ }
if (length == limit)
{
// Equal as far as the tree looks: the candidate's subtrees become the new root's.
@@ -274,11 +337,33 @@ private void TreeWalk(int pos, int stored, int candidate, ref Found found)
}
}
+ ///
+ /// How many of the bytes at and agree, eight at
+ /// a time (most compares end within the first word, where a span compare would spend more on its setup).
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ internal static int Common(byte[] buffer, int a, int b, int limit)
+ {
+ if ((uint)limit > (uint)buffer.Length || (uint)a > (uint)(buffer.Length - limit) || (uint)b > (uint)(buffer.Length - limit))
+ throw new ArgumentOutOfRangeException(nameof(limit));
+ ref var data = ref MemoryMarshal.GetArrayDataReference(buffer);
+ var n = 0;
+ for (; n + 8 <= limit; n += 8)
+ {
+ var x = Unsafe.ReadUnaligned(ref Unsafe.Add(ref data, a + n)) ^ Unsafe.ReadUnaligned(ref Unsafe.Add(ref data, b + n));
+ if (x != 0)
+ return n + (BitConverter.IsLittleEndian ? BitOperations.TrailingZeroCount(x) : BitOperations.LeadingZeroCount(x)) / 8;
+ }
+ while (n < limit && Unsafe.Add(ref data, a + n) == Unsafe.Add(ref data, b + n))
+ n++;
+ return n;
+ }
+
/// Enters every position before not entered yet (the inside of a match).
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void SkipTo(int pos)
{
- if (_nextInsert < pos)
+ if (_nextInsert < pos && _ahead is null)
Insert(pos);
}
@@ -304,4 +389,124 @@ private void Insert(int pos)
}
_nextInsert = pos;
}
+
+ /// The search thread and the batches of match lists it hands over.
+ private sealed class Ahead
+ {
+ private const int BatchSize = 4096;
+ private const int Batches = 8;
+
+ private sealed class Batch
+ {
+ public int First;
+ public readonly int[] Offsets = new int[BatchSize];
+ public int[] Data = new int[BatchSize * 4]; // per position: count, then length and distance per match
+ }
+
+ private readonly BlockingCollection _free = new();
+ private readonly BlockingCollection _full = new();
+ private readonly CancellationTokenSource _stop;
+ private readonly Task _producer;
+ private ExceptionDispatchInfo? _error;
+ private Batch? _current;
+
+ public Ahead(MatchFinder mf, int start, int end, CancellationToken cancellationToken)
+ {
+ for (var i = 0; i < Batches; i++)
+ _free.Add(new Batch());
+ _stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ var token = _stop.Token;
+ _producer = Task.Factory.StartNew(() => Produce(mf, start, end, token), CancellationToken.None,
+ TaskCreationOptions.LongRunning, TaskScheduler.Default);
+ }
+
+ private void Produce(MatchFinder mf, int start, int end, CancellationToken token)
+ {
+ try
+ {
+ var lengths = new int[LzmaModel.MatchMaxLength + 1];
+ var distances = new uint[LzmaModel.MatchMaxLength + 1];
+ for (var first = start; first < end; first += BatchSize)
+ {
+ var batch = _free.Take(token);
+ batch.First = first;
+ var at = 0;
+ var last = Math.Min(end, first + BatchSize);
+ for (var p = first; p < last; p++)
+ {
+ batch.Offsets[p - first] = at;
+ var count = mf.Search(p, Math.Min(LzmaModel.MatchMaxLength, end - p), lengths, distances, out _);
+ if (at + 1 + 2 * count > batch.Data.Length)
+ Array.Resize(ref batch.Data, Math.Max(batch.Data.Length * 2, at + 1 + 2 * count));
+ var data = batch.Data;
+ data[at++] = count;
+ for (var k = 0; k < count; k++)
+ {
+ data[at++] = lengths[k];
+ data[at++] = (int)distances[k];
+ }
+ }
+ _full.Add(batch, token);
+ }
+ }
+ catch (Exception ex)
+ {
+ _error = ExceptionDispatchInfo.Capture(ex);
+ }
+ finally
+ {
+ _full.CompleteAdding();
+ }
+ }
+
+ /// The matches at (positions come in order), cut to .
+ public int Take(int pos, int maxLength, int[] lengths, uint[] distances)
+ {
+ while (_current is null || pos >= _current.First + BatchSize)
+ {
+ if (_current is not null)
+ _free.Add(_current);
+ _current = null;
+ if (!_full.TryTake(out var next, Timeout.Infinite))
+ {
+ _error?.Throw();
+ throw new InvalidOperationException("a position past the input was searched");
+ }
+ _current = next;
+ }
+ if (maxLength < 2)
+ return 0;
+ var data = _current.Data;
+ var at = _current.Offsets[pos - _current.First];
+ var count = data[at++];
+ var result = 0;
+ for (var k = 0; k < count; k++, at += 2)
+ {
+ // The lengths grow: the first match that reaches the limit ends the list, cut to the limit.
+ var length = data[at];
+ lengths[result] = Math.Min(length, maxLength);
+ distances[result] = (uint)data[at + 1];
+ result++;
+ if (length >= maxLength)
+ break;
+ }
+ return result;
+ }
+
+ public void Stop()
+ {
+ _stop.Cancel();
+ try
+ {
+ _producer.Wait();
+ }
+ catch (AggregateException)
+ {
+ // The producer catches everything.
+ }
+ _stop.Dispose();
+ _free.Dispose();
+ _full.Dispose();
+ }
+ }
}
diff --git a/src/Filee.Engines/Archives/Lzma/XzReader.cs b/src/Filee.Engines/Archives/Lzma/XzReader.cs
index c62065ea5..5b411e341 100644
--- a/src/Filee.Engines/Archives/Lzma/XzReader.cs
+++ b/src/Filee.Engines/Archives/Lzma/XzReader.cs
@@ -7,8 +7,15 @@
// Limits: the dictionary each block names is refused past the caller's limit before anything is allocated, and the
// window grows with the output (LzmaDecoder). What the stream unpacks to is not limited here: the caller copies it
// through ArchiveFiles.CopyLimited with an UnpackBudget, like every other single-stream decoder.
+//
+// Threads: a block whose header names both sizes (as xz, 7-Zip and Filee write them with several threads) is read
+// whole, then decoded and checked on a worker thread while the next block headers are read (Lzma2Parallel bounds how
+// many and how large). Blocks are handed out in order; a block that fails on its thread is decoded again on the
+// reader's thread as it is read, so damaged data gives the same bytes and the same error, at the same place, as the
+// one-thread reader. An error in a header read ahead is thrown after the blocks before it.
using System.Buffers.Binary;
+using System.Runtime.ExceptionServices;
using System.Security.Cryptography;
namespace Filee.Engines.Archives.Lzma;
@@ -41,6 +48,15 @@ private enum Step { StreamHeader, BlockOrIndex, BlockData, StreamPadding, Done }
private long _blockUnpacked;
private long _position;
+ // Blocks decoding on worker threads, in order; the one being handed out; an error met while reading ahead.
+ private readonly Queue _ahead = new();
+ private Lzma2Parallel.Decoded? _current;
+ private Replay? _replay;
+ private int _currentPosition;
+ private int _aheadLimit = Environment.ProcessorCount;
+ private ExceptionDispatchInfo? _deferred;
+ private CancellationTokenSource? _cancel;
+
/// The file name used in error messages.
/// Dictionaries larger than this are refused ().
public XzReader(Stream input, string name, uint maxDictionary = DefaultMaxDictionary,
@@ -63,6 +79,58 @@ public override int Read(Span buffer)
while (true)
{
_cancellationToken.ThrowIfCancellationRequested();
+ if (_replay is { } replay)
+ {
+ var n = replay.Reader.Read(buffer);
+ if (n > 0)
+ {
+ replay.Check.Append(buffer[..n]);
+ replay.Unpacked += n;
+ _position += n;
+ if (replay.Unpacked > replay.Block.DeclaredUnpacked)
+ throw Damaged("a block unpacks to more than its header says");
+ return n;
+ }
+ CheckBlockEnd(replay.Input, replay.Input.Consumed, replay.Block.DeclaredPacked, replay.Block.DeclaredUnpacked, replay.Unpacked,
+ _checkType, replay.Check, _name);
+ _replay = null;
+ continue;
+ }
+ if (_current is { } current)
+ {
+ if (_currentPosition < current.Count)
+ {
+ var count = Math.Min(buffer.Length, current.Count - _currentPosition);
+ current.Output.AsSpan(_currentPosition, count).CopyTo(buffer);
+ _currentPosition += count;
+ _position += count;
+ return count;
+ }
+ _current = null;
+ }
+ ReadAhead();
+ if (_ahead.TryDequeue(out var next))
+ {
+ var decoded = next.Work.GetAwaiter().GetResult();
+ if (decoded.Error is { } error)
+ {
+ // Decoded again here, as the one-thread reader reads it.
+ if (error.SourceException is OperationCanceledException)
+ error.Throw();
+ var input = new ByteInput(new MemoryStream(next.Data, 0, next.Length, writable: false));
+ _replay = new Replay(next, input, new Lzma2Reader(input, next.Dictionary, next.DeclaredUnpacked, _cancellationToken),
+ Check.Create(_checkType));
+ continue;
+ }
+ _current = decoded;
+ _currentPosition = 0;
+ continue;
+ }
+ if (_deferred is { } deferred)
+ {
+ _deferred = null;
+ deferred.Throw();
+ }
switch (_step)
{
case Step.StreamHeader:
@@ -95,6 +163,25 @@ public override int Read(Span buffer)
}
}
+ ///
+ /// Reads the next block headers (and the blocks that name both sizes) while blocks decode on worker threads; an
+ /// error is kept until the blocks before it are handed out.
+ ///
+ private void ReadAhead()
+ {
+ while (_ahead.Count > 0 && _ahead.Count < _aheadLimit && _step == Step.BlockOrIndex && _deferred is null)
+ {
+ try
+ {
+ ReadBlockHeaderOrIndex();
+ }
+ catch (Exception ex)
+ {
+ _deferred = ExceptionDispatchInfo.Capture(ex);
+ }
+ }
+ }
+
private void ReadStreamHeader()
{
Span header = stackalloc byte[12];
@@ -168,33 +255,128 @@ private void ReadBlockHeaderOrIndex()
_blockHeaderSize = size;
_blockUnpacked = 0;
+ if (TryDecodeAhead(dictionary!.Value))
+ return;
_check = Check.Create(_checkType);
- _block = new Lzma2Reader(_input, dictionary!.Value, _declaredUnpacked, _cancellationToken);
+ _block = new Lzma2Reader(_input, dictionary.Value, _declaredUnpacked, _cancellationToken);
_step = Step.BlockData;
}
- private void FinishBlock()
+ ///
+ /// Reads a block whose header names both sizes whole and decodes it on a worker thread, when threads are allowed
+ /// and it fits in memory beside the others; false to decode it here as it is read.
+ ///
+ private bool TryDecodeAhead(uint dictionary)
{
- var packed = _input.Consumed - _blockStart - _blockHeaderSize;
- if (_declaredPacked >= 0 && packed != _declaredPacked)
- throw Damaged("a block's packed size does not match its header");
- if (_declaredUnpacked >= 0 && _blockUnpacked != _declaredUnpacked)
- throw Damaged("a block's unpacked size does not match its header");
- for (var padding = (4 - (int)(packed % 4)) % 4; padding > 0; padding--)
- if (Next() != 0)
- throw Damaged("a block has data in its padding");
+ if (!Lzma2Parallel.Enabled || _declaredPacked < 0 || _declaredUnpacked < 0
+ || _declaredPacked > Lzma2Parallel.MaxSegment || _declaredUnpacked > Lzma2Parallel.MaxSegment)
+ return false;
+ // A block that hardly shrank is mostly stored chunks: copying them here is quicker than a thread.
+ if (!Lzma2Parallel.Forced && _declaredPacked > _declaredUnpacked / 10 * 9)
+ return false;
+ var checkSize = XzFormat.CheckSize(_checkType);
+ var length = (int)_declaredPacked + (int)((4 - _declaredPacked % 4) % 4) + checkSize;
+ var workers = Lzma2Parallel.Workers(_declaredUnpacked, length, dictionary);
+ if (workers < 2)
+ return false;
+ _aheadLimit = workers;
+ var data = GC.AllocateUninitializedArray(length);
+ var read = _input.ReadAtMost(data);
+ // Recorded as the header says: a block that differs fails on its own thread, before the index is compared.
+ _blocks.Add((_blockHeaderSize + _declaredPacked + checkSize, _declaredUnpacked));
+ _cancel ??= CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken);
+ var (token, packed, unpacked, checkType, name) = (_cancel.Token, _declaredPacked, (int)_declaredUnpacked, _checkType, _name);
+ var work = Task.Run(() => DecodeBlock(data, read, dictionary, packed, unpacked, checkType, name, token), CancellationToken.None);
+ _ahead.Enqueue(new AheadBlock(work, data, read, dictionary, packed, unpacked));
+ _step = Step.BlockOrIndex;
+ return true;
+ }
- var stored = new byte[XzFormat.CheckSize(_checkType)];
- _input.ReadExactly(stored);
- if (!_check!.Matches(stored))
- throw Damaged("the check of the unpacked data does not match");
+ /// A block read whole and decoding on a worker thread, with what it takes to decode it again here.
+ private sealed record AheadBlock(Task Work, byte[] Data, int Length, uint Dictionary, long DeclaredPacked,
+ int DeclaredUnpacked);
- _blocks.Add((_blockHeaderSize + packed + stored.Length, _blockUnpacked));
+ /// A block that failed on its thread, decoded again on this one as it is read.
+ private sealed class Replay(AheadBlock block, ByteInput input, Lzma2Reader reader, Check check)
+ {
+ public AheadBlock Block { get; } = block;
+ public ByteInput Input { get; } = input;
+ public Lzma2Reader Reader { get; } = reader;
+ public Check Check { get; } = check;
+ public long Unpacked { get; set; }
+ }
+
+ /// A block read whole (data, padding and check), decoded and checked as does.
+ private static Lzma2Parallel.Decoded DecodeBlock(byte[] data, int length, uint dictionary, long declaredPacked, int declaredUnpacked,
+ int checkType, string name, CancellationToken cancellationToken)
+ {
+ var result = new Lzma2Parallel.Decoded(GC.AllocateUninitializedArray(declaredUnpacked));
+ try
+ {
+ var input = new ByteInput(new MemoryStream(data, 0, length, writable: false));
+ var reader = new Lzma2Reader(input, dictionary, declaredUnpacked, cancellationToken);
+ var check = Check.Create(checkType);
+ var output = result.Output;
+ while (true)
+ {
+ var read = result.Count < declaredUnpacked ? reader.Read(output.AsSpan(result.Count)) : 0;
+ if (read == 0)
+ {
+ // All of it, or the data ended; one more byte would be past the header's size.
+ Span probe = stackalloc byte[1];
+ if (result.Count == declaredUnpacked && reader.Read(probe) > 0)
+ throw ArchiveSafety.Damaged(name, "a block unpacks to more than its header says");
+ break;
+ }
+ check.Append(output.AsSpan(result.Count, read));
+ result.Count += read;
+ }
+ CheckBlockEnd(input, input.Consumed, declaredPacked, declaredUnpacked, result.Count, checkType, check, name);
+ }
+ catch (Exception ex)
+ {
+ result.Error = ExceptionDispatchInfo.Capture(ex);
+ }
+ return result;
+ }
+
+ private void FinishBlock()
+ {
+ var packed = _input.Consumed - _blockStart - _blockHeaderSize;
+ var checkSize = CheckBlockEnd(_input, packed, _declaredPacked, _declaredUnpacked, _blockUnpacked, _checkType, _check!, _name);
+ _blocks.Add((_blockHeaderSize + packed + checkSize, _blockUnpacked));
_block = null;
_check = null;
_step = Step.BlockOrIndex;
}
+ ///
+ /// Compares a block's sizes with its header, reads its padding and check from and
+ /// compares the check; returns the check's size.
+ ///
+ private static int CheckBlockEnd(ByteInput input, long packed, long declaredPacked, long declaredUnpacked, long unpacked, int checkType,
+ Check check, string name)
+ {
+ if (declaredPacked >= 0 && packed != declaredPacked)
+ throw ArchiveSafety.Damaged(name, "a block's packed size does not match its header");
+ if (declaredUnpacked >= 0 && unpacked != declaredUnpacked)
+ throw ArchiveSafety.Damaged(name, "a block's unpacked size does not match its header");
+ for (var padding = (4 - (int)(packed % 4)) % 4; padding > 0; padding--)
+ {
+ var b = input.ReadByte();
+ if (b < 0)
+ throw ArchiveSafety.Damaged(name, "the data ends too early");
+ if (b != 0)
+ throw ArchiveSafety.Damaged(name, "a block has data in its padding");
+ }
+
+ var stored = new byte[XzFormat.CheckSize(checkType)];
+ input.ReadExactly(stored);
+ if (!check.Matches(stored))
+ throw ArchiveSafety.Damaged(name, "the check of the unpacked data does not match");
+ return stored.Length;
+ }
+
private void ReadIndexAndFooter()
{
// The indicator byte (0) is already read; the index CRC covers it.
@@ -315,8 +497,13 @@ private int Next()
protected override void Dispose(bool disposing)
{
- if (disposing && !_leaveOpen)
- _stream.Dispose();
+ if (disposing)
+ {
+ _cancel?.Cancel();
+ _cancel?.Dispose();
+ if (!_leaveOpen)
+ _stream.Dispose();
+ }
base.Dispose(disposing);
}
diff --git a/src/Filee.Engines/Archives/NsisReader.cs b/src/Filee.Engines/Archives/NsisReader.cs
index 081c0cfe5..05b92cca0 100644
--- a/src/Filee.Engines/Archives/NsisReader.cs
+++ b/src/Filee.Engines/Archives/NsisReader.cs
@@ -387,7 +387,7 @@ public void Write(string folder, NsisScope scope, CancellationToken cancellation
// Sizes are known only once decoded: the listing checks names, paths and counts, the budget the bytes.
ArchiveSafety.CheckListing([.. files.Select(f => new ArchiveItem(f.Path, false, 0))], _name, _file.Length, folder);
CheckFolders(files, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(_name, _file.Length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(_name, _file.Length, folder), cancellationToken);
long position = 0; // in a solid stream, after the header block
foreach (var group in files.GroupBy(f => f.Offset).OrderBy(g => g.Key))
{
@@ -411,10 +411,7 @@ public void Write(string folder, NsisScope scope, CancellationToken cancellation
}
// The same data under other names: copies of the file just written.
foreach (var other in group.Skip(1))
- {
- using var copy = new FileStream(written, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16);
- writer.AddFile(other.Path, copy, copy.Length, null, other.Modified);
- }
+ writer.AddCopy(other.Path, written, other.Modified);
}
writer.Finish();
}
diff --git a/src/Filee.Engines/Archives/SevenZipCoders.cs b/src/Filee.Engines/Archives/SevenZipCoders.cs
index e8c509455..4b80c5b05 100644
--- a/src/Filee.Engines/Archives/SevenZipCoders.cs
+++ b/src/Filee.Engines/Archives/SevenZipCoders.cs
@@ -2,9 +2,10 @@
// stream of the archive or the output of another coder), and every coder's output is held to the unpacked size the
// header names. Filee's own LZMA and LZMA2 decoders (Archives/Lzma), the Copy coder, the x86 branch converters BCJ and
// BCJ2, the branch converters for ARM64, ARM, ARM Thumb, PowerPC, SPARC and IA-64 (7-Zip picks ARM64 for ARM64
-// programs by itself) and the Delta filter are written here; Deflate comes from .NET and BZip2 from SharpCompress
-// (MIT), as for the .bz2 files Filee reads. Other methods (PPMd, RISC-V …) are reported as unsupported before anything
-// is unpacked.
+// programs by itself) and the Delta filter are written here; Deflate comes from .NET and BZip2 from Filee's own
+// decoder (Bzip2Decoder, on threads of its own), as for the .bz2 files Filee reads; in the rare folder where BZip2
+// shares the archive with other packed streams (read on this thread), from SharpCompress (MIT). Other methods (PPMd,
+// RISC-V …) are reported as unsupported before anything is unpacked.
using System.Buffers.Binary;
using System.IO.Compression;
@@ -70,7 +71,12 @@ internal static class SevenZipCoders
/// The decoder for reading , giving exactly
/// bytes (more or fewer is damage). Properties are checked here.
///
- public static Stream Create(SevenZipCoder coder, Stream[] inputs, long size, string name, CancellationToken cancellationToken)
+ ///
+ /// True when the folder has a single packed stream: then a decoder may read it on a thread of its own (nothing else
+ /// reads the archive meanwhile).
+ ///
+ public static Stream Create(SevenZipCoder coder, Stream[] inputs, long size, string name, CancellationToken cancellationToken,
+ bool onePackedStream = false)
{
var props = coder.Properties;
Stream decoder;
@@ -93,7 +99,9 @@ public static Stream Create(SevenZipCoder coder, Stream[] inputs, long size, str
if (props.Length < 1 || XzFormat.DictionaryFromProperty(props[0]) is not { } dictionary)
throw ArchiveSafety.Damaged(name, "the LZMA2 dictionary size is out of range");
CheckDictionary(dictionary, name);
- decoder = new Lzma2Stream(new Lzma2Reader(new ByteInput(inputs[0]), dictionary, size, cancellationToken));
+ // Data with several dictionary resets (as 7-Zip writes with several threads) is decoded on several.
+ decoder = (inputs[0] is SharedWindowStream window ? Lzma2Parallel.TryOpen(window.Window, dictionary, size, cancellationToken) : null)
+ ?? new Lzma2Stream(new Lzma2Reader(new ByteInput(inputs[0]), dictionary, size, cancellationToken));
break;
}
case Bcj:
@@ -112,7 +120,12 @@ public static Stream Create(SevenZipCoder coder, Stream[] inputs, long size, str
decoder = new DeflateStream(inputs[0], CompressionMode.Decompress, leaveOpen: true);
break;
case Bzip2:
- decoder = new BZip2Stream(new NonClosingStream(inputs[0]), SharpCompressMode.Decompress, decompressConcatenated: false);
+ // Filee's decoder reads its input on a thread of its own: only where nothing else reads the archive.
+ // SharpCompress also takes over randomised blocks (old encoders), from the start of the packed stream.
+ decoder = onePackedStream && inputs[0] is SharedWindowStream packed
+ ? Bzip2Decoder.OpenRead(packed, cancellationToken,
+ () => new BZip2Stream(packed.Restart(), SharpCompressMode.Decompress, decompressConcatenated: false))
+ : new BZip2Stream(new NonClosingStream(inputs[0]), SharpCompressMode.Decompress, decompressConcatenated: false);
break;
default:
throw new SevenZipUnsupportedException(NameOf(coder.Method));
@@ -186,8 +199,15 @@ protected override void Dispose(bool disposing)
///
internal sealed class SharedWindowStream(Stream shared, long offset, long length) : ReadOnlyDecoderStream
{
- private long _position = offset;
+ private readonly long _start = offset;
private readonly long _end = offset + length;
+ private long _position = offset;
+
+ /// The same bytes, for reading at any place ().
+ public PackedWindow Window => new(shared, _start, _end - _start);
+
+ /// A new window over the same bytes, from their start.
+ public SharedWindowStream Restart() => new(shared, _start, _end - _start);
public override int Read(Span buffer)
{
diff --git a/src/Filee.Engines/Archives/SevenZipReader.cs b/src/Filee.Engines/Archives/SevenZipReader.cs
index 9f1e3e495..7468a5252 100644
--- a/src/Filee.Engines/Archives/SevenZipReader.cs
+++ b/src/Filee.Engines/Archives/SevenZipReader.cs
@@ -179,7 +179,7 @@ public static void Extract(Stream stream, string name, string folder, Cancellati
ArchiveSafety.CheckListing([.. listing.Items.Where(i => !i.IsAnti).Select(i => new ArchiveItem(i.Name, i.IsFolder, i.IsFolder ? 0 : i.Size, IsLink: i.IsLink))],
name, length, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(name, length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, length, folder), cancellationToken);
using var folders = new FolderSequence(stream, listing, name, cancellationToken);
foreach (var item in listing.Items)
{
@@ -746,7 +746,8 @@ Stream OpenCoder(int coder)
var pack = folder.FirstPackStream + Array.IndexOf(folder.PackedStreams, input);
inputs[j] = new SharedWindowStream(stream, listing.PackOffsets[pack], listing.PackSizes[pack]);
}
- return SevenZipCoders.Create(c, inputs, folder.UnpackSizes[firstOut[coder]], name, cancellationToken);
+ return SevenZipCoders.Create(c, inputs, folder.UnpackSizes[firstOut[coder]], name, cancellationToken,
+ onePackedStream: folder.PackedStreams.Length == 1);
}
}
}
diff --git a/src/Filee.Engines/Archives/TarArchiveReader.cs b/src/Filee.Engines/Archives/TarArchiveReader.cs
index 176596d77..1766dba9f 100644
--- a/src/Filee.Engines/Archives/TarArchiveReader.cs
+++ b/src/Filee.Engines/Archives/TarArchiveReader.cs
@@ -37,7 +37,7 @@ public static void Extract(Stream stream, string name, string folder, Cancellati
}, cancellationToken);
ArchiveSafety.CheckListing(items, name, stream.Length, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
stream.Position = 0;
Read(stream, name, copyData: false, entry =>
{
diff --git a/src/Filee.Engines/Archives/XzWriter.cs b/src/Filee.Engines/Archives/XzWriter.cs
index fe483d250..abc80a1fd 100644
--- a/src/Filee.Engines/Archives/XzWriter.cs
+++ b/src/Filee.Engines/Archives/XzWriter.cs
@@ -1,8 +1,11 @@
// Writes .xz files (The .xz File Format 1.2.1) with Filee's own LZMA2 encoder (Archives/Lzma): one stream with a
-// CRC-64 check, one block whose header names the LZMA2 filter and its dictionary (no sizes, since the data is
-// written as it comes), the index with that block's unpadded and unpacked size, and the footer. Nothing is written
-// for an empty input but the stream header, an empty index and the footer, as xz does. It is a write-only stream, so a
-// TAR can be compressed while it is written (.tar.xz) without a temporary file.
+// CRC-64 check and one block per LZMA2 block (Lzma2Writer cuts the input into blocks of a few dictionaries and codes
+// them on several threads). Each block header names the LZMA2 filter, its dictionary and both sizes, so readers can
+// unpack the blocks side by side (as xz and 7-Zip do with several threads, and Filee's XzReader); then the block's
+// data, padding to four bytes and the CRC-64 of what it unpacks to. The index lists every block's unpadded and
+// unpacked size, then the footer. Nothing is written for an empty input but the stream header, an empty index and the
+// footer, as xz does. It is a write-only stream, so a TAR can be compressed while it is written (.tar.xz) without a
+// temporary file.
using System.Buffers.Binary;
using Filee.Core.Presets;
@@ -13,17 +16,15 @@ namespace Filee.Engines.Archives;
/// A write-only stream that compresses into an .xz file.
internal sealed class XzWriter : Stream
{
- private const int BlockHeaderSize = 12;
private const int CheckSize = 8;
private readonly Stream _output;
private readonly bool _leaveOpen;
- private readonly Lzma2Settings _settings;
- private readonly long _expectedSize;
- private readonly CancellationToken _cancellationToken;
- private readonly Crc64 _check = new();
+ private readonly Lzma2Writer _lzma2;
+ private readonly byte _property;
private readonly byte[] _flags = [0x00, XzFormat.CheckCrc64];
- private Lzma2Writer? _block;
+ private readonly List<(long Unpadded, long Unpacked)> _records = [];
+ private readonly CancellationToken _cancellationToken;
private long _unpacked;
private bool _finished;
@@ -36,9 +37,10 @@ public XzWriter(Stream output, ArchiveLevel level, long expectedSize = -1, Cance
{
_output = output;
_leaveOpen = leaveOpen;
- _settings = Lzma2Settings.For(level, expectedSize);
- _expectedSize = expectedSize;
+ var settings = Lzma2Settings.For(level, expectedSize);
+ _property = XzFormat.PropertyFor((uint)settings.Dictionary);
_cancellationToken = cancellationToken;
+ _lzma2 = new Lzma2Writer(output, settings, expectedSize, cancellationToken, WriteBlock);
Span header = stackalloc byte[12];
XzFormat.HeaderMagic.CopyTo(header);
@@ -55,53 +57,53 @@ public override void Write(ReadOnlySpan buffer)
if (buffer.IsEmpty)
return;
_cancellationToken.ThrowIfCancellationRequested();
- if (_block is null)
- StartBlock();
- _check.Append(buffer);
_unpacked += buffer.Length;
- _block!.Write(buffer);
+ _lzma2.Write(buffer);
}
- private void StartBlock()
+ /// One .xz block: header with both sizes, the LZMA2 data and its end byte, padding, CRC-64.
+ private void WriteBlock(Lzma2Block block)
{
- // Size / 4 − 1, flags (one filter, no sizes), filter id 0x21, 1 property byte, the dictionary, padding, CRC-32.
- Span header = stackalloc byte[BlockHeaderSize];
+ var packed = block.Chunks.Length + 1L;
+ // Size / 4 − 1, flags (one filter, both sizes), the sizes, filter id 0x21, 1 property byte, the dictionary,
+ // padding to four bytes, CRC-32.
+ Span header = stackalloc byte[32];
header.Clear();
- header[0] = BlockHeaderSize / 4 - 1;
- header[1] = 0x00;
- header[2] = (byte)XzFormat.Lzma2FilterId;
- header[3] = 1;
- header[4] = XzFormat.PropertyFor((uint)_settings.Dictionary);
- BinaryPrimitives.WriteUInt32LittleEndian(header[8..], Crc32.Compute(header[..8]));
- _output.Write(header);
- _block = new Lzma2Writer(_output, _settings, _expectedSize, _cancellationToken);
+ header[1] = 0xC0;
+ var at = 2;
+ at += XzFormat.WriteVarint(header[at..], (ulong)packed);
+ at += XzFormat.WriteVarint(header[at..], (ulong)block.Unpacked);
+ header[at++] = (byte)XzFormat.Lzma2FilterId;
+ header[at++] = 1;
+ header[at++] = _property;
+ at = (at + 3) & ~3;
+ header[0] = (byte)(at / 4);
+ BinaryPrimitives.WriteUInt32LittleEndian(header[at..], Crc32.Compute(header[..at]));
+ at += 4;
+ _output.Write(header[..at]);
+ _output.Write(block.Chunks);
+
+ Span tail = stackalloc byte[1 + 3 + CheckSize];
+ tail.Clear();
+ var padding = (int)((4 - packed % 4) % 4);
+ BinaryPrimitives.WriteUInt64LittleEndian(tail[(1 + padding)..], block.Check);
+ _output.Write(tail[..(1 + padding + CheckSize)]);
+ _records.Add((at + packed + CheckSize, block.Unpacked));
}
- /// Compresses what is left and writes the block's end, the index and the footer.
+ /// Compresses what is left and writes the last blocks, the index and the footer.
public void Finish()
{
if (_finished)
return;
_finished = true;
-
- var records = new List<(long Unpadded, long Unpacked)>();
- if (_block is not null)
- {
- _block.Finish();
- var packed = _block.Written;
- Span tail = stackalloc byte[3 + CheckSize];
- tail.Clear();
- var padding = (int)((4 - packed % 4) % 4);
- BinaryPrimitives.WriteUInt64LittleEndian(tail[padding..], _check.Value);
- _output.Write(tail[..(padding + CheckSize)]);
- records.Add((BlockHeaderSize + packed + CheckSize, _unpacked));
- }
+ _lzma2.Finish();
// Index: indicator 0, record count, (unpadded, unpacked) per block, padding to four bytes, CRC-32.
- var index = new byte[1 + 9 + records.Count * 18 + 3 + 4];
+ var index = new byte[1 + 9 + _records.Count * 18 + 3 + 4];
var at = 1;
- at += XzFormat.WriteVarint(index.AsSpan(at), (ulong)records.Count);
- foreach (var (unpadded, unpacked) in records)
+ at += XzFormat.WriteVarint(index.AsSpan(at), (ulong)_records.Count);
+ foreach (var (unpadded, unpacked) in _records)
{
at += XzFormat.WriteVarint(index.AsSpan(at), (ulong)unpadded);
at += XzFormat.WriteVarint(index.AsSpan(at), (ulong)unpacked);
diff --git a/src/Filee.Engines/Archives/ZipReader.cs b/src/Filee.Engines/Archives/ZipReader.cs
index e82e0e7b1..dd013f017 100644
--- a/src/Filee.Engines/Archives/ZipReader.cs
+++ b/src/Filee.Engines/Archives/ZipReader.cs
@@ -72,7 +72,7 @@ public static bool TryExtract(Stream stream, string name, string folder, Cancell
ArchiveSafety.CheckListing([.. entries.Select(e => new ArchiveItem(e.FullName, IsFolder(e), IsFolder(e) ? 0 : e.Length, IsLink: IsLink(e)))],
name, stream.Length, folder);
- var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
+ using var writer = new EntryWriter(folder, new UnpackBudget(name, stream.Length, folder), cancellationToken);
foreach (var entry in entries)
{
cancellationToken.ThrowIfCancellationRequested();
diff --git a/tests/Filee.Engines.Tests/ArchiveThreadTests.cs b/tests/Filee.Engines.Tests/ArchiveThreadTests.cs
new file mode 100644
index 000000000..d75db106b
--- /dev/null
+++ b/tests/Filee.Engines.Tests/ArchiveThreadTests.cs
@@ -0,0 +1,383 @@
+// Archives coded and decoded on several threads (Lzma2Writer blocks, Lzma2Parallel, XzReader blocks, Bzip2Decoder's
+// stream): the same bytes as on one thread, the same errors at the same place, 7-Zip reads what Filee writes (skipped
+// when it is missing), and cancellation stops the workers. Block sizes are made small so that little data gives many
+// blocks.
+
+using Filee.Core.Presets;
+using Filee.Engines.Archives;
+using Filee.Engines.Archives.Lzma;
+using static Filee.Engines.Tests.ArchiveBuilders;
+
+namespace Filee.Engines.Tests;
+
+public class ArchiveThreadTests(EngineFixture fx) : IClassFixture
+{
+ private static CancellationToken Ct => TestContext.Current.CancellationToken;
+
+ private const int SmallBlock = 256 << 10;
+
+ /// Runs with LZMA2 blocks of and the given threads.
+ private static T WithBlocks(int? threads, Func action)
+ {
+ Lzma2Writer.TestBlockSize.Value = SmallBlock;
+ Lzma2Writer.TestThreads.Value = threads;
+ try
+ {
+ return action();
+ }
+ finally
+ {
+ Lzma2Writer.TestBlockSize.Value = null;
+ Lzma2Writer.TestThreads.Value = null;
+ }
+ }
+
+ /// Runs with decoding threads off (false) or on for any data (true).
+ private static T Decoding(bool threads, Func action)
+ {
+ Lzma2Parallel.TestMode.Value = threads;
+ try
+ {
+ return action();
+ }
+ finally
+ {
+ Lzma2Parallel.TestMode.Value = null;
+ }
+ }
+
+ private static byte[] Xz(byte[] data, ArchiveLevel level, int? threads) => WithBlocks(threads, () =>
+ {
+ using var output = new MemoryStream();
+ using (var xz = new XzWriter(output, level, data.Length, Ct, leaveOpen: true))
+ {
+ for (var at = 0; at < data.Length; at += 100_003)
+ xz.Write(data, at, Math.Min(100_003, data.Length - at));
+ xz.Finish();
+ }
+ return output.ToArray();
+ });
+
+ /// Reads all of in reads of odd sizes.
+ private static byte[] ReadAll(Stream stream)
+ {
+ using var output = new MemoryStream();
+ var buffer = new byte[70_001];
+ var random = new Random(5);
+ int read;
+ while ((read = stream.Read(buffer, 0, random.Next(1, buffer.Length))) > 0)
+ output.Write(buffer, 0, read);
+ return output.ToArray();
+ }
+
+ private static byte[] Unxz(byte[] xz, bool threads) => Decoding(threads, () =>
+ {
+ using var reader = new XzReader(new MemoryStream(xz), "test.xz", cancellationToken: Ct);
+ return ReadAll(reader);
+ });
+
+ /// The bytes read before the error and the error, of reading through.
+ private static (int Read, Exception Error) UnxzFailing(byte[] xz, bool threads) => Decoding(threads, () =>
+ {
+ using var reader = new XzReader(new MemoryStream(xz), "test.xz", cancellationToken: Ct);
+ var buffer = new byte[1 << 16];
+ var total = 0;
+ try
+ {
+ int read;
+ while ((read = reader.Read(buffer)) > 0)
+ total += read;
+ }
+ catch (Exception ex)
+ {
+ return (total, ex);
+ }
+ throw new Xunit.Sdk.XunitException("the damaged .xz read without an error");
+ });
+
+ [Theory]
+ [InlineData(ArchiveLevel.Store)]
+ [InlineData(ArchiveLevel.Fast)]
+ [InlineData(ArchiveLevel.Normal)]
+ public void Lzma2_blocks_give_the_same_bytes_on_one_thread_or_four(ArchiveLevel level)
+ {
+ var data = Bzip2WriterTests.Mixed(2_000_000);
+ var one = Xz(data, level, threads: 1);
+ var four = Xz(data, level, threads: 4);
+ Assert.Equal(one, four);
+ Assert.Equal(data, Unxz(four, threads: false));
+ Assert.Equal(data, Unxz(four, threads: true));
+ }
+
+ [Theory]
+ [InlineData(ArchiveLevel.Normal)]
+ [InlineData(ArchiveLevel.Maximum)]
+ public void The_match_finder_on_its_own_thread_gives_the_same_bytes(ArchiveLevel level)
+ {
+ var data = LzmaTests.Prose(700_000);
+ byte[] Coded(bool ahead)
+ {
+ Lzma2Writer.TestSearchAhead.Value = ahead;
+ try
+ {
+ return Xz(data, level, threads: 1);
+ }
+ finally
+ {
+ Lzma2Writer.TestSearchAhead.Value = null;
+ }
+ }
+ var alone = Coded(false);
+ Assert.Equal(alone, Coded(true));
+ Assert.Equal(data, Unxz(alone, threads: true));
+ }
+
+ [Fact]
+ public async Task An_xz_of_several_blocks_names_its_sizes_and_opens_in_7zip()
+ {
+ var data = LzmaTests.Prose(1_500_000);
+ var xz = Xz(data, ArchiveLevel.Normal, threads: 3);
+ // One block header per 256 KiB block, each with both sizes (flags 0xC0).
+ var blocks = 0;
+ for (var at = 12; xz[at] != 0;)
+ {
+ Assert.Equal(0xC0, xz[at + 1]);
+ var size = (xz[at] + 1) * 4;
+ var packed = 0L;
+ var shift = 0;
+ var p = at + 2;
+ do
+ {
+ packed |= (long)(xz[p] & 0x7F) << shift;
+ shift += 7;
+ } while ((xz[p++] & 0x80) != 0);
+ at += size + (int)((packed + 3) & ~3L) + 8;
+ blocks++;
+ }
+ Assert.Equal((data.Length + SmallBlock - 1) / SmallBlock, blocks);
+ if (SevenZipExe is null)
+ return;
+ var dir = fx.NewFolder();
+ var file = Path.Combine(dir, "data.bin.xz");
+ await File.WriteAllBytesAsync(file, xz, Ct);
+ await Run7zAsync("t", file);
+ await Run7zAsync("e", "-y", "-o" + Path.Combine(dir, "out"), file);
+ Assert.Equal(data, await File.ReadAllBytesAsync(Path.Combine(dir, "out", "data.bin"), Ct));
+ }
+
+ [Fact]
+ public void Damage_in_a_later_xz_block_is_the_same_error_on_threads()
+ {
+ var data = LzmaTests.Prose(1_500_000);
+ var xz = Xz(data, ArchiveLevel.Fast, threads: 2);
+ foreach (var at in new[] { xz.Length / 2, xz.Length * 3 / 4, xz.Length - 40 })
+ {
+ var damaged = (byte[])xz.Clone();
+ damaged[at] ^= 0x20;
+ var (oneRead, oneError) = UnxzFailing(damaged, threads: false);
+ var (manyRead, manyError) = UnxzFailing(damaged, threads: true);
+ Assert.Equal(oneError.GetType(), manyError.GetType());
+ Assert.Equal(oneError.Message, manyError.Message);
+ // The damaged block is decoded again on the reader's thread: the same bytes come out before the error.
+ Assert.Equal(oneRead, manyRead);
+ }
+ // Cut off in the middle: the same error as well.
+ var cut = xz[..(xz.Length * 2 / 3)];
+ Assert.Equal(UnxzFailing(cut, threads: false).Error.Message, UnxzFailing(cut, threads: true).Error.Message);
+ }
+
+ [Fact]
+ public void A_cancelled_xz_read_on_threads_stops()
+ {
+ var xz = Xz(LzmaTests.Prose(1_200_000), ArchiveLevel.Fast, threads: 2);
+ using var cancel = new CancellationTokenSource();
+ Lzma2Parallel.TestMode.Value = true;
+ try
+ {
+ using var reader = new XzReader(new MemoryStream(xz), "test.xz", cancellationToken: cancel.Token);
+ var buffer = new byte[1 << 16];
+ Assert.True(reader.Read(buffer) > 0);
+ cancel.Cancel();
+ Assert.ThrowsAny(() =>
+ {
+ while (reader.Read(buffer) > 0)
+ {
+ }
+ });
+ }
+ finally
+ {
+ Lzma2Parallel.TestMode.Value = null;
+ }
+ }
+
+ /// A 7z of one solid folder whose LZMA2 data has a dictionary reset every bytes.
+ private static byte[] SevenZip(params (string Name, byte[] Data)[] files) => WithBlocks(threads: 2, () =>
+ {
+ using var output = new MemoryStream();
+ var writer = new SevenZipWriter(output, ArchiveLevel.Fast, files.Sum(f => (long)f.Data.Length), Ct);
+ foreach (var (name, data) in files)
+ writer.AddFile(name, new MemoryStream(data), data.Length, DateTime.UtcNow, 0x20);
+ writer.Finish();
+ return output.ToArray();
+ });
+
+ private Dictionary Un7z(byte[] archive, bool threads) => Decoding(threads, () =>
+ {
+ var dir = fx.NewFolder();
+ SevenZipReader.Extract(new MemoryStream(archive), "test.7z", dir, Ct);
+ return Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories)
+ .ToDictionary(f => Path.GetRelativePath(dir, f).Replace('\\', '/'), File.ReadAllBytes);
+ });
+
+ [Fact]
+ public async Task A_7z_with_dictionary_resets_unpacks_on_threads_like_on_one()
+ {
+ var files = new[] { ("a.txt", LzmaTests.Prose(900_000)), ("b.bin", Noise(300_000, 3)), ("c.txt", LzmaTests.Prose(700_000)) };
+ var archive = SevenZip(files);
+ foreach (var threads in new[] { false, true })
+ {
+ var opened = Lzma2Parallel.Opened;
+ var unpacked = Un7z(archive, threads);
+ foreach (var (name, data) in files)
+ Assert.Equal(data, unpacked[name]);
+ if (threads)
+ Assert.True(Lzma2Parallel.Opened > opened, "the LZMA2 data was not decoded on threads");
+ }
+ if (SevenZipExe is null)
+ return;
+ var dir = fx.NewFolder();
+ var file = Path.Combine(dir, "test.7z");
+ await File.WriteAllBytesAsync(file, archive, Ct);
+ await Run7zAsync("t", file);
+ }
+
+ [Fact]
+ public void Damage_in_a_later_7z_segment_is_the_same_error_on_threads()
+ {
+ var archive = SevenZip(("a.txt", LzmaTests.Prose(1_200_000)), ("b.txt", LzmaTests.Prose(600_000)));
+ // Past the signature header, somewhere in the second half of the packed data.
+ foreach (var at in new[] { archive.Length / 2, archive.Length * 2 / 3 })
+ {
+ var damaged = (byte[])archive.Clone();
+ damaged[at] ^= 0x08;
+ Exception Fail(bool threads) => Assert.ThrowsAny(() => Un7z(damaged, threads));
+ var one = Fail(false);
+ var many = Fail(true);
+ Assert.Equal(one.GetType(), many.GetType());
+ Assert.Equal(one.Message, many.Message);
+ }
+ }
+
+ // ───────── bzip2 ─────────
+
+ private static byte[] Bz2(byte[] data)
+ {
+ using var output = new MemoryStream();
+ using (var writer = new Bzip2Writer(output, ArchiveLevel.Fast, Ct, leaveOpen: true))
+ {
+ writer.Write(data);
+ writer.Finish();
+ }
+ return output.ToArray();
+ }
+
+ private static byte[] PushDecode(byte[] bz2)
+ {
+ using var output = new MemoryStream();
+ Bzip2Decoder.Decompress(new MemoryStream(bz2), output, alzFraming: false, long.MaxValue, Ct);
+ return output.ToArray();
+ }
+
+ private static byte[] StreamDecode(byte[] bz2)
+ {
+ using var stream = Bzip2Decoder.OpenRead(new MemoryStream(bz2), Ct);
+ return ReadAll(stream);
+ }
+
+ [Theory]
+ [InlineData(Bzip2WriterTests.Kind.Empty, 0)]
+ [InlineData(Bzip2WriterTests.Kind.One, 1)]
+ [InlineData(Bzip2WriterTests.Kind.Text, 1_000_000)]
+ [InlineData(Bzip2WriterTests.Kind.Runs, 1_500_000)]
+ [InlineData(Bzip2WriterTests.Kind.Mixed, 2_500_000)]
+ [InlineData(Bzip2WriterTests.Kind.Edge, 0)]
+ public void The_bzip2_stream_reads_what_the_writer_wrote(Bzip2WriterTests.Kind kind, int length)
+ {
+ var data = Bzip2WriterTests.Data(kind, length);
+ var bz2 = Bz2(data);
+ Assert.Equal(data, StreamDecode(bz2));
+ // Two streams one after the other (as pbzip2 writes, or "cat a.bz2 b.bz2").
+ Assert.Equal([.. data, .. data], StreamDecode([.. bz2, .. bz2]));
+ }
+
+ [Fact]
+ public void Damaged_bzip2_gives_the_same_error_as_the_one_thread_decoder()
+ {
+ var data = Bzip2WriterTests.Mixed(2_500_000);
+ var bz2 = Bz2(data);
+ var random = new Random(9);
+ for (var i = 0; i < 30; i++)
+ {
+ var damaged = (byte[])bz2.Clone();
+ if (i < 25)
+ damaged[random.Next(4, damaged.Length)] ^= (byte)(1 << random.Next(8));
+ else
+ damaged = damaged[..random.Next(4, damaged.Length)];
+ Exception? push = null, pull = null;
+ try
+ {
+ PushDecode(damaged);
+ }
+ catch (Exception ex)
+ {
+ push = ex;
+ }
+ try
+ {
+ StreamDecode(damaged);
+ }
+ catch (Exception ex)
+ {
+ pull = ex;
+ }
+ Assert.Equal(push?.GetType(), pull?.GetType());
+ Assert.Equal(push?.Message, pull?.Message);
+ }
+ }
+
+ [Fact]
+ public void Randomised_bzip2_blocks_go_to_the_fallback_decoder_past_what_was_read()
+ {
+ // SharpCompress's encoder randomises very repetitive blocks, as bzip2 0.9.0 did; text first, so some blocks
+ // come out of Filee's decoder before the randomised one.
+ byte[] data = [.. LzmaTests.Prose(2_000_000), .. new byte[3_000_000], .. LzmaTests.Prose(300_000)];
+ var bz2 = Bzip2(data);
+ var error = Assert.Throws(() => PushDecode(bz2));
+ Assert.Contains("randomised", error.Message);
+ Stream Fallback() => new SharpCompress.Compressors.BZip2.BZip2Stream(new MemoryStream(bz2),
+ SharpCompress.Compressors.CompressionMode.Decompress, decompressConcatenated: true);
+ using (var stream = Bzip2Decoder.OpenRead(new MemoryStream(bz2), Ct, Fallback))
+ Assert.Equal(data, ReadAll(stream));
+ // Without one, the stream refuses it as before.
+ using (var stream = Bzip2Decoder.OpenRead(new MemoryStream(bz2), Ct))
+ Assert.Contains("randomised", Assert.Throws(() => ReadAll(stream)).Message);
+ }
+
+ [Fact]
+ public void A_cancelled_bzip2_stream_stops()
+ {
+ var bz2 = Bz2(LzmaTests.Prose(3_000_000));
+ using var cancel = new CancellationTokenSource();
+ using var stream = Bzip2Decoder.OpenRead(new MemoryStream(bz2), cancel.Token);
+ var buffer = new byte[1 << 16];
+ Assert.True(stream.Read(buffer) > 0);
+ cancel.Cancel();
+ Assert.ThrowsAny(() =>
+ {
+ while (stream.Read(buffer) > 0)
+ {
+ }
+ });
+ }
+}
diff --git a/tests/Filee.Engines.Tests/LzmaTests.cs b/tests/Filee.Engines.Tests/LzmaTests.cs
index 1e14ec520..91f098e7a 100644
--- a/tests/Filee.Engines.Tests/LzmaTests.cs
+++ b/tests/Filee.Engines.Tests/LzmaTests.cs
@@ -157,7 +157,7 @@ public void The_window_slides_past_a_small_dictionary()
// 4 KiB dictionary (input size unknown to the writer is not the case here: say it's tiny), 3 MB of input.
var data = Mixed(3_000_000);
var xz = Compress(data, ArchiveLevel.Normal, expected: 1000);
- Assert.Equal(12, XzFormat.DictionaryFromProperty(xz[16]) is { } d ? System.Numerics.BitOperations.Log2(d) : 0);
+ Assert.Equal(12, XzFormat.DictionaryFromProperty(xz[PropertyOffset(xz)]) is { } d ? System.Numerics.BitOperations.Log2(d) : 0);
Assert.Equal(data, Decompress(xz));
// And with the input size not known at all (full level dictionary).
Assert.Equal(data, Decompress(Compress(data, ArchiveLevel.Fast, expected: -1)));
@@ -293,12 +293,26 @@ byte[] Changed(int at, byte value)
private static byte[] WithDictionaryProperty(byte property)
{
var xz = Compress(Prose(10_000), ArchiveLevel.Store);
- Assert.Equal(0x21, xz[14]);
- xz[16] = property;
- BinaryPrimitives.WriteUInt32LittleEndian(xz.AsSpan(20), Crc32.Compute(xz.AsSpan(12, 8)));
+ xz[PropertyOffset(xz)] = property;
+ var size = (xz[12] + 1) * 4;
+ BinaryPrimitives.WriteUInt32LittleEndian(xz.AsSpan(12 + size - 4), Crc32.Compute(xz.AsSpan(12, size - 4)));
return xz;
}
+ /// Where the dictionary property of the first block header is: after the sizes the header may name.
+ private static int PropertyOffset(byte[] xz)
+ {
+ var at = 14;
+ var flags = xz[13];
+ for (var sizes = ((flags & 0x40) != 0 ? 1 : 0) + ((flags & 0x80) != 0 ? 1 : 0); sizes > 0; sizes--)
+ while ((xz[at++] & 0x80) != 0)
+ {
+ }
+ Assert.Equal(0x21, xz[at]);
+ Assert.Equal(1, xz[at + 1]);
+ return at + 2;
+ }
+
[Fact]
public void A_hostile_dictionary_size_is_refused_before_anything_is_allocated()
{