From 193d79fd63436b3bf698b7f0cf4d5baa7432e726 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 20:38:10 +0000 Subject: [PATCH] Moving pictures: Kitty animation frames A TerminalPicture can be made from TerminalPictureFrames, each a whole frame with its duration. Kitty is sent the later frames after the image (a=f,X=1,z=), the first frame's gap, and a looping start (a=a,s=3,v=1). Every other method draws the first frame, which is the picture's Rgba. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01448U5MvpKkRkBjwzkkpPFP --- .../Emitters/TerminalPictureWriter.cs | 54 ++++++++++++++++--- MarkupString.Ansi/PublicAPI.Unshipped.txt | 16 ++++++ MarkupString.Ansi/README.md | 4 ++ MarkupString.Ansi/TerminalPicture.cs | 45 ++++++++++++++-- .../Ansi/TerminalFeatureTests.cs | 47 ++++++++++++++++ 5 files changed, 155 insertions(+), 11 deletions(-) diff --git a/MarkupString.Ansi/Emitters/TerminalPictureWriter.cs b/MarkupString.Ansi/Emitters/TerminalPictureWriter.cs index 4c9bdcb..d605b21 100644 --- a/MarkupString.Ansi/Emitters/TerminalPictureWriter.cs +++ b/MarkupString.Ansi/Emitters/TerminalPictureWriter.cs @@ -156,19 +156,60 @@ private static void WriteKitty(PictureCellsMarkup cells, TerminalPicture picture /// so a connection shown the picture after the first is sent a copy. /// /// + /// /// Sent as PNG (f=100) rather than zlib-compressed raw pixels (f=32,o=z): the PNG's row /// filtering and dropped alpha make it about half the size for the same compression time, and every /// terminal that speaks the protocol decodes PNG. + /// + /// + /// A moving picture's first frame is the image; each later frame follows as a whole frame written over its + /// base (a=f,X=1) with its own duration (z), then the first frame's duration is set and the + /// animation started looping (a=a,s=3,v=1). The terminal plays it from then on; nothing more is sent. + /// /// private static string EncodeKitty(TerminalPicture picture, KittyKey key) { var (width, height) = PictureScaler.FitWithin(picture.Width, picture.Height, key.Columns * key.CellWidth, key.Rows * key.CellHeight, upscale: false); - var pixels = PictureScaler.Scale(picture.Rgba.Span, picture.Width, picture.Height, width, height); + var text = new StringBuilder(); + AppendKittyChunks(text, picture.Rgba.Span, picture, width, height, + string.Create(CultureInfo.InvariantCulture, $"a=T,U=1,i={key.Id},f=100,c={key.Columns},r={key.Rows},q=2"), ""); + + if (picture.Frames.Count > 1) + { + for (var frame = 1; frame < picture.Frames.Count; frame++) + { + AppendKittyChunks(text, picture.Frames[frame].Rgba.Span, picture, width, height, + string.Create(CultureInfo.InvariantCulture, $"a=f,i={key.Id},f=100,X=1,z={KittyGap(picture.Frames[frame].Duration)},q=2"), + "a=f,"); + } + + text.Append(CultureInfo.InvariantCulture, + $"{Esc}_Ga=a,i={key.Id},r=1,z={KittyGap(picture.Frames[0].Duration)},q=2{StringTerminator}"); + text.Append(CultureInfo.InvariantCulture, $"{Esc}_Ga=a,i={key.Id},s=3,v=1,q=2{StringTerminator}"); + } + + return text.ToString(); + } + + /// + /// A frame's duration as Kitty's gap in milliseconds: at least one, since Kitty ignores a zero gap and + /// reads a negative one as no gap at all. + /// + private static int KittyGap(TimeSpan duration) => (int)Math.Clamp(duration.TotalMilliseconds, 1, int.MaxValue); + + /// + /// scaled to × , as a PNG in base64 + /// chunks of at most : the first carrying , the rest + /// (which a frame needs, a=f,) and each whether more follow. + /// + private static void AppendKittyChunks(StringBuilder text, ReadOnlySpan rgba, TerminalPicture picture, int width, int height, + string first, string rest) + { + var pixels = PictureScaler.Scale(rgba, picture.Width, picture.Height, width, height); var payload = Convert.ToBase64String(PngWriter.Encode(pixels, width, height)); - var chunks = (payload.Length + KittyChunk - 1) / KittyChunk; + text.EnsureCapacity(text.Length + payload.Length + 96 + payload.Length / KittyChunk * 20); - var text = new StringBuilder(payload.Length + 96 + chunks * 16); var offset = 0; do { @@ -177,20 +218,17 @@ private static string EncodeKitty(TerminalPicture picture, KittyKey key) text.Append(Esc + "_G"); if (offset == 0) { - text.Append(CultureInfo.InvariantCulture, - $"a=T,U=1,i={key.Id},f=100,c={key.Columns},r={key.Rows},q=2,m={more}"); + text.Append(first).Append(",m=").Append(more); } else { - text.Append("m=").Append(more).Append(",q=2"); + text.Append(rest).Append("m=").Append(more).Append(",q=2"); } text.Append(';').Append(payload, offset, length).Append(StringTerminator); offset += length; } while (offset < payload.Length); - - return text.ToString(); } /// diff --git a/MarkupString.Ansi/PublicAPI.Unshipped.txt b/MarkupString.Ansi/PublicAPI.Unshipped.txt index ddcf9fa..6b06033 100644 --- a/MarkupString.Ansi/PublicAPI.Unshipped.txt +++ b/MarkupString.Ansi/PublicAPI.Unshipped.txt @@ -28,14 +28,30 @@ MarkupString.Ansi.TerminalFeatures.None = 0 -> MarkupString.Ansi.TerminalFeature MarkupString.Ansi.TerminalFeatures.Pictures = MarkupString.Ansi.TerminalFeatures.KittyGraphics | MarkupString.Ansi.TerminalFeatures.InlineImages | MarkupString.Ansi.TerminalFeatures.Sixel | MarkupString.Ansi.TerminalFeatures.BlockArt -> MarkupString.Ansi.TerminalFeatures MarkupString.Ansi.TerminalFeatures.Sixel = 16 -> MarkupString.Ansi.TerminalFeatures MarkupString.Ansi.TerminalPicture +MarkupString.Ansi.TerminalPicture.Frames.get -> System.Collections.Generic.IReadOnlyList! MarkupString.Ansi.TerminalPicture.Height.get -> int MarkupString.Ansi.TerminalPicture.Key.get -> string! MarkupString.Ansi.TerminalPicture.Rgba.get -> System.ReadOnlyMemory +MarkupString.Ansi.TerminalPicture.TerminalPicture(string! key, int width, int height, System.Collections.Generic.IReadOnlyList! frames) -> void MarkupString.Ansi.TerminalPicture.TerminalPicture(string! key, int width, int height, System.ReadOnlyMemory rgba) -> void MarkupString.Ansi.TerminalPicture.Width.get -> int +MarkupString.Ansi.TerminalPictureFrame +MarkupString.Ansi.TerminalPictureFrame.Deconstruct(out System.ReadOnlyMemory Rgba, out System.TimeSpan Duration) -> void +MarkupString.Ansi.TerminalPictureFrame.Duration.get -> System.TimeSpan +MarkupString.Ansi.TerminalPictureFrame.Duration.init -> void +MarkupString.Ansi.TerminalPictureFrame.Equals(MarkupString.Ansi.TerminalPictureFrame other) -> bool +MarkupString.Ansi.TerminalPictureFrame.Rgba.get -> System.ReadOnlyMemory +MarkupString.Ansi.TerminalPictureFrame.Rgba.init -> void +MarkupString.Ansi.TerminalPictureFrame.TerminalPictureFrame() -> void +MarkupString.Ansi.TerminalPictureFrame.TerminalPictureFrame(System.ReadOnlyMemory Rgba, System.TimeSpan Duration) -> void override MarkupString.Ansi.AnsiOutputOptions.Equals(object? obj) -> bool override MarkupString.Ansi.AnsiOutputOptions.GetHashCode() -> int override MarkupString.Ansi.AnsiOutputOptions.ToString() -> string! +~override MarkupString.Ansi.TerminalPictureFrame.Equals(object obj) -> bool +override MarkupString.Ansi.TerminalPictureFrame.GetHashCode() -> int +~override MarkupString.Ansi.TerminalPictureFrame.ToString() -> string static MarkupString.Ansi.AnsiOutputOptions.operator !=(MarkupString.Ansi.AnsiOutputOptions? left, MarkupString.Ansi.AnsiOutputOptions? right) -> bool static MarkupString.Ansi.AnsiOutputOptions.operator ==(MarkupString.Ansi.AnsiOutputOptions? left, MarkupString.Ansi.AnsiOutputOptions? right) -> bool static MarkupString.Ansi.AnsiRegistration.WithAnsiOutput(this MarkupString.MarkupRegistry! registry, MarkupString.Ansi.AnsiOutputOptions! options) -> MarkupString.MarkupRegistry! +static MarkupString.Ansi.TerminalPictureFrame.operator !=(MarkupString.Ansi.TerminalPictureFrame left, MarkupString.Ansi.TerminalPictureFrame right) -> bool +static MarkupString.Ansi.TerminalPictureFrame.operator ==(MarkupString.Ansi.TerminalPictureFrame left, MarkupString.Ansi.TerminalPictureFrame right) -> bool diff --git a/MarkupString.Ansi/README.md b/MarkupString.Ansi/README.md index 71bb88b..bb59da3 100644 --- a/MarkupString.Ansi/README.md +++ b/MarkupString.Ansi/README.md @@ -81,6 +81,10 @@ neither fetches nor decodes a file. Without the pixels, or without the feature, nothing comes back as input) and writes each cell as `U+10EEEE` with row and column diacritics, the image id as a truecolor foreground. Placeholders are text: they wrap, scroll and are erased like it. `MarkTransmitted` is how the source says whether the terminal already has it. + A moving picture (a `TerminalPicture` made from `TerminalPictureFrame`s) sends its later frames + after the first (`a=f,X=1`, each with its duration as `z`), sets the first frame's duration, and + starts it looping (`a=a,s=3,v=1`); the terminal plays it with nothing more sent. Every other way + of drawing shows its first frame. - **iTerm2 and sixel** are pixels over the screen. On the picture's first row the cursor makes room below (`ESC D` per row), comes back up, and draws the picture between `ESC 7` and `ESC 8`; every row then steps over its cells with `CSI n C`, so no text is written over it. Sixel is the cells' size in diff --git a/MarkupString.Ansi/TerminalPicture.cs b/MarkupString.Ansi/TerminalPicture.cs index 1f2420e..273e777 100644 --- a/MarkupString.Ansi/TerminalPicture.cs +++ b/MarkupString.Ansi/TerminalPicture.cs @@ -19,17 +19,48 @@ public sealed class TerminalPicture /// is empty, or is not the size the dimensions give. /// A dimension is not positive. public TerminalPicture(string key, int width, int height, ReadOnlyMemory rgba) + : this(key, width, height, rgba, []) + { + } + + /// + /// Creates a moving picture from , each a whole frame as it is shown. The first + /// frame is the picture's , which is what a terminal that cannot animate draws. + /// + /// What identifies these frames, such as a hash of the file. + /// Its width in pixels. + /// Its height in pixels. + /// The frames in order, each × × 4 bytes; at least one. + /// is empty, there are no frames, or a frame is not the size the dimensions give. + /// A dimension is not positive. + public TerminalPicture(string key, int width, int height, IReadOnlyList frames) + : this(key, width, height, FirstOf(frames), frames.Count > 1 ? frames : []) + { + } + + private TerminalPicture(string key, int width, int height, ReadOnlyMemory rgba, IReadOnlyList frames) { ArgumentException.ThrowIfNullOrEmpty(key); ArgumentOutOfRangeException.ThrowIfNegativeOrZero(width); ArgumentOutOfRangeException.ThrowIfNegativeOrZero(height); - if ((long)width * height * 4 != rgba.Length) - throw new ArgumentException($"A {width}x{height} picture is {(long)width * height * 4} bytes of RGBA, not {rgba.Length}.", nameof(rgba)); + foreach (var pixels in frames.Select(frame => frame.Rgba).Prepend(rgba)) + { + if ((long)width * height * 4 != pixels.Length) + throw new ArgumentException($"A {width}x{height} picture is {(long)width * height * 4} bytes of RGBA, not {pixels.Length}.", nameof(rgba)); + } Key = key; Width = width; Height = height; Rgba = rgba; + Frames = frames; + } + + private static ReadOnlyMemory FirstOf(IReadOnlyList frames) + { + ArgumentNullException.ThrowIfNull(frames); + if (frames.Count == 0) throw new ArgumentException("A picture has at least one frame.", nameof(frames)); + return frames[0].Rgba; } /// What identifies these pixels. @@ -41,10 +72,18 @@ public TerminalPicture(string key, int width, int height, ReadOnlyMemory r /// Its height in pixels. public int Height { get; } - /// The pixels: RGBA, row by row from the top left. + /// The pixels: RGBA, row by row from the top left. For a moving picture, its first frame. public ReadOnlyMemory Rgba { get; } + + /// Every frame of a moving picture, the first included; empty for a still one. + public IReadOnlyList Frames { get; } } +/// One frame of a moving picture: the whole picture as it is shown, and for how long. +/// The pixels: RGBA, row by row from the top left. +/// How long the frame is shown before the next. +public readonly record struct TerminalPictureFrame(ReadOnlyMemory Rgba, TimeSpan Duration); + /// /// Where a terminal render finds a picture's pixels, and what it knows about the one connection it is /// rendering for. A host builds one per connection and hands it to . diff --git a/MarkupString.Tests/Ansi/TerminalFeatureTests.cs b/MarkupString.Tests/Ansi/TerminalFeatureTests.cs index 5a3a9bb..682dab9 100644 --- a/MarkupString.Tests/Ansi/TerminalFeatureTests.cs +++ b/MarkupString.Tests/Ansi/TerminalFeatureTests.cs @@ -194,6 +194,53 @@ public async Task KittyWithoutThePixelsSendsTheArt() await Assert.That(output).DoesNotContain(Placeholder); } + [Test] + public async Task KittySendsEachFrameOfAMovingPictureThenStartsIt() + { + var red = RedBlue(2, 2).Rgba; + var blue = red.ToArray().Reverse().ToArray(); + var picture = new TerminalPicture("moving", 2, 2, + [new TerminalPictureFrame(red, TimeSpan.FromMilliseconds(100)), new TerminalPictureFrame(blue, TimeSpan.FromMilliseconds(250))]); + var source = new Source(picture); + var options = new AnsiOutputOptions(Features: TerminalFeatures.KittyGraphics) { Pictures = source }; + var laid = Laid(new Figure(Cat, MarkupText.Empty), 10); + + var first = RenderString(laid, options); + var second = RenderString(laid, options); + var id = source.Sent.Single(); + + await Assert.That(Regex.Matches(first, "a=T,U=1").Count).IsEqualTo(1); + await Assert.That(first).Contains($"a=f,i={id},f=100,X=1,z=250,q=2,m=0;"); + await Assert.That(first).Contains($"a=a,i={id},r=1,z=100,q=2"); + await Assert.That(first.IndexOf($"a=a,i={id},s=3,v=1,q=2", StringComparison.Ordinal)) + .IsGreaterThan(first.IndexOf("a=f,", StringComparison.Ordinal)); + await Assert.That(second).DoesNotContain("a=f,").And.DoesNotContain("a=a,"); + } + + [Test] + public async Task AMovingPictureIsItsFirstFrameWhereItCannotMove() + { + var still = RedBlue(2, 2); + var moving = new TerminalPicture("moving", 2, 2, + [new TerminalPictureFrame(still.Rgba, TimeSpan.FromMilliseconds(100)), new TerminalPictureFrame(new byte[16], TimeSpan.FromMilliseconds(100))]); + var laid = Laid(new Figure(Cat, MarkupText.Empty), 10); + string Render(TerminalPicture picture) => + RenderString(laid, new AnsiOutputOptions(Features: TerminalFeatures.BlockArt) { Pictures = new Source(picture) }); + + await Assert.That(Render(moving)).IsEqualTo(Render(still)); + await Assert.That(moving.Rgba.ToArray()).IsEquivalentTo(still.Rgba.ToArray()); + } + + [Test] + public async Task AMovingPictureNeedsFramesOfItsOwnSize() + { + await Assert.That(() => new TerminalPicture("bad", 2, 2, + [new TerminalPictureFrame(new byte[16], TimeSpan.Zero), new TerminalPictureFrame(new byte[4], TimeSpan.Zero)])) + .Throws(); + await Assert.That(() => new TerminalPicture("none", 2, 2, Array.Empty())).Throws(); + await Assert.That(new TerminalPicture("one", 2, 2, [new TerminalPictureFrame(new byte[16], TimeSpan.Zero)]).Frames).IsEmpty(); + } + [Test] public async Task KittyTransmissionIsChunkedAtFourKilobytes() {