From d951a39dd9313c7795e485628a47c87131f82ab6 Mon Sep 17 00:00:00 2001 From: "Alex \"mcmonkey\" Goodwin" Date: Wed, 21 Oct 2026 16:13:47 -0700 Subject: [PATCH] Swap ImageSharp to ImageMagick because imagesharp's last few updates have been record breakingly stupid so we're gonna just not f wit dat --- README.md | 2 +- docs/Troubleshooting.md | 2 +- src/Accounts/Session.cs | 2 +- src/Accounts/UserImageHistoryHelper.cs | 36 +- .../GridGenerator/GridGeneratorExtension.cs | 181 +++++----- .../ImageBatchTool/ImageBatchToolExtension.cs | 13 +- src/Core/Program.cs | 1 - src/Media/ImageFile.cs | 318 ++++++++++-------- src/SwarmUI.deps.props | 3 +- src/Text2Image/T2IMultiStepObjectBuilder.cs | 34 +- src/Utils/Image.cs | 15 +- src/Utils/MetadataHelper.cs | 106 ++++-- src/Utils/OutputMetadataTracker.cs | 2 +- src/WebAPI/T2IAPI.cs | 24 +- 14 files changed, 415 insertions(+), 324 deletions(-) diff --git a/README.md b/README.md index 7006991213..29fa1bca8c 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ This project: - can automatically install MIT/BSD/Apache2/PythonSoftwareFoundation pip packages: [spandrel](https://pypi.org/project/spandrel/), [dill](https://pypi.org/project/dill/), [imageio-ffmpeg](https://pypi.org/project/imageio-ffmpeg/), [opencv-python-headless](https://pypi.org/project/opencv-python-headless/), [matplotlib](https://pypi.org/project/matplotlib/), [rembg](https://pypi.org/project/rembg/), [kornia](https://pypi.org/project/kornia/), [Cython](https://pypi.org/project/Cython/) - can automatically install [ultralytics](https://github.com/ultralytics/ultralytics) (AGPL) for `YOLOv8` face detection (ie `SwarmYoloDetection` node or `` syntax usage may become subject to AGPL terms), - can automatically install [insightface](https://github.com/deepinsight/insightface) (MIT) for `IP Adapter - Face` support -- uses [JSON.NET](https://github.com/JamesNK/Newtonsoft.Json) (MIT), [FreneticUtilities](https://github.com/FreneticLLC/FreneticUtilities) (MIT), [LiteDB](https://github.com/mbdavid/LiteDB) (MIT), [ImageSharp](https://github.com/SixLabors/ImageSharp/) (Apache2 under open-source Split License), [ATL.NET](https://github.com/Zeugma440/atldotnet) (MIT) +- uses [JSON.NET](https://github.com/JamesNK/Newtonsoft.Json) (MIT), [FreneticUtilities](https://github.com/FreneticLLC/FreneticUtilities) (MIT), [LiteDB](https://github.com/mbdavid/LiteDB) (MIT), [Magick.NET](https://github.com/dlemstra/Magick.NET) (Apache2), [ATL.NET](https://github.com/Zeugma440/atldotnet) (MIT) - embeds copies of web assets from [BootStrap](https://getbootstrap.com/) (MIT), [Select2](https://select2.org/) (MIT), [JQuery](https://jquery.com/) (MIT), [exif-reader](https://github.com/mattiasw/ExifReader) (MPL-2.0), [music-metadata](https://github.com/Borewit/music-metadata) (MIT). - contains some icons from [Cristian Munoz](https://www.figma.com/community/file/1311159026125960259/7000-free-ui-icons) (CC-BY-4.0), the font [inter by rsms](https://github.com/rsms/inter) (OFL), [Unifont by GNU](https://unifoundry.com/unifont/) (OFL), [Material Symbols Outlined by Google](https://fonts.google.com/icons) (Apache2). - can be used to install some custom node packs, which have individual license notices for any non-pure-FOSS licenses before install. diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index ced957ef04..55627e57ac 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -18,7 +18,7 @@ That is the build dependency service, connections to it are required for the fir If you see errors like: - `The type or namespace name 'Hardware' could not be found (are you missing a using directive or an assembly reference?) [src/SwarmUI.csproj]`, noting the mention of `Hardware` specifically (this is a NuGet package) -- `The type or namespace name 'Image<>' does not exist in the namespace 'SixLabors.ImageSharp'` +- `The type or namespace name 'MagickImage' does not exist in the namespace 'ImageMagick'` that might mean your NuGet cache is corrupt. This is rare, but can happen. In that case, you must reset your NuGet cache. diff --git a/src/Accounts/Session.cs b/src/Accounts/Session.cs index a231a93f45..2c84e828ed 100644 --- a/src/Accounts/Session.cs +++ b/src/Accounts/Session.cs @@ -288,7 +288,7 @@ public void Dispose() File.WriteAllBytes(fullPath, actualFile.RawData); if ((User.Settings.FileFormat.SaveTextFileMetadata || extension == "webp" || !OutputMetadataTracker.ExtensionsWithMetadata.Contains(extension)) && !string.IsNullOrWhiteSpace(metadata)) { - if (extension == "webp" && actualFile is ImageFile imageFile && imageFile.ToIS.Frames.Count == 1) + if (extension == "webp" && actualFile is ImageFile imageFile && imageFile.ToMagickFrames.Count == 1) { // no .json write for still-image webps } diff --git a/src/Accounts/UserImageHistoryHelper.cs b/src/Accounts/UserImageHistoryHelper.cs index 619ce822ab..83e6425341 100644 --- a/src/Accounts/UserImageHistoryHelper.cs +++ b/src/Accounts/UserImageHistoryHelper.cs @@ -3,10 +3,8 @@ using ATL; using FreneticUtilities.FreneticExtensions; using FreneticUtilities.FreneticToolkit; -using SixLabors.Fonts; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.Drawing.Processing; -using SixLabors.ImageSharp.Processing; +using ImageMagick; +using ImageMagick.Drawing; using SwarmUI.Core; using SwarmUI.Media; using SwarmUI.Utils; @@ -91,29 +89,23 @@ public static async Task DoFfmpegPreviewGeneration(string file) double durationMs = new Track(file).DurationMs; long totalSeconds = double.IsFinite(durationMs) ? Math.Max(0, (long)(durationMs / 1000)) : 0; string duration = totalSeconds >= 3600 ? $"{totalSeconds / 3600:00}:{totalSeconds / 60 % 60:00}:{totalSeconds % 60:00}" : $"{totalSeconds / 60:00}:{totalSeconds % 60:00}"; - FontCollection fonts = new(); - Font font = fonts.Add("src/wwwroot/fonts/Inter.woff2").CreateFont(32, FontStyle.Bold); - using SixLabors.ImageSharp.Image preview = SixLabors.ImageSharp.Image.Load(tempPreviewPath); - RichTextOptions textOptions = new(font) { Origin = new(128, 128), HorizontalAlignment = HorizontalAlignment.Center, VerticalAlignment = VerticalAlignment.Center }; - Brush outlineBrush = Brushes.Solid(Color.FromRgb(39, 39, 42)); - Brush textBrush = Brushes.Solid(Color.FromRgb(228, 228, 228)); - preview.Mutate(m => + using MagickImage preview = new(tempPreviewPath); + IDrawables text = new Drawables().Font("src/wwwroot/fonts/Inter.woff2", FontStyleType.Normal, FontWeight.Bold, FontStretch.Normal).FontPointSize(32).TextAlignment(TextAlignment.Center); + ITypeMetric metrics = text.FontTypeMetrics($"Audio {duration}"); + double baseline = 128 + (metrics.Ascent + metrics.Descent) / 2; + text.FillColor(new MagickColor("#27272a")); + for (int x = -2; x <= 2; x += 2) { - for (int x = -2; x <= 2; x += 2) + for (int y = -2; y <= 2; y += 2) { - for (int y = -2; y <= 2; y += 2) + if (x != 0 || y != 0) { - if (x != 0 || y != 0) - { - textOptions.Origin = new(128 + x, 128 + y); - m.DrawText(textOptions, $"Audio {duration}", outlineBrush); - } + text.Text(128 + x, baseline + y, $"Audio {duration}"); } } - textOptions.Origin = new(128, 128); - m.DrawText(textOptions, $"Audio {duration}", textBrush); - }); - preview.SaveAsJpeg(previewPath); + } + text.FillColor(new MagickColor("#e4e4e4")).Text(128, baseline, $"Audio {duration}").Draw(preview); + preview.Write(previewPath, MagickFormat.Jpeg); } finally { diff --git a/src/BuiltinExtensions/GridGenerator/GridGeneratorExtension.cs b/src/BuiltinExtensions/GridGenerator/GridGeneratorExtension.cs index 968ab4e77c..9e417b8bba 100644 --- a/src/BuiltinExtensions/GridGenerator/GridGeneratorExtension.cs +++ b/src/BuiltinExtensions/GridGenerator/GridGeneratorExtension.cs @@ -12,15 +12,10 @@ using SwarmUI.Utils; using SwarmUI.WebAPI; using System.Net.WebSockets; -using SixLabors.Fonts; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.Processing; -using SixLabors.ImageSharp.Drawing.Processing; +using ImageMagick; +using ImageMagick.Drawing; using static SwarmUI.Builtin_GridGeneratorExtension.GridGenCore; -using Image = SwarmUI.Utils.Image; -using ISImage = SixLabors.ImageSharp.Image; -using ISImageRGBA = SixLabors.ImageSharp.Image; namespace SwarmUI.Builtin_GridGeneratorExtension; @@ -444,17 +439,9 @@ public async Task GridGenDoesExist(Session session, string folderName) return new JObject() { ["exists"] = exists }; } - public static FontCollection MainFontCollection; - public static ConcurrentDictionary Fonts = new(); - - public static Font GetFont(float sizeMult) + public static IDrawables GetFont(float sizeMult) { - if (MainFontCollection is null) - { - MainFontCollection = new(); - MainFontCollection.Add("src/wwwroot/fonts/unifont-12.0.01.woff2"); - } - return Fonts.GetOrCreate(sizeMult, () => MainFontCollection.Families.First().CreateFont(16 * sizeMult, FontStyle.Bold)); + return new Drawables().Font("src/wwwroot/fonts/unifont-12.0.01.woff2", FontStyleType.Normal, FontWeight.Bold, FontStretch.Normal).FontPointSize(16 * sizeMult); } public async Task GridGenRun(WebSocket socket, Session session, JObject raw, string outputFolderName, bool doOverwrite, bool fastSkip, bool generatePage, bool publishGenMetadata, bool dryRun, bool weightOrder, string outputType, bool continueOnError, bool showOutputs) @@ -516,103 +503,129 @@ async Task sendStatus() List<(string, string)> xAxis = [.. grid.Axes[0].Values.Where(v => !v.Skip).Select(proc)]; List<(string, string)> yAxis = grid.Axes.Count > 1 ? [.. grid.Axes[1].Values.Where(v => !v.Skip).Select(proc)] : [(null, null)]; List<(string, string)> y2Axis = grid.Axes.Count > 2 ? [.. grid.Axes[2].Values.Where(v => !v.Skip).Select(proc)] : [(null, null)]; - int maxWidth = images.Max(x => x.ToIS.Width); - int maxHeight = images.Max(x => x.ToIS.Height); + int maxWidth = (int)images.Max(x => x.ToMagick.Width); + int maxHeight = (int)images.Max(x => x.ToMagick.Height); float extraSizeMult = 1; if (maxWidth > 800 || maxHeight > 800) { extraSizeMult = 2; } - Font font = GetFont(extraSizeMult); - TextOptions options = new(font); - FontRectangle rect = TextMeasurer.MeasureSize("ABCdefg Word Prefix", options); - int textWidth = (int)Math.Ceiling(rect.Width); - int rawTextHeight = (int)Math.Ceiling(rect.Height * 1.1); + IDrawables font = GetFont(extraSizeMult); + ITypeMetric rect = font.FontTypeMetrics("ABCdefg Word Prefix"); + int textWidth = (int)Math.Ceiling(rect.TextWidth); + int rawTextHeight = (int)Math.Ceiling(rect.TextHeight * 1.1); int textHeight = rawTextHeight * 2; int totalWidth = maxWidth * xAxis.Count + textWidth; int totalHeight = maxHeight * (yAxis.Count * y2Axis.Count) + textHeight * y2Axis.Count; Logs.Info($"Will generate grid image of size {totalWidth}x{totalHeight}"); - ISImageRGBA gridImg = new(totalWidth, totalHeight); - gridImg.Mutate(m => + using MagickImage gridImg = new(MagickColors.White, (uint)totalWidth, (uint)totalHeight) { Depth = 8 }; + void DrawTextAutoScale(string text, float x, float y, float width, float height) { - Brush brush = new SolidBrush(Color.Black); - void DrawTextAutoScale(string text, float x, float y, float width, float height) + if (text.Length == 0) { - if (text.Length > 500) - { - text = $"{text[..500]}..."; - } - RichTextOptions rto = new(font) { WrappingLength = width, Origin = new(x, y) }; - float lines = height / rawTextHeight; - FontRectangle measured = TextMeasurer.MeasureSize(text, options); - Logs.Verbose($"Measured text '{text}' as {measured.Width}x{measured.Height} in {width}x{height} with {lines} lines"); - if (measured.Width < width * lines * 0.5) - { - rto.Font = GetFont(2 * extraSizeMult); - } - else if (measured.Width > width * lines * 2) - { - rto.Font = GetFont(0.5f * extraSizeMult); - } - else if (measured.Width > width * lines) - { - rto.Font = GetFont(0.75f * extraSizeMult); - } - m.DrawText(rto, text, brush); + return; } - m.Fill(Color.White); - int xIndex = 0; - float yIndex = 0; - foreach ((string x, _) in xAxis) + if (text.Length > 500) { - DrawTextAutoScale(x, xIndex * maxWidth + textWidth, 0, maxWidth, textHeight); - xIndex++; + text = $"{text[..500]}..."; } - foreach ((string y2, _) in y2Axis) + float lines = height / rawTextHeight; + ITypeMetric measured = font.FontTypeMetrics(text); + Logs.Verbose($"Measured text '{text}' as {measured.TextWidth}x{measured.TextHeight} in {width}x{height} with {lines} lines"); + float sizeMult = extraSizeMult; + if (measured.TextWidth < width * lines * 0.5) { - if (y2 is not null) - { - DrawTextAutoScale(y2, 0, yIndex * maxHeight + textHeight, textWidth, maxHeight); - yIndex += textHeight / (float)maxHeight; - } - foreach ((string y, _) in yAxis) + sizeMult *= 2; + } + else if (measured.TextWidth > width * lines * 2) + { + sizeMult *= 0.5f; + } + else if (measured.TextWidth > width * lines) + { + sizeMult *= 0.75f; + } + IDrawables labelFont = GetFont(sizeMult); + IDrawables label = GetFont(sizeMult).FillColor(MagickColors.Black); + ITypeMetric lineMetrics = labelFont.FontTypeMetrics("Ag"); + double baseline = y + lineMetrics.Ascent; + foreach (string paragraph in text.Replace("\r", "").Split('\n')) + { + string line = ""; + foreach (Rune character in paragraph.EnumerateRunes()) { - if (y is not null) + string candidate = line + character; + if (line.Length > 0 && labelFont.FontTypeMetrics(candidate).TextWidth > width) + { + int space = line.LastIndexOf(' '); + label.Text(x, baseline, space > 0 ? line[..space] : line); + baseline += lineMetrics.TextHeight; + line = (space > 0 ? line[(space + 1)..] : "") + character; + } + else { - DrawTextAutoScale(y, 0, yIndex * maxHeight + textHeight * 2, textWidth, maxHeight); + line = candidate; } - yIndex++; } + if (line.Length > 0) + { + label.Text(x, baseline, line); + } + baseline += lineMetrics.TextHeight; } - yIndex = 0; - foreach ((_, string y2) in y2Axis) + label.Draw(gridImg); + } + int xIndex = 0; + float yIndex = 0; + foreach ((string x, _) in xAxis) + { + DrawTextAutoScale(x, xIndex * maxWidth + textWidth, 0, maxWidth, textHeight); + xIndex++; + } + foreach ((string y2, _) in y2Axis) + { + if (y2 is not null) { - if (y2 is not null) + DrawTextAutoScale(y2, 0, yIndex * maxHeight + textHeight, textWidth, maxHeight); + yIndex += textHeight / (float)maxHeight; + } + foreach ((string y, _) in yAxis) + { + if (y is not null) { - yIndex += textHeight / (float)maxHeight; + DrawTextAutoScale(y, 0, yIndex * maxHeight + textHeight * 2, textWidth, maxHeight); } - foreach ((_, string y) in yAxis) + yIndex++; + } + } + yIndex = 0; + foreach ((_, string y2) in y2Axis) + { + if (y2 is not null) + { + yIndex += textHeight / (float)maxHeight; + } + foreach ((_, string y) in yAxis) + { + xIndex = 0; + foreach ((_, string x) in xAxis) { - xIndex = 0; - foreach ((_, string x) in xAxis) + string imgPath = x; + if (y is not null) { - string imgPath = x; - if (y is not null) + imgPath = $"{imgPath}/{y}"; + if (y2 is not null) { - imgPath = $"{imgPath}/{y}"; - if (y2 is not null) - { - imgPath = $"{imgPath}/{y2}"; - } + imgPath = $"{imgPath}/{y2}"; } - ISImage img = (data.GeneratedOutputs[imgPath] as ImageFile).ToIS; - m.DrawImage(img, new Point(xIndex * maxWidth + textWidth, (int)(yIndex * maxHeight + textHeight)), 1); - xIndex++; } - yIndex++; + IMagickImage img = (data.GeneratedOutputs[imgPath] as ImageFile).ToMagick; + gridImg.Composite(img, xIndex * maxWidth + textWidth, (int)(yIndex * maxHeight + textHeight), CompositeOperator.Over); + xIndex++; } + yIndex++; } - }); + } Logs.Info("Generated, saving..."); Image outImg = new(gridImg); int batchId = (xAxis.Count * yAxis.Count * y2Axis.Count) + 1; diff --git a/src/BuiltinExtensions/ImageBatchTool/ImageBatchToolExtension.cs b/src/BuiltinExtensions/ImageBatchTool/ImageBatchToolExtension.cs index 6293e5ee4b..e076da0d7b 100644 --- a/src/BuiltinExtensions/ImageBatchTool/ImageBatchToolExtension.cs +++ b/src/BuiltinExtensions/ImageBatchTool/ImageBatchToolExtension.cs @@ -10,7 +10,6 @@ using System; using System.IO; using System.Net.WebSockets; -using ISImage = SixLabors.ImageSharp.Image; namespace SwarmUI.Builtin_ImageBatchToolExtension; @@ -128,7 +127,7 @@ void removeDoneTasks() break; } Image image = new(File.ReadAllBytes(file), MediaType.GetByExtension(file.AfterLast('.'))); - ISImage imgData = image.ToIS; + (int imageWidth, int imageHeight) = image.GetResolution(); T2IParamInput param = baseParams.Clone(); void setRes(int width, int height) { @@ -147,17 +146,17 @@ void setRes(int width, int height) case "From Parameter": break; case "From Image": - setRes(imgData.Width, imgData.Height); + setRes(imageWidth, imageHeight); break; case "Scale To Model": - (int width, int height) = Utilities.ResToModelFit(imgData.Width, imgData.Height, param.Get(T2IParamTypes.Model)); + (int width, int height) = Utilities.ResToModelFit(imageWidth, imageHeight, param.Get(T2IParamTypes.Model)); setRes(width, height); break; case "Scale To Model Or Above": - (width, height) = Utilities.ResToModelFit(imgData.Width, imgData.Height, param.Get(T2IParamTypes.Model)); - if (width < imgData.Width || height < imgData.Height) + (width, height) = Utilities.ResToModelFit(imageWidth, imageHeight, param.Get(T2IParamTypes.Model)); + if (width < imageWidth || height < imageHeight) { - setRes(imgData.Width, imgData.Height); + setRes(imageWidth, imageHeight); } else { diff --git a/src/Core/Program.cs b/src/Core/Program.cs index c5f49e8ad3..a94ca272b4 100644 --- a/src/Core/Program.cs +++ b/src/Core/Program.cs @@ -6,7 +6,6 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; -using SixLabors.ImageSharp; using SwarmUI.Accounts; using SwarmUI.Backends; using SwarmUI.Text2Image; diff --git a/src/Media/ImageFile.cs b/src/Media/ImageFile.cs index 878f8ce256..26a19a536c 100644 --- a/src/Media/ImageFile.cs +++ b/src/Media/ImageFile.cs @@ -1,28 +1,16 @@ using SwarmUI.Utils; -using SixLabors.ImageSharp; +using ImageMagick; +using ImageMagick.Formats; +using System.Buffers.Binary; using System.IO; -using SixLabors.ImageSharp.Metadata.Profiles.Exif; using Newtonsoft.Json.Linq; -using SixLabors.ImageSharp.Processing; using FreneticUtilities.FreneticExtensions; -using SixLabors.ImageSharp.Formats.Png; -using SixLabors.ImageSharp.Formats.Webp; -using SixLabors.ImageSharp.Formats.Jpeg; -using SixLabors.ImageSharp.PixelFormats; - -using Image = SwarmUI.Utils.Image; -using ISImage = SixLabors.ImageSharp.Image; -using ISImage32 = SixLabors.ImageSharp.Image; -using ISImageFrame32 = SixLabors.ImageSharp.ImageFrame; namespace SwarmUI.Media; /// A still image file. Also encompasses animation file types (eg gif, animated webp, etc.) public class ImageFile : MediaFile { - /// Used for . - public static PngEncoder FastPngEncoder = new() { CompressionLevel = PngCompressionLevel.Level1 }; - /// Creates an image object from a web image data URL string. public static ImageFile FromDataString(string data) { @@ -38,46 +26,117 @@ public static ImageFile FromBase64(string b64, MediaType type) return new Image(raw, type); } - /// Helper to convert an ImageSharp image to png bytes. - public static byte[] ISImgToPngBytes(ISImage img) + /// Helper to convert a Magick image to png bytes. + public static byte[] MagickImageToPngBytes(IMagickImage img) { - using MemoryStream stream = new(); - img.SaveAsPng(stream, FastPngEncoder); - return stream.ToArray(); + using IMagickImage copy = img.Clone(); + copy.Settings.SetDefine(MagickFormat.Png, "compression-level", 1); + return copy.ToByteArray(MagickFormat.Png); } - /// Helper to convert an ImageSharp image to jpg bytes. - public static byte[] ISImgToJpgBytes(ISImage img) + /// Helper to convert a Magick image to jpg bytes. + public static byte[] MagickImageToJpgBytes(IMagickImage img) { - using MemoryStream stream = new(); - img.SaveAsJpeg(stream); - return stream.ToArray(); + using IMagickImage copy = img.Clone(); + copy.Quality = 75; + return copy.ToByteArray(MagickFormat.Jpeg); } - /// Internal cache of to avoid reprocessing. - public ISImage _CacheISImg; + /// Encodes all frames, using lossless WebP for animations and fast PNG for still images. + public static byte[] MagickFramesToImageBytes(IMagickImageCollection frames) + { + using IMagickImageCollection copy = frames.Clone(); + foreach (IMagickImage frame in copy) + { + if (copy.Count > 1) + { + frame.Quality = 100; + frame.Settings.SetDefines(new WebPWriteDefines() { Lossless = true, Exact = true, AlphaQuality = 100 }); + } + else + { + frame.Settings.SetDefine(MagickFormat.Png, "compression-level", 1); + } + } + return copy.ToByteArray(copy.Count > 1 ? MagickFormat.WebP : MagickFormat.Png); + } + + /// Internal cache of to avoid reprocessing. + public MagickImageCollection _CacheMagick; - /// Gets an ImageSharp for this image. - public ISImage ToIS + /// Gets the decoded frames, owned by this file. Do not dispose or modify these shared images. + public MagickImageCollection ToMagickFrames { get { - if (_CacheISImg is null) + if (_CacheMagick is null) { lock (this) { - _CacheISImg ??= ISImage.Load(RawData); + if (_CacheMagick is null) + { + MagickImageCollection images = new(RawData); + try + { + if (images.Count > 1) + { + images.Coalesce(); + if (images[0].Format == MagickFormat.WebP) + { + RestoreWebpFrameDelays(images); + } + } + _CacheMagick = images; + } + catch + { + images.Dispose(); + throw; + } + } } } - return _CacheISImg; + return _CacheMagick; + } + } + + /// Gets the first decoded frame, owned by this file. Clone it before editing. + public IMagickImage ToMagick => ToMagickFrames[0]; + + /// Workaround/patch for webp issue: Restores millisecond WebP delays that ImageMagick 7.1.2-29 rounds to centiseconds when decoding. + public void RestoreWebpFrameDelays(MagickImageCollection images) + { + int frameIndex = 0; + for (int offset = 12; offset <= RawData.Length - 8;) + { + uint length = BinaryPrimitives.ReadUInt32LittleEndian(RawData.AsSpan(offset + 4, 4)); + long nextOffset = offset + 8L + length + (length & 1); + if (nextOffset > RawData.Length) + { + throw new InvalidDataException("Truncated WebP chunk."); + } + if (RawData.AsSpan(offset, 4).SequenceEqual("ANMF"u8)) + { + if (length < 16 || frameIndex >= images.Count) + { + throw new InvalidDataException("Invalid WebP animation frame."); + } + IMagickImage frame = images[frameIndex++]; + frame.AnimationTicksPerSecond = 1000; + frame.AnimationDelay = (uint)(RawData[offset + 20] | RawData[offset + 21] << 8 | RawData[offset + 22] << 16); + } + offset = (int)nextOffset; + } + if (frameIndex != images.Count) + { + throw new InvalidDataException("WebP animation frame count does not match its decoded frames."); } } /// Returns the (width, height) of the image. public (int, int) GetResolution() { - Size size = ToIS.Size; - return (size.Width, size.Height); + return ((int)ToMagick.Width, (int)ToMagick.Height); } /// Returns a metadata-format of the image. @@ -89,17 +148,17 @@ public ImageFile ToMetadataJpg(string metadataText = null) return null; } metadataText ??= GetMetadata(); - ISImage img = ToIS; + using IMagickImage img = ToMagick.Clone(); float factor = 256f / Math.Min(img.Width, img.Height); - img = img.Clone(i => i.Resize((int)(img.Width * factor), (int)(img.Height * factor))); + img.Resize(new MagickGeometry((uint)(img.Width * factor), (uint)(img.Height * factor)) { IgnoreAspectRatio = true }); if (!string.IsNullOrWhiteSpace(metadataText)) { - img.Metadata.XmpProfile = null; + img.RemoveProfile("xmp"); ExifProfile prof = new(); - prof.SetValue(ExifTag.UserComment, metadataText); - img.Metadata.ExifProfile = prof; + prof.SetValue(ExifTag.UserComment, MetadataHelper.EncodeExifUserComment(metadataText)); + img.SetProfile(prof); } - return new Image(ISImgToJpgBytes(img), MediaType.ImageJpg); + return new Image(MagickImageToJpgBytes(img), MediaType.ImageJpg); } /// Returns a simplified webp animation for preview purposes. Returns null if the input was not animated. @@ -109,41 +168,41 @@ public ImageFile ToWebpPreviewAnim() { return null; } - ISImage isimg = ToIS; - if (isimg.Frames.Count <= 1) + MagickImageCollection images = ToMagickFrames; + if (images.Count <= 1) { return null; } int timeTotal = 0; int timeAccum = 166; - int maxFrame = isimg.Frames.Count; - using ISImage32 oldImage = isimg.CloneAs(); - int targetWidth = isimg.Width, targetHeight = isimg.Height; + int maxFrame = images.Count; + uint targetWidth = images[0].Width, targetHeight = images[0].Height; if (targetWidth > 256 || targetHeight > 256) { float factor = 256f / Math.Min(targetWidth, targetHeight); - targetWidth = (int)(targetWidth * factor); - targetHeight = (int)(targetHeight * factor); + targetWidth = (uint)(targetWidth * factor); + targetHeight = (uint)(targetHeight * factor); } - using ISImage32 newImage = new(targetWidth, targetHeight); + using MagickImageCollection newImages = []; for (int i = 0; i < maxFrame; i++) { - ISImageFrame32 frame = oldImage.Frames[i]; - int timeMs = Type == MediaType.ImageWebp ? (int)frame.Metadata.GetWebpMetadata().FrameDelay : frame.Metadata.GetGifMetadata().FrameDelay * 10; + IMagickImage frame = images[i]; + int timeMs = (int)(frame.AnimationDelay * 1000L / frame.AnimationTicksPerSecond); timeTotal += timeMs; timeAccum += timeMs; if (timeAccum > 99) { - if (frame.Width != targetWidth || frame.Height != targetHeight) + IMagickImage previewFrame = frame.Clone(); + newImages.Add(previewFrame); + if (previewFrame.Width != targetWidth || previewFrame.Height != targetHeight) { - ISImage32 frameImage = new(frame.Width, frame.Height); - frameImage.Frames.AddFrame(frame); - frameImage.Frames.RemoveFrame(0); - frameImage.Mutate(i => i.Resize(targetWidth, targetHeight)); - frame = frameImage.Frames[0]; + previewFrame.Resize(new MagickGeometry(targetWidth, targetHeight) { IgnoreAspectRatio = true }); } - frame.Metadata.GetWebpMetadata().FrameDelay = (uint)timeAccum; - newImage.Frames.AddFrame(frame); + previewFrame.ResetPage(); + previewFrame.AnimationTicksPerSecond = 1000; + previewFrame.AnimationDelay = (uint)timeAccum; + previewFrame.AnimationIterations = 0; + previewFrame.Quality = 50; timeAccum = 0; } if (timeTotal > 5000) @@ -151,12 +210,8 @@ public ImageFile ToWebpPreviewAnim() break; } } - newImage.Frames.RemoveFrame(0); - newImage.Metadata.GetWebpMetadata().RepeatCount = 0; - using MemoryStream saveStream = new(); - newImage.SaveAsWebp(saveStream, new WebpEncoder() { Quality = 50 }); - saveStream.Seek(0, SeekOrigin.Begin); - return new Image(saveStream.ToArray(), MediaType.ImageWebp); + newImages[0].Settings.SetDefine(MagickFormat.WebP, "lossless", false); + return new Image(newImages.ToByteArray(MagickFormat.WebP), MediaType.ImageWebp); } /// Returns a metadata-format of the image. @@ -172,13 +227,15 @@ public ImageFile Resize(int width, int height) { return this; } - ISImage img = ToIS; - if (ToIS.Width == width && ToIS.Height == height) + IMagickImage img = ToMagick; + if (img.Width == width && img.Height == height) { return this; } - img = img.Clone(i => i.Resize(width, height)); - return new Image(ISImgToPngBytes(img), Type); + using IMagickImage copy = img.Clone(); + copy.Resize(new MagickGeometry((uint)width, (uint)height) { IgnoreAspectRatio = true }); + copy.ResetPage(); + return new Image(MagickImageToPngBytes(copy), MediaType.ImagePng); } /// Returns a copy of this image that's definitely in '.png' format. @@ -188,7 +245,7 @@ public ImageFile ForceToPng() { return this; } - return new Image(ISImgToPngBytes(ToIS), Type); + return new Image(MagickImageToPngBytes(ToMagick), MediaType.ImagePng); } /// @@ -196,31 +253,17 @@ public override string GetMetadata() { try { - ISImage img = ToIS; - string pngMetadata = img.Metadata?.GetPngMetadata()?.TextData?.FirstOrDefault(t => t.Keyword.ToLowerFast() == "parameters").Value; - if (pngMetadata is not null) - { - return pngMetadata; - } - string output = null; - if (img.Metadata?.ExifProfile?.TryGetValue(ExifTag.Model, out IExifValue data) ?? false) - { - output = data.Value; - } - if (img.Metadata?.ExifProfile?.TryGetValue(ExifTag.UserComment, out IExifValue data2) ?? false) + IMagickImage img = ToMagick; + string parameterName = img.AttributeNames.FirstOrDefault(name => name.ToLowerFast() == "parameters"); + if (parameterName is not null) { - output = data2.Value.Text; + return img.GetAttribute(parameterName); } - if (output is not null && output.Length > 0) - { - // Special fix for ImageSharp not parsing BigEndian unicode, so detect inverted strings (slightly hack) and flip them - byte[] encoded = Encoding.Unicode.GetBytes(output); - if (encoded[0] == 0 && encoded[1] != 0) - { - output = Encoding.BigEndianUnicode.GetString(encoded); - } - } - return output; + IExifProfile prof = img.GetExifProfile(); + // Preserve the original TIFF byte order before parsing, which makes subsequent serialization rewrite the profile. + byte[] profileData = prof?.ToByteArray(); + byte[] comment = prof?.GetValue(ExifTag.UserComment)?.Value; + return comment is null ? prof?.GetValue(ExifTag.Model)?.Value : MetadataHelper.DecodeExifUserComment(comment, profileData); } catch (ArgumentNullException ex) { @@ -274,102 +317,111 @@ public ImageFile ConvertTo(string format, string metadata = null, int dpi = 0, i { return this; } - using MemoryStream ms = new(); - ISImage img = ToIS; + using IMagickImageCollection frames = ToMagickFrames.Clone(); + (MagickFormat outputFormat, MediaType type) = ConvertMagickFrame(frames[0], format, metadata, dpi, quality, stealthMetadata); + for (int i = 1; i < frames.Count; i++) + { + ConvertMagickFrame(frames[i], format, metadata, dpi, quality, stealthMetadata); + } + return new Image(frames.ToByteArray(outputFormat), type); + } + + /// Helper for . + public static (MagickFormat, MediaType) ConvertMagickFrame(IMagickImage img, string format, string metadata, int dpi, int quality, string stealthMetadata) + { bool isLossyWebp = format.StartsWith("WEBP") && format != "WEBP_LOSSLESS"; bool canDoStealth = metadata is not null && (format == "PNG" || format.StartsWith("WEBP")) && !(isLossyWebp && stealthMetadata.ToLowerFast() == "rgb"); if (stealthMetadata.ToLowerFast() != "false" && canDoStealth) { string actualStealthMode = stealthMetadata.ToLowerInvariant(); - ISImage32 rgbaImage = img.CloneAs(); - MetadataHelper.EncodeStealthMetadata(rgbaImage, metadata, actualStealthMode, format); - img = rgbaImage; + MetadataHelper.EncodeStealthMetadata(img, metadata, actualStealthMode, format); + } + foreach (string oldProf in new List(img.ProfileNames)) + { + img.RemoveProfile(oldProf); } - img.Metadata.XmpProfile = null; ExifProfile prof = new(); if (dpi > 0) { prof.SetValue(ExifTag.XResolution, new Rational((uint)dpi, 1)); prof.SetValue(ExifTag.YResolution, new Rational((uint)dpi, 1)); prof.SetValue(ExifTag.ResolutionUnit, (ushort)2); - img.Metadata.HorizontalResolution = dpi; - img.Metadata.VerticalResolution = dpi; + img.Density = new Density(dpi, dpi, DensityUnit.PixelsPerInch); } if (metadata is not null) { if (format == "PNG") { - img.Metadata.GetPngMetadata().TextData.Add(new("parameters", metadata, "", "")); + img.SetAttribute("parameters", metadata); } else { - prof.SetValue(ExifTag.UserComment, metadata); + prof.SetValue(ExifTag.UserComment, MetadataHelper.EncodeExifUserComment(metadata)); } } - img.Metadata.ExifProfile = prof; + img.SetProfile(prof); MediaType type; + MagickFormat outputFormat; switch (format) { case "PNG": - if (stealthMetadata.ToLowerFast() == "alpha") - { - PngEncoder encoder = new() - { - TextCompressionThreshold = int.MaxValue, - BitDepth = img.PixelType.BitsPerPixel > 32 ? PngBitDepth.Bit16 : PngBitDepth.Bit8, - CompressionLevel = PngCompressionLevel.Level1, - ColorType = PngColorType.RgbWithAlpha, - TransparentColorMode = PngTransparentColorMode.Preserve - }; - img.SaveAsPng(ms, encoder); - } - else + img.Settings.SetDefines(new PngWriteDefines() { - PngEncoder encoder = new() - { - TextCompressionThreshold = int.MaxValue, - BitDepth = img.PixelType.BitsPerPixel > 32 ? PngBitDepth.Bit16 : PngBitDepth.Bit8, - CompressionLevel = PngCompressionLevel.Level1 - }; - img.SaveAsPng(ms, encoder); - } + CompressionLevel = 1, + BitDepth = img.Depth > 8 ? 16u : 8u, + ExcludeChunks = PngChunkFlags.zTXt, + ColorType = stealthMetadata.ToLowerFast() == "alpha" ? ColorType.TrueColorAlpha : null + }); type = MediaType.ImagePng; + outputFormat = MagickFormat.Png; break; case "JPG": type = MediaType.ImageJpg; - img.SaveAsJpeg(ms, new JpegEncoder() { Quality = quality }); + outputFormat = MagickFormat.Jpeg; + img.Quality = (uint)quality; break; case "JPG90": // NOTE: Legacy (0.9.6) format variants with built-in quality selector type = MediaType.ImageJpg; - img.SaveAsJpeg(ms, new JpegEncoder() { Quality = 90 }); + outputFormat = MagickFormat.Jpeg; + img.Quality = 90; break; case "JPG75": type = MediaType.ImageJpg; - img.SaveAsJpeg(ms, new JpegEncoder() { Quality = 75 }); + outputFormat = MagickFormat.Jpeg; + img.Quality = 75; break; case "WEBP_LOSSLESS": type = MediaType.ImageWebp; - img.SaveAsWebp(ms, new WebpEncoder() { NearLossless = true, FileFormat = WebpFileFormatType.Lossless, Quality = 100 }); + outputFormat = MagickFormat.WebP; + img.Quality = 100; break; case "WEBP": type = MediaType.ImageWebp; - img.SaveAsWebp(ms, new WebpEncoder() { NearLossless = false, FileFormat = WebpFileFormatType.Lossy, Quality = quality }); + outputFormat = MagickFormat.WebP; + img.Quality = (uint)quality; break; case "WEBP_100": type = MediaType.ImageWebp; - img.SaveAsWebp(ms, new WebpEncoder() { NearLossless = false, FileFormat = WebpFileFormatType.Lossy, Quality = 100 }); + outputFormat = MagickFormat.WebP; + img.Quality = 100; break; case "WEBP_90": type = MediaType.ImageWebp; - img.SaveAsWebp(ms, new WebpEncoder() { NearLossless = false, FileFormat = WebpFileFormatType.Lossy, Quality = 90 }); + outputFormat = MagickFormat.WebP; + img.Quality = 90; break; case "WEBP_75": type = MediaType.ImageWebp; - img.SaveAsWebp(ms, new WebpEncoder() { NearLossless = false, FileFormat = WebpFileFormatType.Lossy, Quality = 75 }); + outputFormat = MagickFormat.WebP; + img.Quality = 75; break; default: throw new SwarmReadableErrorException($"User setting for image format is '{format}', which is invalid"); } - return new Image(ms.ToArray(), type); + if (outputFormat == MagickFormat.WebP) + { + img.Settings.SetDefines(new WebPWriteDefines() { Lossless = !isLossyWebp, Exact = true, AlphaQuality = 100 }); + } + return (outputFormat, type); } } diff --git a/src/SwarmUI.deps.props b/src/SwarmUI.deps.props index 852523c7b2..c9b279d42e 100644 --- a/src/SwarmUI.deps.props +++ b/src/SwarmUI.deps.props @@ -7,8 +7,7 @@ - - + diff --git a/src/Text2Image/T2IMultiStepObjectBuilder.cs b/src/Text2Image/T2IMultiStepObjectBuilder.cs index 4f0f127bf8..4ae5c089e9 100644 --- a/src/Text2Image/T2IMultiStepObjectBuilder.cs +++ b/src/Text2Image/T2IMultiStepObjectBuilder.cs @@ -1,13 +1,9 @@ +using System; using FreneticUtilities.FreneticExtensions; using Newtonsoft.Json.Linq; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.Processing; +using ImageMagick; using SwarmUI.Accounts; using SwarmUI.Utils; -using ISImage = SixLabors.ImageSharp.Image; -using Image = SwarmUI.Utils.Image; -using SixLabors.ImageSharp.PixelFormats; -using System; using SwarmUI.Media; namespace SwarmUI.Text2Image; @@ -60,7 +56,7 @@ async Task createImageDirect(T2IParamInput user_input) return null; } //user_input.Set(T2IParamTypes.EndStepsEarly, 0.6); // TODO: Configurable - ISImage liveImg = img.ToIS.Clone(_ => { }); + using IMagickImage liveImg = img.ToMagick.Clone(); float overBound = 0.1f; foreach (PromptRegion.Part part in objects) { @@ -70,7 +66,7 @@ async Task createImageDirect(T2IParamInput user_input) } user_input.Set(T2IParamTypes.Seed, user_input.Get(T2IParamTypes.Seed) + 1); T2IParamInput objInput = user_input.Clone(); - int mpTarget = liveImg.Width * liveImg.Height; + int mpTarget = (int)(liveImg.Width * liveImg.Height); if (objInput.TryGet(T2IParamTypes.RegionalObjectInpaintingModel, out T2IModel model)) { objInput.Set(T2IParamTypes.Model, model); @@ -86,16 +82,18 @@ async Task createImageDirect(T2IParamInput user_input) int pixelHeight = (int)(part.Height * liveImg.Height); int extraX = Math.Max((int)((part.X - overBound * 0.5) * liveImg.Width), 0); int extraY = Math.Max((int)((part.Y - overBound * 0.5) * liveImg.Height), 0); - int extraWidth = Math.Min((int)((part.Width + overBound) * liveImg.Width), liveImg.Width - extraX); - int extraHeight = Math.Min((int)((part.Height + overBound) * liveImg.Height), liveImg.Height - extraY); + int extraWidth = Math.Min((int)((part.Width + overBound) * liveImg.Width), (int)liveImg.Width - extraX); + int extraHeight = Math.Min((int)((part.Height + overBound) * liveImg.Height), (int)liveImg.Height - extraY); (int fixedWidth, int fixedHeight) = Utilities.ResToModelFit(extraWidth, extraHeight, mpTarget); double scaleWidth = fixedWidth / (double)extraWidth; double scaleHeight = fixedHeight / (double)extraHeight; - using ISImage subImage = liveImg.Clone(i => i.Crop(new Rectangle(extraX, extraY, extraWidth, extraHeight)).Resize(fixedWidth, fixedHeight)); - using ISImage maskImage = ISImage.LoadPixelData(Enumerable.Repeat(Color.Black.ToPixel(), 1).ToArray(), 1, 1); - using ISImage maskInnerImage = ISImage.LoadPixelData(Enumerable.Repeat(Color.White.ToPixel(), 1).ToArray(), 1, 1); - maskInnerImage.Mutate(i => i.Resize((int)(pixelWidth * scaleWidth), (int)(pixelHeight * scaleHeight))); - maskImage.Mutate(i => i.Resize(fixedWidth, fixedHeight).DrawImage(maskInnerImage, new Point((int)((pixelX - extraX) * scaleWidth), (int)((pixelY - extraY) * scaleHeight)), 1)); + using IMagickImage subImage = liveImg.Clone(); + subImage.Crop(new MagickGeometry(extraX, extraY, (uint)extraWidth, (uint)extraHeight)); + subImage.ResetPage(); + subImage.Resize(new MagickGeometry((uint)fixedWidth, (uint)fixedHeight) { IgnoreAspectRatio = true }); + using MagickImage maskImage = new(MagickColors.Black, (uint)fixedWidth, (uint)fixedHeight) { Depth = 8 }; + using MagickImage maskInnerImage = new(MagickColors.White, (uint)(pixelWidth * scaleWidth), (uint)(pixelHeight * scaleHeight)) { Depth = 8 }; + maskImage.Composite(maskInnerImage, (int)((pixelX - extraX) * scaleWidth), (int)((pixelY - extraY) * scaleHeight), CompositeOperator.Over); objInput.Set(T2IParamTypes.MaskImage, new(maskImage)); objInput.Set(T2IParamTypes.Prompt, part.Prompt); objInput.Set(T2IParamTypes.InitImage, new Image(subImage)); @@ -107,9 +105,9 @@ async Task createImageDirect(T2IParamInput user_input) { return null; } - ISImage objISImg = objImg.ToIS; - objISImg = objISImg.Clone(i => i.Resize(extraWidth, extraHeight)); - liveImg.Mutate(i => i.DrawImage(objISImg, new Point(extraX, extraY), 1)); + using IMagickImage objectImage = objImg.ToMagick.Clone(); + objectImage.Resize(new MagickGeometry((uint)extraWidth, (uint)extraHeight) { IgnoreAspectRatio = true }); + liveImg.Composite(objectImage, extraX, extraY, CompositeOperator.Over); if (user_input.Get(T2IParamTypes.OutputIntermediateImages, false)) { output(new JObject() { ["image"] = "data:image/png;base64," + new Image(liveImg).AsBase64, ["batch_index"] = $"{batchId}{obj++}", ["request_id"] = $"{user_input.UserRequestId}", ["metadata"] = null }); diff --git a/src/Utils/Image.cs b/src/Utils/Image.cs index a8a09366e2..5fa8345fa1 100644 --- a/src/Utils/Image.cs +++ b/src/Utils/Image.cs @@ -1,8 +1,7 @@ namespace SwarmUI.Utils; using FreneticUtilities.FreneticExtensions; - -using ISImage = SixLabors.ImageSharp.Image; +using ImageMagick; using SwarmUI.Media; /// Helper to represent an image file cleanly and quickly. @@ -15,19 +14,11 @@ public Image(byte[] data, MediaType type) Type = type; } - /// Construct an image from an ISImage, treating it as a png. - public Image(ISImage img) : this(ImageFile.ISImgToPngBytes(img), MediaType.ImagePng) + /// Construct an image from a MagickImage, treating it as a png. + public Image(IMagickImage img) : this(MagickImageToPngBytes(img), MediaType.ImagePng) { } - /// Helper to convert an ImageSharp image to png bytes. - [Obsolete("Use ImageFile.ISImgToPngBytes instead")] - public static new byte[] ISImgToPngBytes(ISImage img) => ImageFile.ISImgToPngBytes(img); - - /// Helper to convert an ImageSharp image to jpg bytes. - [Obsolete("Use ImageFile.ISImgToJpgBytes instead")] - public static new byte[] ISImgToJpgBytes(ISImage img) => ImageFile.ISImgToJpgBytes(img); - /// Gets the correct mime type for this image, eg 'image/png'. [Obsolete("Use MediaType.MimeType instead")] public string MimeType() => Type.MimeType; diff --git a/src/Utils/MetadataHelper.cs b/src/Utils/MetadataHelper.cs index b3b5299d5d..34e3bd327a 100644 --- a/src/Utils/MetadataHelper.cs +++ b/src/Utils/MetadataHelper.cs @@ -1,8 +1,8 @@ -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.PixelFormats; +using System; using System.IO; using System.IO.Compression; using System.Text; +using ImageMagick; namespace SwarmUI.Utils; @@ -11,14 +11,62 @@ public static class MetadataHelper { // The stealth info implementation of reforge was used as a reference for this file. + /// Encodes an EXIF UserComment as little-endian Unicode. + public static byte[] EncodeExifUserComment(string text) + { + return [.. "UNICODE\0"u8, .. Encoding.Unicode.GetBytes(text)]; + } + + /// Decodes EXIF UserComment text, respecting its byte order mark or the containing TIFF byte order. + public static string DecodeExifUserComment(byte[] comment, byte[] profile) + { + if (comment.Length < 8) + { + return Encoding.UTF8.GetString(comment).TrimEnd('\0'); + } + ReadOnlySpan text = comment.AsSpan(8); + if (comment.AsSpan(0, 8).SequenceEqual("UNICODE\0"u8)) + { + if (text.StartsWith([0xFF, 0xFE])) + { + return Encoding.Unicode.GetString(text[2..]).TrimEnd('\0'); + } + if (text.StartsWith([0xFE, 0xFF])) + { + return Encoding.BigEndianUnicode.GetString(text[2..]).TrimEnd('\0'); + } + int tiffOffset = profile.AsSpan().StartsWith("Exif\0\0"u8) ? 6 : 0; + bool bigEndian = profile.Length >= tiffOffset + 2 && profile[tiffOffset] == 'M' && profile[tiffOffset + 1] == 'M'; + // Some generators write comments in the opposite byte order from the TIFF profile. Preserve support for those files. + if (text.Length >= 2 && text[0] == 0 && text[1] != 0) + { + bigEndian = true; + } + else if (text.Length >= 2 && text[0] != 0 && text[1] == 0) + { + bigEndian = false; + } + return (bigEndian ? Encoding.BigEndianUnicode : Encoding.Unicode).GetString(text).TrimEnd('\0'); + } + if (comment.AsSpan(0, 8).SequenceEqual("JIS\0\0\0\0\0"u8)) + { + return CodePagesEncodingProvider.Instance.GetEncoding(20932).GetString(text).TrimEnd('\0'); + } + bool hasHeader = comment.AsSpan(0, 8).SequenceEqual("ASCII\0\0\0"u8) || comment.AsSpan(0, 8).SequenceEqual(new byte[8]); + return Encoding.UTF8.GetString(hasHeader ? text : comment.AsSpan()).TrimEnd('\0'); + } + /// Encodes the given metadata string into the LSBs of the image's pixels. - /// The image to modify. Must be in Rgba32 format. + /// The image to modify. Converted to 8-bit RGBA to match the stealth metadata format. /// The metadata string to embed. /// The encoding mode: "Alpha" or "RGB". /// The target image format (e.g., "webp", "png"). - public static void EncodeStealthMetadata(Image image, string metadata, string mode, string format) + public static void EncodeStealthMetadata(IMagickImage image, string metadata, string mode, string format) { string actualMode = mode.ToLowerInvariant(); + image.ColorSpace = ColorSpace.sRGB; + image.ColorType = ColorType.TrueColorAlpha; + image.Depth = 8; // stupid hack to stop the alpha layer from getting deleted if (actualMode == "alpha" && format.StartsWith("WEBP")) { @@ -26,31 +74,36 @@ public static void EncodeStealthMetadata(Image image, string metadata, s } string binaryData = PrepareData(metadata, actualMode, true); + using IPixelCollection pixels = image.GetPixels(); + byte[] data = pixels.ToByteArray(PixelMapping.RGBA); if (actualMode == "alpha") { - EmbedAlpha(image, binaryData); + EmbedAlpha(data, (int)image.Width, (int)image.Height, binaryData); } else if (actualMode == "rgb") { - EmbedRgb(image, binaryData); + EmbedRgb(data, (int)image.Width, (int)image.Height, binaryData); } + pixels.SetPixels(Array.ConvertAll(data, value => (ushort)(value * 257))); } /// /// Pre-processes the image to prevent the WebP encoder from discarding the alpha channel. /// It does this by creating a semi-transparent vertical line on the right edge of the image. /// - public static void PrepareImageForWebPAlpha(Image image) + public static void PrepareImageForWebPAlpha(IMagickImage image) { + image.ColorType = ColorType.TrueColorAlpha; + using IPixelCollection pixels = image.GetPixels(); for (int y = 0; y < image.Height; y++) { - Rgba32 pixel = image[image.Width - 1, y]; - pixel.A = 254; - image[image.Width - 1, y] = pixel; + IPixel pixel = pixels.GetPixel((int)image.Width - 1, y); + pixel.SetChannel(3, 254 * 257); } } - private static string PrepareData(string metadata, string mode, bool compressed) + /// Prepares the signature, payload length, and metadata bits for stealth encoding. + public static string PrepareData(string metadata, string mode, bool compressed) { string signature = $"stealth_{(mode == "alpha" ? "png" : "rgb")}{(compressed ? "comp" : "info")}"; byte[] signatureBytes = Encoding.UTF8.GetBytes(signature); @@ -70,7 +123,8 @@ private static string PrepareData(string metadata, string mode, bool compressed) return binarySignature + binaryParamLen + binaryParam; } - private static string BytesToBinaryString(byte[] bytes) + /// Converts bytes to their binary string representation. + public static string BytesToBinaryString(byte[] bytes) { StringBuilder sb = new(bytes.Length * 8); foreach (byte b in bytes) @@ -80,48 +134,48 @@ private static string BytesToBinaryString(byte[] bytes) return sb.ToString(); } - private static void EmbedAlpha(Image image, string binaryData) + /// Embeds metadata bits in the alpha channel of an RGBA byte buffer. + public static void EmbedAlpha(byte[] pixels, int width, int height, string binaryData) { int dataIndex = 0; - for (int x = 0; x < image.Width; x++) + for (int x = 0; x < width; x++) { - for (int y = 0; y < image.Height; y++) + for (int y = 0; y < height; y++) { if (dataIndex >= binaryData.Length) { return; } - Rgba32 pixel = image[x, y]; + int alphaIndex = (y * width + x) * 4 + 3; int bit = binaryData[dataIndex] - '0'; - pixel.A = (byte)((pixel.A & 0xFE) | bit); - image[x, y] = pixel; + pixels[alphaIndex] = (byte)((pixels[alphaIndex] & 0xFE) | bit); dataIndex++; } } } - private static void EmbedRgb(Image image, string binaryData) + /// Embeds metadata bits in the RGB channels of an RGBA byte buffer. + public static void EmbedRgb(byte[] pixels, int width, int height, string binaryData) { int dataIndex = 0; - for (int x = 0; x < image.Width; x++) + for (int x = 0; x < width; x++) { - for (int y = 0; y < image.Height; y++) + for (int y = 0; y < height; y++) { if (dataIndex >= binaryData.Length) { return; } - Rgba32 pixel = image[x, y]; - pixel.R = (byte)((pixel.R & 0xFE) | (binaryData[dataIndex] - '0')); + int pixelIndex = (y * width + x) * 4; + pixels[pixelIndex] = (byte)((pixels[pixelIndex] & 0xFE) | (binaryData[dataIndex] - '0')); if (dataIndex + 1 < binaryData.Length) { - pixel.G = (byte)((pixel.G & 0xFE) | (binaryData[dataIndex + 1] - '0')); + pixels[pixelIndex + 1] = (byte)((pixels[pixelIndex + 1] & 0xFE) | (binaryData[dataIndex + 1] - '0')); } if (dataIndex + 2 < binaryData.Length) { - pixel.B = (byte)((pixel.B & 0xFE) | (binaryData[dataIndex + 2] - '0')); + pixels[pixelIndex + 2] = (byte)((pixels[pixelIndex + 2] & 0xFE) | (binaryData[dataIndex + 2] - '0')); } - image[x, y] = pixel; dataIndex += 3; } } diff --git a/src/Utils/OutputMetadataTracker.cs b/src/Utils/OutputMetadataTracker.cs index 84427c9ea6..237a3eb224 100644 --- a/src/Utils/OutputMetadataTracker.cs +++ b/src/Utils/OutputMetadataTracker.cs @@ -231,7 +231,7 @@ public static OutputPreviewEntry GetOrCreatePreviewFor(string file) byte[] data = File.ReadAllBytes(file); fileData = data; ImageFile img = new Image(data, MediaType.GetByExtension(ext)); - if (ext == "webp" && img.ToIS.Frames.Count == 1) + if (ext == "webp" && img.ToMagickFrames.Count == 1) { fileData = img.ToMetadataJpg()?.RawData; } diff --git a/src/WebAPI/T2IAPI.cs b/src/WebAPI/T2IAPI.cs index 2a06149dba..cf0138b1b6 100644 --- a/src/WebAPI/T2IAPI.cs +++ b/src/WebAPI/T2IAPI.cs @@ -1,8 +1,7 @@ using FreneticUtilities.FreneticExtensions; using FreneticUtilities.FreneticToolkit; using Newtonsoft.Json.Linq; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.Processing; +using ImageMagick; using SwarmUI.Accounts; using SwarmUI.Core; using SwarmUI.Media; @@ -15,8 +14,6 @@ using System.Net.WebSockets; using System.Runtime.InteropServices; using Image = SwarmUI.Utils.Image; -using ISImage = SixLabors.ImageSharp.Image; -using ISImageRGBA = SixLabors.ImageSharp.Image; namespace SwarmUI.WebAPI; @@ -454,7 +451,7 @@ void saveImage(T2IEngine.ImageOutput image, int actualIndex, T2IParamInput thisP T2IEngine.ImageOutput[] griddables = [.. imageSet.Where(i => i.IsReal)]; if (griddables.Length <= session.User.Settings.MaxImagesInMiniGrid && griddables.Length > 1 && griddables.All(i => i.File.Type.MetaType == MediaMetaType.Image)) { - ISImage[] imgs = [.. griddables.Select(i => (i.File as Image).ToIS)]; + IMagickImage[] imgs = [.. griddables.Select(i => (i.File as Image).ToMagick)]; int columns = (int)Math.Ceiling(Math.Sqrt(imgs.Length)); int rows = columns; if (griddables.Length <= columns * (columns - 1)) @@ -465,17 +462,14 @@ void saveImage(T2IEngine.ImageOutput image, int actualIndex, T2IParamInput thisP { (columns, rows) = GridShapeTable[imgs.Length - 1]; } - int widthPerImage = imgs.Max(i => i.Width); - int heightPerImage = imgs.Max(i => i.Height); - ISImageRGBA grid = new(widthPerImage * columns, heightPerImage * rows); - grid.Mutate(m => + int widthPerImage = (int)imgs.Max(i => i.Width); + int heightPerImage = (int)imgs.Max(i => i.Height); + using MagickImage grid = new(MagickColors.Transparent, (uint)(widthPerImage * columns), (uint)(heightPerImage * rows)) { Depth = 8 }; + for (int i = 0; i < imgs.Length; i++) { - for (int i = 0; i < imgs.Length; i++) - { - int x = (i % columns) * widthPerImage, y = (i / columns) * heightPerImage; - m.DrawImage(imgs[i], new Point(x, y), 1); - } - }); + int x = (i % columns) * widthPerImage, y = (i / columns) * heightPerImage; + grid.Composite(imgs[i], x, y, CompositeOperator.Over); + } Image gridImg = new(grid); long genTime = Environment.TickCount64 - timeStart; T2IParamInput finalInput = user_input.Clone();