From aaa9903fbf21f75cef99a3874aa33c5427cd9f91 Mon Sep 17 00:00:00 2001 From: Ygor Duarte Lemos Pereira Date: Wed, 28 Oct 2026 15:58:07 -0300 Subject: [PATCH] feat: palette mode in the container, encode options, faster decoding Encoder: - phf_encode_memory() and phf_write_file() take phf_encode_options_t { mode, effort } (NULL or {0} = AUTO, BEST). mode: MED (photos, camera, video), PALETTE (graphics, few colours: the monograph's compression by pixel, already in the NHF3 engine), AUTO. effort: BEST tries MED and palette up to 4096 colours and both plane variants (plain/run-length), keeping the smaller; FAST encodes once, for streaming (AUTO uses palette only up to 256 colours). - FAST picks the run-length variant only from 60% repeated symbols: the discarded second pass was 40-65% of encode time on photos and camera. - Palette files carry flags 0x08/0x09 (no MED/DECORR bits). Decoder (format unchanged): - Fix palette mode leaving alpha at 0 when decoding RGB into a 4-byte stride (images were transparent in the browser). - Allocate Huffman decoders only for the tables a plane carries instead of all 259 (618 KB -> typically 50-160 KB per plane). - Inverse MED runs the three colour planes in one interleaved pass when OpenMP does not split them (about 2x for that stage). Measured single-threaded on Apple M3 Pro, main -> this branch: - MED+FAST: 1080p photo encode 38.8 -> 15.4 ms, decode 30.9 -> 27.2 ms; sizes within +-2%. - AUTO+BEST on graphics: 71-76% smaller, 2-3x faster to decode. Also: SPEC.md section 4 documents the palette mode byte by byte (validated with an independent decoder written from the text only), CLI gains --mode=auto|med|palette and --fast and reports the coding mode, bench.py and README follow the new API, and the test suite grows from 470 to 2686 checks (every mode and effort, mode selection, palette fuzz). Co-Authored-By: Claude Opus 5.5 --- README.md | 32 ++++++- SPEC.md | 72 ++++++++++++-- apps/cli/phf_tool.py | 55 ++++++++--- bench/bench.py | 2 +- libphf/include/phf.h | 34 ++++++- libphf/src/nhf3_core.h | 12 ++- libphf/src/phf.c | 206 +++++++++++++++++++++++++++++++---------- libphf/src/phf_simd.h | 35 +++++++ tests/test_phf.c | 152 ++++++++++++++++++++++-------- 9 files changed, 484 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index a72f1d9..2111a9b 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,16 @@ By combining order-1 context-adaptive Huffman coding (N-Huffman) with SIMD-vectorized Median Edge Detection (MED) and length-limited (15-bit) canonical codes, PHF delivered **higher compression ratios than WebP Lossless** at **~300–400 MB/s** in the benchmark below. +PHF has two coding modes, and the encoder can pick between them: + +| Mode | Best for | How | +| :--- | :--- | :--- | +| **MED** | photos, camera, video | R−G/G/B−G planes, MED prediction, N-Huffman per plane | +| **Palette** | graphics, screenshots, UI, few colours | each pixel coded whole as an index into the image's colours, with the same N-Huffman model (the monograph's "compression by pixel") | + +On graphics the palette mode is about 70–75% smaller than MED and decodes 2–3× faster; on +photos MED wins. Files say which mode they use, so decoders need no option. + --- ## Benchmark: High-Resolution Photographic Dataset (Unsplash) @@ -132,8 +142,9 @@ Writes `apps/web/phf.js` and `apps/web/phf.wasm` (SIMD128 enabled). After rebuil ```bash ctest --test-dir build -C Release --output-on-failure ``` -Runs round-trips over many sizes and image types (RGB/RGBA, stride 3/4), header and error -checks, CRC detection, a corruption fuzz, file I/O, and decoding of `apps/web/sample.phf`. +Runs round-trips over many sizes and image types (RGB/RGBA, stride 3/4) in every mode and +effort, mode-selection checks, header and error checks, CRC detection, a corruption fuzz of +both modes, file I/O, and decoding of `apps/web/sample.phf`. CI runs the suite on Linux (with OpenMP and with sanitizers), macOS and Windows. ```bash @@ -151,8 +162,12 @@ PHF, WebP Lossless, PNG-6 and PNG-1, plus the machine and thread settings used. Requires the shared library (see above; or point `PHF_LIB` at it) and `pip install opencv-python numpy`. ```bash -# Encode an image to PHF (8-bit RGB, RGBA or grayscale; grayscale is stored as RGB) +# Encode an image to PHF (8-bit RGB, RGBA or grayscale; grayscale is stored as RGB). +# The default mode is auto: it also tries the palette mode when the image has at most +# 4096 colours and keeps the smaller file. python apps/cli/phf_tool.py encode photo.png photo.phf +python apps/cli/phf_tool.py encode logo.png logo.phf --mode=palette # or --mode=med +python apps/cli/phf_tool.py encode frame.png frame.phf --fast # one encode, no trials # Decode a PHF image back to PNG python apps/cli/phf_tool.py decode photo.phf photo.png @@ -171,10 +186,17 @@ JPEG inputs work too: PHF stores the decoded JPEG pixels losslessly (it cannot u ```c #include "phf.h" -// Encode RGB image to a .phf file (max 65535 px per side, 2^26 px total) -int err = phf_write_file("output.phf", rgb_buffer, width, height, 3); +// Encode RGB image to a .phf file (max 65535 px per side, 2^26 px total). +// NULL options = { PHF_MODE_AUTO, PHF_EFFORT_BEST }: best size for files. +int err = phf_write_file("output.phf", rgb_buffer, width, height, 3, NULL); if (err != PHF_OK) { /* PHF_ERR_* code */ } +// Streaming: fix the mode up front and encode once per frame. +phf_encode_options_t live = { PHF_MODE_MED, PHF_EFFORT_FAST }; // PALETTE for screen content +size_t size; +uint8_t *frame_phf = phf_encode_memory(rgb_frame, width, height, 3, &live, &size); +phf_free(frame_phf); + // Read and decode a .phf file from disk (CRC-32 is verified when present) phf_info_t info; uint8_t *decoded_rgb = phf_read_file("output.phf", &info); diff --git a/SPEC.md b/SPEC.md index d686d93..fbd2522 100644 --- a/SPEC.md +++ b/SPEC.md @@ -44,9 +44,10 @@ All pixel arithmetic is modulo 256. | 3 | `0x08` | `CRC32` | A 4-byte CRC-32 follows the payload (see 1.4). | | 4–7 | | | Reserved, must be `0`. | -A v1 writer produces `0x0E` for RGB and `0x0F` for RGBA. Bits 0–2 are descriptive: the -decoder takes the actual coding modes from the NHF3 packet header (section 2), and they -must agree with it. Files written before bit 3 existed carry no CRC; files from early +A v1 writer produces `0x0E` (RGB) or `0x0F` (RGBA) for the MED mode and `0x08` (RGB) or +`0x09` (RGBA) for the palette mode (section 4). Bits 0–2 are descriptive: the decoder takes +the actual coding modes from the NHF3 packet header (section 2), and they must agree with +it. Files written before bit 3 existed carry no CRC; files from early encoder builds also carry `0x04`/`0x05` (the MED bit was not set) and are still valid. ### 1.3 Size Limits @@ -78,7 +79,7 @@ A decoder must check, in this order: | :---: | :---: | :---: | :--- | :--- | | 0 | 4 | ASCII | `magic` | `"NHF3"`. | | 4 | 1 | u8 | `version` | `1`. | -| 5 | 1 | u8 | `mode` | bit 0: P-frame (must be 0 in PHF); bits 1–2: predictor (`0` none, `1` left, `2` MED, `3` palette); bit 3: color decorrelation. PHF v1 writers use `0x0C` (MED + decorrelation). | +| 5 | 1 | u8 | `mode` | bit 0: P-frame (must be 0 in PHF); bits 1–2: predictor (`0` none, `1` left, `2` MED, `3` palette); bit 3: color decorrelation. PHF v1 writers use `0x0C` (MED + decorrelation) or `0x06` (palette, section 4). | | 6 | 2 | u16 BE | `width` | | | 8 | 2 | u16 BE | `height` | | | 10 | 1 | u8 | `ch` | Channel count (3 or 4 in PHF). | @@ -88,9 +89,9 @@ The plane streams follow, concatenated in plane order. The sum of `plane_len` pl header must equal `payload_size` exactly. Each plane can be decoded independently once its bytes are available. -PHF v1 writers only produce predictor `2` with decorrelation. The reference decoder also -accepts predictors `0`/`1` from the NHF3 engine; the palette mode (`3`) is not part of this -specification, and independent decoders may reject it. +PHF v1 writers produce predictor `2` with decorrelation (sections 2.2–3) or predictor `3` +without it (section 4). The reference decoder also accepts predictors `0`/`1` from the NHF3 +engine; independent decoders may reject those. ### 2.2 Planes and Color Transform @@ -191,13 +192,66 @@ sequence, consecutive symbols always differ. --- -## 4. Streaming Notes +## 4. Palette Mode (predictor `3`) + +Each pixel is coded whole, as an index into the image's own list of colours, with the same +order-1 N-Huffman model as section 3 but over indices instead of bytes. It suits graphics, +screenshots and UI; for photos the MED planes are smaller. + +The packet header has `mode = 0x06` (predictor 3, no decorrelation). The first plane length +holds the whole palette stream and the other `ch − 1` lengths are `0`. + +### 4.1 Layout + +``` +P : u24 BE number of colours, 1 <= P <= w·h +palette : P · ch bytes colour j is bytes [j·ch, (j+1)·ch), in R, G, B(, A) order +if P == 1 : nothing else follows (every pixel is colour 0) +otherwise : a bitstream (MSB-first, zero-padded to a byte) with b = bits(P − 1) = ⌊log2(P − 1)⌋ + 1: + first : b bits first index (< P) + rle : 1 bit as in 3.1, followed by the two run tables when set + fb_mode : 2 bits 0 = no fallback; 1 = fallback indices are raw b-bit values; + 2 = fallback code table (3.2) over indices 0..P−1, only if P <= 256; + 3 = invalid + if fb_mode == 2: code table (3.2) + contexts : P descriptors, for c = 0..P−1: + used : 1 bit + if used: kind : 2 bits + 0 = SINGLE : idx : b bits (< P) + 1 = OWN : index code table (4.2) + 2 = FALLBACK : uses the fallback (requires fb_mode != 0) + 3 = invalid + indices : as in 3.3 (rle = 0) or 3.4 (rle = 1), with indices for symbols and the + previous index as the context; run table t = (index != 0) ? 1 : 0 +``` + +Pixel `i` is `palette[index_i]`. An RGB image decoded into a 4-byte stride gets an alpha +of 255, as in the MED mode. + +### 4.2 Index Code Table + +Like 3.2, widened to `b`-bit indices: + +``` +k − 1 : b bits k = number of coded indices, 2 <= k <= min(P, 16384) +k entries, in increasing index order: + gap : Elias-gamma index − previous index (previous starts at −1); m <= b + len : 4 bits 1..15 +``` + +The lengths must form a complete prefix code; codes are canonical (3.2) over the listed +indices. + +--- + +## 5. Streaming Notes - **Header probe:** the first 20 bytes give width, height, channel count, the required output size (`width · height · stride`) and the total file size, so a receiver can allocate buffers before the rest arrives (`phf_probe_memory()`). - **Independent planes:** after `20 + 11 + 4·ch` bytes the receiver knows where each plane - starts and ends, and each plane decodes on its own. The reference API + starts and ends, and each plane decodes on its own (MED mode; the palette mode is a + single stream). The reference API (`phf_decode_memory()`) currently needs the complete file; incremental decoding is a possible future API, not something v1 provides. - **Inter-frame (P-frame) coding** exists in the underlying NHF3 engine (`nhf3_encode` with diff --git a/apps/cli/phf_tool.py b/apps/cli/phf_tool.py index 4b962f1..71b8e7f 100644 --- a/apps/cli/phf_tool.py +++ b/apps/cli/phf_tool.py @@ -26,6 +26,11 @@ PHF_FLAG_DECORR = 0x04 PHF_FLAG_CRC32 = 0x08 +# phf_mode_t / phf_effort_t +PHF_MODES = {"auto": 0, "med": 1, "palette": 2} +PHF_EFFORT_BEST = 0 +PHF_EFFORT_FAST = 1 + PHF_ERRORS = { -1: "invalid argument", -2: "not a PHF file (bad magic)", @@ -79,7 +84,11 @@ class PHFInfo(ctypes.Structure): libphf.phf_probe_memory.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(PHFInfo)] libphf.phf_probe_memory.restype = ctypes.c_int -libphf.phf_encode_memory.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint8, ctypes.POINTER(ctypes.c_size_t)] +class PHFEncodeOptions(ctypes.Structure): + _fields_ = [("mode", ctypes.c_int), ("effort", ctypes.c_int)] + +libphf.phf_encode_memory.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint8, + ctypes.POINTER(PHFEncodeOptions), ctypes.POINTER(ctypes.c_size_t)] libphf.phf_encode_memory.restype = ctypes.c_void_p libphf.phf_decode_memory.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint8] @@ -111,7 +120,7 @@ def load_rgb(input_path): return None, f"unsupported channel count {img.shape[2]}" -def encode_image(input_path, output_path=None): +def encode_image(input_path, output_path=None, mode="auto", fast=False): if not os.path.exists(input_path): print(f"Error: File not found: {input_path}") return False @@ -130,9 +139,10 @@ def encode_image(input_path, output_path=None): print(f"Encoding: {input_path}") print(f" Dimensions: {w}x{h} ({channels} channels) | Raw: {raw_bytes / (1024*1024):.2f} MB") + opts = PHFEncodeOptions(PHF_MODES[mode], PHF_EFFORT_FAST if fast else PHF_EFFORT_BEST) out_sz = ctypes.c_size_t(0) t0 = time.perf_counter() - ptr = libphf.phf_encode_memory(rgb.ctypes.data, w, h, channels, ctypes.byref(out_sz)) + ptr = libphf.phf_encode_memory(rgb.ctypes.data, w, h, channels, ctypes.byref(opts), ctypes.byref(out_sz)) enc_time = time.perf_counter() - t0 if not ptr: @@ -150,12 +160,19 @@ def encode_image(input_path, output_path=None): savings = (1.0 - comp_bytes / raw_bytes) * 100.0 throughput = (raw_bytes / (1024*1024)) / enc_time - print(f" -> Saved: {output_path}") + print(f" -> Saved: {output_path} ({coding_mode(phf_bytes)} mode)") print(f" -> Size: {comp_bytes / (1024*1024):.2f} MB (Ratio: {ratio:.2f}x, Space Savings: {savings:.1f}%)") print(f" -> Speed: {enc_time*1000:.1f} ms ({throughput:.1f} MB/s)") return True +def coding_mode(phf_bytes): + """Name of the NHF3 predictor stored in the payload header (byte 5 of the payload).""" + if len(phf_bytes) < 26: + return "unknown" + return {0: "none", 1: "left", 2: "MED", 3: "palette"}[(phf_bytes[25] >> 1) & 3] + + def probe(phf_bytes): info = PHFInfo() rc = libphf.phf_probe_memory(phf_bytes, len(phf_bytes), ctypes.byref(info)) @@ -254,6 +271,7 @@ def info_image(input_path): print(f"Resolution: {info.width} x {info.height} pixels") print(f"Channels: {info.channels} ({'RGBA' if info.channels == 4 else 'RGB'})") print(f"Flags: 0x{info.flags:02x} ({', '.join(flag_names) or 'none'})") + print(f"Coding mode: {coding_mode(phf_bytes)}") print(f"Raw Size: {raw_sz / (1024*1024):.2f} MB ({raw_sz} bytes)") print(f"PHF Size: {len(phf_bytes) / (1024*1024):.2f} MB ({len(phf_bytes)} bytes)") print(f"Compression: {ratio:.2f}x ({savings:.1f}% space savings)") @@ -263,22 +281,37 @@ def info_image(input_path): def main(): - if len(sys.argv) < 3: + args = [a for a in sys.argv[1:] if not a.startswith("--")] + flags = [a for a in sys.argv[1:] if a.startswith("--")] + mode, fast = "auto", False + for f in flags: + if f.startswith("--mode=") and f[7:] in PHF_MODES: + mode = f[7:] + elif f == "--fast": + fast = True + else: + print(f"Unknown option: {f}") + return 2 + + if len(args) < 2: print("PHF Utility v1.0 — Pereira-Huffman Format") print("Usage:") - print(" python phf_tool.py encode [output.phf]") + print(" python phf_tool.py encode [output.phf] [--mode=auto|med|palette] [--fast]") print(" python phf_tool.py decode [output.png]") print(" python phf_tool.py info ") + print("Modes: med = photos, camera, video; palette = graphics, screenshots, few colours;") + print(" auto (default) tries both when the image has at most 4096 colours.") + print("--fast: encode once without trials (what streaming uses); default is best size.") return 2 - cmd = sys.argv[1].lower() - out = sys.argv[3] if len(sys.argv) >= 4 else None + cmd = args[0].lower() + out = args[2] if len(args) >= 3 else None if cmd == "encode": - ok = encode_image(sys.argv[2], out) + ok = encode_image(args[1], out, mode, fast) elif cmd == "decode": - ok = decode_image(sys.argv[2], out) + ok = decode_image(args[1], out) elif cmd == "info": - ok = info_image(sys.argv[2]) + ok = info_image(args[1]) else: print(f"Unknown command: {cmd}") return 2 diff --git a/bench/bench.py b/bench/bench.py index 268b458..86010ee 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -30,7 +30,7 @@ def phf_encode(rgb, ch): h, w = rgb.shape[:2] out_sz = ctypes.c_size_t(0) - ptr = lib.phf_encode_memory(rgb.ctypes.data, w, h, ch, ctypes.byref(out_sz)) + ptr = lib.phf_encode_memory(rgb.ctypes.data, w, h, ch, None, ctypes.byref(out_sz)) if not ptr: raise RuntimeError("phf_encode_memory failed") data = ctypes.string_at(ptr, out_sz.value) diff --git a/libphf/include/phf.h b/libphf/include/phf.h index be5752d..e4db419 100644 --- a/libphf/include/phf.h +++ b/libphf/include/phf.h @@ -44,6 +44,31 @@ extern "C" { #define PHF_ERR_MEM -9 /* allocation failure */ #define PHF_ERR_IO -10 /* file open/write failure */ +/* Coding mode. MED predicts each colour plane from its neighbours: best for photos, + * camera and video. PALETTE codes whole pixels as indices into the image's colours: + * best (and faster) for graphics, screenshots and UI with few colours. AUTO picks + * per image; see phf_effort_t. */ +typedef enum { + PHF_MODE_AUTO = 0, + PHF_MODE_MED = 1, + PHF_MODE_PALETTE = 2 +} phf_mode_t; + +/* Encoder effort. BEST is for files: AUTO encodes both modes when the image has at + * most 4096 colours and keeps the smaller one, and each plane tries both of its + * variants. FAST is for streaming: one encode, no trials; AUTO uses PALETTE only + * for images with at most 256 colours. Decoding speed does not depend on it. */ +typedef enum { + PHF_EFFORT_BEST = 0, + PHF_EFFORT_FAST = 1 +} phf_effort_t; + +/* A zero-initialised struct (or a NULL pointer) means { AUTO, BEST }. */ +typedef struct { + phf_mode_t mode; + phf_effort_t effort; +} phf_encode_options_t; + typedef struct { uint32_t width; uint32_t height; @@ -65,14 +90,17 @@ PHF_API int phf_probe_memory(const uint8_t *data, size_t size, phf_info_t *out_i * Encodes an RGB or RGBA pixel buffer into the full PHF file container (.phf). * The returned pointer includes the 20-byte header, the N-Huffman compressed * payload and a 4-byte CRC-32 trailer. Must be freed with phf_free(). + * options may be NULL for { PHF_MODE_AUTO, PHF_EFFORT_BEST }; streaming callers + * usually want a fixed mode with PHF_EFFORT_FAST. * Width and height must each be <= PHF_MAX_DIM and width*height <= PHF_MAX_PIXELS; - * returns NULL otherwise or on allocation failure. + * returns NULL otherwise, on an invalid option, or on allocation failure. */ PHF_API uint8_t* phf_encode_memory( const uint8_t *pixels, uint32_t width, uint32_t height, uint8_t channels, + const phf_encode_options_t *options, size_t *out_size ); @@ -99,6 +127,7 @@ PHF_API int phf_decode_memory( /** * phf_write_file: * Encodes and writes an RGB/RGBA image directly to a .phf file on disk. + * options as in phf_encode_memory (NULL = AUTO, BEST). * Returns PHF_OK on success, or a negative PHF_ERR_* code. */ PHF_API int phf_write_file( @@ -106,7 +135,8 @@ PHF_API int phf_write_file( const uint8_t *pixels, uint32_t width, uint32_t height, - uint8_t channels + uint8_t channels, + const phf_encode_options_t *options ); /** diff --git a/libphf/src/nhf3_core.h b/libphf/src/nhf3_core.h index 0780186..2f0039d 100644 --- a/libphf/src/nhf3_core.h +++ b/libphf/src/nhf3_core.h @@ -34,6 +34,11 @@ enum { NHF3_PRED_NONE = 0, NHF3_PRED_LEFT = 1, NHF3_PRED_MED = 2, NHF3_PRED_PIXEL = 3 }; +/* Encoder effort. FAST commits to one plane variant (plain or run-length) from the + * share of repeated symbols and encodes once: for streaming. BEST encodes both + * variants whenever runs are plausible and keeps the smaller: for files. */ +enum { NHF3_EFFORT_FAST = 0, NHF3_EFFORT_BEST = 1 }; + enum { NHF3_OK = 0, NHF3_ERR_ARG = -1, @@ -51,8 +56,8 @@ typedef struct { } NHF3PlaneStats; /* ---- Plane level (one stream of bytes, order-1 context) ----------------- */ -/* Encodes n symbols. Returns a malloc'd buffer (free with nhf3_free). */ -NHF3_API uint8_t *nhf3_plane_encode(const uint8_t *sym, uint32_t n, size_t *out_size, NHF3PlaneStats *stats); +/* Encodes n symbols with NHF3_EFFORT_*. Returns a malloc'd buffer (free with nhf3_free). */ +NHF3_API uint8_t *nhf3_plane_encode(const uint8_t *sym, uint32_t n, int effort, size_t *out_size, NHF3PlaneStats *stats); /* Decodes exactly n symbols. Returns NHF3_OK or a negative error. */ NHF3_API int nhf3_plane_decode(const uint8_t *in, size_t in_size, uint8_t *out, uint32_t n); @@ -61,9 +66,10 @@ NHF3_API int nhf3_plane_decode(const uint8_t *in, size_t in_size, uint8_t *out, * prev: previous reconstructed frame (same layout) for a P-frame, or NULL. * pred: NHF3_PRED_NONE, NHF3_PRED_LEFT, NHF3_PRED_MED, or NHF3_PRED_PIXEL * (whole-pixel palette N-Huffman mode, activated by --pixelmode). + * effort: NHF3_EFFORT_FAST or NHF3_EFFORT_BEST. * Returns a malloc'd NHF3 packet. stats (optional) receives one entry per channel. */ NHF3_API uint8_t *nhf3_encode(const uint8_t *src, int w, int h, int ch, int stride, - const uint8_t *prev, int pred, int decorrelate, + const uint8_t *prev, int pred, int decorrelate, int effort, size_t *out_size, NHF3PlaneStats *stats); /* Same as nhf3_encode (always lossless). `delta` is reserved for a future diff --git a/libphf/src/phf.c b/libphf/src/phf.c index ec4e0db..941177a 100644 --- a/libphf/src/phf.c +++ b/libphf/src/phf.c @@ -284,21 +284,28 @@ static uint8_t *encode_stream(const uint8_t *v, uint32_t m, const uint32_t *runs return w.buf; } -NHF3_API uint8_t *nhf3_plane_encode(const uint8_t *sym, uint32_t n, size_t *out_size, NHF3PlaneStats *st) { +/* Which variants of a stream to encode, given how many symbols repeat the previous + * one. FAST encodes once: on photos, camera and video frames the run-length variant + * only won from about 60% repeats up. BEST encodes both whenever runs are plausible + * and the caller keeps the smaller. */ +static void rle_plan(uint32_t reps, uint32_t n, int effort, int *try_raw, int *try_rle) { + if (effort == NHF3_EFFORT_FAST) { + *try_rle = (uint64_t)reps * 100 >= (uint64_t)n * 60; + *try_raw = !*try_rle; + } else { + *try_raw = 1; + *try_rle = (uint64_t)reps * 20 >= n; + } +} + +NHF3_API uint8_t *nhf3_plane_encode(const uint8_t *sym, uint32_t n, int effort, size_t *out_size, NHF3PlaneStats *st) { if (!sym || !n || !out_size) return NULL; uint32_t reps = 0; for (uint32_t i = 1; i < n; i++) reps += (sym[i] == sym[i - 1]); - /* For large video planes (n >= 64000), avoid encoding both non-RLE and RLE - * when one regime clearly dominates: - * - If reps >= 45% of n (typical P-frame or flat anime region), RLE always wins. - * - If reps < 18% of n (noisy texture), non-RLE always wins because RLE pays >= 1 bit/run. */ - int try_raw = 1, try_rle = ((uint64_t)reps * 20 >= n); - if (n >= 64000) { - if ((uint64_t)reps * 100 >= (uint64_t)n * 45) try_raw = 0; - else if ((uint64_t)reps * 100 < (uint64_t)n * 30) try_rle = 0; - } + int try_raw, try_rle; + rle_plan(reps, n, effort, &try_raw, &try_rle); size_t sa = 0; NHF3PlaneStats sta; @@ -345,15 +352,33 @@ static int read_run_table(BitReader *r, int *single, int *cls, HuffDec *d) { return huff_build_decoder(len, RUN_CLASSES, d); } +/* Appends one decoder to a growable pool and returns its index, or -1 when out of + * memory. Indices stay valid across reallocations; pointers do not. */ +static int dec_pool_add(HuffDec **pool, int *count, int *cap) { + if (*count == *cap) { + int ncap = *cap ? *cap * 2 : 16; + HuffDec *np = (HuffDec *)realloc(*pool, (size_t)ncap * sizeof(HuffDec)); + if (!np) return -1; + *pool = np; + *cap = ncap; + } + return (*count)++; +} + NHF3_API int nhf3_plane_decode(const uint8_t *in, size_t in_size, uint8_t *out, uint32_t n) { if (!in || !out || !n) return NHF3_ERR_ARG; if (in_size < 1) return NHF3_ERR_TRUNCATED; - HuffDec *dec = (HuffDec *)malloc(259 * sizeof(HuffDec)); /* 0..255 ctx, 256 fallback, 257/258 runs */ - if (!dec) return NHF3_ERR_MEM; + /* Decoders (~2.4 KB each) exist only for the tables the stream carries: + * typically 10-60 per plane rather than one per possible context. */ + HuffDec *dec = NULL; + int dec_count = 0, dec_cap = 0; + int16_t ctx_idx[256]; + int run_idx[2] = { -1, -1 }, fb_idx = -1; uint8_t kind[256], ssym[256], len[256]; memset(kind, 0, sizeof(kind)); memset(ssym, 0, sizeof(ssym)); + for (int c = 0; c < 256; c++) ctx_idx[c] = -1; int rc = NHF3_OK; int rsingle[2] = { 1, 1 }, rcls[2] = { 0, 0 }; @@ -362,31 +387,44 @@ NHF3_API int nhf3_plane_decode(const uint8_t *in, size_t in_size, uint8_t *out, uint8_t first = (uint8_t)br_get(&r, 8); int rle = (int)br_get(&r, 1); if (rle) { - for (int t = 0; t < 2; t++) - if (!read_run_table(&r, &rsingle[t], &rcls[t], &dec[257 + t])) { rc = NHF3_ERR_CORRUPT; goto done; } + for (int t = 0; t < 2; t++) { + int idx = dec_pool_add(&dec, &dec_count, &dec_cap); + if (idx < 0) { rc = NHF3_ERR_MEM; goto done; } + if (!read_run_table(&r, &rsingle[t], &rcls[t], &dec[idx])) { rc = NHF3_ERR_CORRUPT; goto done; } + if (rsingle[t]) dec_count--; /* fixed class: no table */ + else run_idx[t] = idx; + } } int have_fb = (int)br_get(&r, 1); if (have_fb) { - if (!get_table(&r, len) || !huff_build_decoder(len, 256, &dec[256])) { rc = NHF3_ERR_CORRUPT; goto done; } + fb_idx = dec_pool_add(&dec, &dec_count, &dec_cap); + if (fb_idx < 0) { rc = NHF3_ERR_MEM; goto done; } + if (!get_table(&r, len) || !huff_build_decoder(len, 256, &dec[fb_idx])) { rc = NHF3_ERR_CORRUPT; goto done; } } - HuffDec *ctx_dec[256]; - memset(ctx_dec, 0, sizeof(ctx_dec)); for (int c = 0; c < 256; c++) { if (!br_get(&r, 1)) continue; uint32_t k = br_get(&r, 2); if (k == 0) { kind[c] = K_SINGLE; ssym[c] = (uint8_t)br_get(&r, 8); } else if (k == 1) { - if (!get_table(&r, len) || !huff_build_decoder(len, 256, &dec[c])) { rc = NHF3_ERR_CORRUPT; goto done; } + int idx = dec_pool_add(&dec, &dec_count, &dec_cap); + if (idx < 0) { rc = NHF3_ERR_MEM; goto done; } + if (!get_table(&r, len) || !huff_build_decoder(len, 256, &dec[idx])) { rc = NHF3_ERR_CORRUPT; goto done; } kind[c] = K_OWN; - ctx_dec[c] = &dec[c]; + ctx_idx[c] = (int16_t)idx; } else if (k == 2) { if (!have_fb) { rc = NHF3_ERR_CORRUPT; goto done; } kind[c] = K_FALLBACK; - ctx_dec[c] = &dec[256]; + ctx_idx[c] = (int16_t)fb_idx; } else { rc = NHF3_ERR_CORRUPT; goto done; } } if (br_overrun(&r)) { rc = NHF3_ERR_TRUNCATED; goto done; } + /* The pool no longer grows: resolve indices to pointers. */ + const HuffDec *ctx_dec[256]; + for (int c = 0; c < 256; c++) ctx_dec[c] = ctx_idx[c] >= 0 ? &dec[ctx_idx[c]] : NULL; + const HuffDec *run_dec[2] = { run_idx[0] >= 0 ? &dec[run_idx[0]] : NULL, + run_idx[1] >= 0 ? &dec[run_idx[1]] : NULL }; + { uint8_t prev = first; uint32_t pos = 0; @@ -446,7 +484,7 @@ NHF3_API int nhf3_plane_decode(const uint8_t *in, size_t in_size, uint8_t *out, int t_ = ((sym_) != 0); \ int k_; \ if (rsingle[t_]) k_ = rcls[t_]; \ - else LOCAL_DECODE(&dec[257 + t_], k_); \ + else LOCAL_DECODE(run_dec[t_], k_); \ if (k_ < 0 || k_ >= RUN_CLASSES) { rc = NHF3_ERR_CORRUPT; goto done; } \ uint32_t ex_ = 0; \ if (k_) { \ @@ -778,7 +816,7 @@ static uint8_t *encode_pixel_stream(const uint32_t *v, uint32_t m, const uint32_ } static uint8_t *pixel_plane_encode(const uint8_t *pal, uint32_t P, int ch, - const uint32_t *sym, uint32_t n, + const uint32_t *sym, uint32_t n, int effort, size_t *out_size, NHF3PlaneStats *st) { size_t pal_hdr = 3 + (size_t)P * (size_t)ch; if (P == 1) { @@ -793,11 +831,8 @@ static uint8_t *pixel_plane_encode(const uint8_t *pal, uint32_t P, int ch, uint32_t reps = 0; for (uint32_t i = 1; i < n; i++) reps += (sym[i] == sym[i - 1]); - int try_raw = 1, try_rle = ((uint64_t)reps * 20 >= n); - if (n >= 64000) { - if ((uint64_t)reps * 100 >= (uint64_t)n * 45) try_raw = 0; - else if ((uint64_t)reps * 100 < (uint64_t)n * 30) try_rle = 0; - } + int try_raw, try_rle; + rle_plan(reps, n, effort, &try_raw, &try_rle); size_t sa = 0; NHF3PlaneStats sta; @@ -855,12 +890,15 @@ static int pixel_plane_decode(const uint8_t *in, size_t in_size, uint8_t *dst, const uint8_t *pal = in + 3; size_t rem = in_size - 3 - pal_bytes; + /* RGB decoded into a 4-byte stride gets an opaque alpha, as in the plane path. */ + int fill_alpha = (ch == 3 && stride >= 4); if (P == 1) { if (rem != 0) return NHF3_ERR_CORRUPT; for (uint32_t i = 0; i < n; i++) { uint8_t *d = dst + (size_t)i * stride; if (pf) for (int c = 0; c < ch; c++) d[c] = (uint8_t)(d[c] + pal[c]); else for (int c = 0; c < ch; c++) d[c] = pal[c]; + if (fill_alpha) d[3] = 255; } return NHF3_OK; } @@ -959,11 +997,13 @@ static int pixel_plane_decode(const uint8_t *in, size_t in_size, uint8_t *dst, for (; pos < end_; pos++) { \ uint8_t *d_ = dst + (size_t)pos * stride; \ for (int c_ = 0; c_ < ch; c_++) d_[c_] = (uint8_t)(d_[c_] + sc_[c_]); \ + if (fill_alpha) d_[3] = 255; \ } \ } else { \ for (; pos < end_; pos++) { \ uint8_t *d_ = dst + (size_t)pos * stride; \ for (int c_ = 0; c_ < ch; c_++) d_[c_] = sc_[c_]; \ + if (fill_alpha) d_[3] = 255; \ } \ } \ } while (0) @@ -1030,7 +1070,7 @@ static uint32_t get_u16(const uint8_t *p) { return ((uint32_t)p[0] << 8) | p[1]; static uint32_t get_u32(const uint8_t *p) { return (get_u16(p) << 16) | get_u16(p + 2); } static uint8_t *encode_pixel_frame(const uint8_t *src, int w, int h, int ch, int stride, - const uint8_t *prev, uint8_t *recon_out, + const uint8_t *prev, int effort, uint8_t *recon_out, size_t *out_size, NHF3PlaneStats *stats) { uint32_t n = (uint32_t)w * (uint32_t)h; int pframe = prev != NULL; @@ -1068,7 +1108,7 @@ static uint8_t *encode_pixel_frame(const uint8_t *src, int w, int h, int ch, int if (stats) memset(stats, 0, (size_t)ch * sizeof(NHF3PlaneStats)); size_t plen = 0; - pbuf = pixel_plane_encode(pal, P, ch, pix, n, &plen, stats ? &stats[0] : NULL); + pbuf = pixel_plane_encode(pal, P, ch, pix, n, effort, &plen, stats ? &stats[0] : NULL); if (!pbuf) goto fail; size_t total = NHF3_HEADER_BASE + 4 * (size_t)ch + plen; @@ -1093,13 +1133,13 @@ static uint8_t *encode_pixel_frame(const uint8_t *src, int w, int h, int ch, int } NHF3_API uint8_t *nhf3_encode(const uint8_t *src, int w, int h, int ch, int stride, - const uint8_t *prev, int pred, int decorrelate, + const uint8_t *prev, int pred, int decorrelate, int effort, size_t *out_size, NHF3PlaneStats *stats) { if (!src || !out_size || w <= 0 || h <= 0 || w > 65535 || h > 65535) return NULL; if (ch < 1 || ch > 4 || stride < ch || pred < 0 || pred > 3) return NULL; if ((uint64_t)w * (uint64_t)h > MAX_PIXELS) return NULL; if (pred == NHF3_PRED_PIXEL) { - return encode_pixel_frame(src, w, h, ch, stride, prev, NULL, out_size, stats); + return encode_pixel_frame(src, w, h, ch, stride, prev, effort, NULL, out_size, stats); } uint32_t n = (uint32_t)w * (uint32_t)h; int pframe = prev != NULL; @@ -1177,7 +1217,7 @@ NHF3_API uint8_t *nhf3_encode(const uint8_t *src, int w, int h, int ch, int stri #pragma omp parallel for schedule(dynamic) if(n >= 64000) #endif for (c_omp = 0; c_omp < ch; c_omp++) { - bufs[c_omp] = nhf3_plane_encode(planes + (size_t)c_omp * n, n, &lens[c_omp], stats ? &stats[c_omp] : NULL); + bufs[c_omp] = nhf3_plane_encode(planes + (size_t)c_omp * n, n, effort, &lens[c_omp], stats ? &stats[c_omp] : NULL); } for (int c = 0; c < ch; c++) if (!bufs[c]) goto fail; for (int c = 0; c < ch; c++) total += lens[c]; @@ -1206,14 +1246,14 @@ NHF3_API uint8_t *nhf3_encode_near(const uint8_t *src, int w, int h, int ch, int uint8_t *recon_out, size_t *out_size, NHF3PlaneStats *stats) { (void)delta; if (recon_out && recon_out != src) memcpy(recon_out, src, (size_t)w * h * stride); - return nhf3_encode(src, w, h, ch, stride, prev, pred, decorrelate, out_size, stats); + return nhf3_encode(src, w, h, ch, stride, prev, pred, decorrelate, NHF3_EFFORT_BEST, out_size, stats); } NHF3_API uint8_t *nhf3_encode_pixel(const uint8_t *src, int w, int h, int ch, int stride, const uint8_t *prev, int delta, uint8_t *recon_out, size_t *out_size, NHF3PlaneStats *stats) { (void)delta; - return encode_pixel_frame(src, w, h, ch, stride, prev, recon_out, out_size, stats); + return encode_pixel_frame(src, w, h, ch, stride, prev, NHF3_EFFORT_BEST, recon_out, out_size, stats); } NHF3_API uint8_t *nhf3_encode_auto(const uint8_t *src, int w, int h, int ch, int stride, @@ -1222,7 +1262,7 @@ NHF3_API uint8_t *nhf3_encode_auto(const uint8_t *src, int w, int h, int ch, int (void)delta; if (!src || w <= 0 || h <= 0 || ch < 1 || ch > 4 || stride < ch || !out_size) return NULL; if (recon_out && recon_out != src) memcpy(recon_out, src, (size_t)w * h * stride); - if (!prev) return nhf3_encode(src, w, h, ch, stride, NULL, pred, decorrelate, out_size, stats); + if (!prev) return nhf3_encode(src, w, h, ch, stride, NULL, pred, decorrelate, NHF3_EFFORT_BEST, out_size, stats); int p_pred = (pred == NHF3_PRED_PIXEL) ? NHF3_PRED_PIXEL : NHF3_PRED_NONE; @@ -1254,15 +1294,15 @@ NHF3_API uint8_t *nhf3_encode_auto(const uint8_t *src, int w, int h, int ch, int } } if (static_p * 4 < samples && sad_p > sad_i * 2) { - return nhf3_encode(src, w, h, ch, stride, NULL, pred, decorrelate, out_size, stats); + return nhf3_encode(src, w, h, ch, stride, NULL, pred, decorrelate, NHF3_EFFORT_BEST, out_size, stats); } - return nhf3_encode(src, w, h, ch, stride, prev, p_pred, decorrelate, out_size, stats); + return nhf3_encode(src, w, h, ch, stride, prev, p_pred, decorrelate, NHF3_EFFORT_BEST, out_size, stats); } size_t sz_i = 0, sz_p = 0; NHF3PlaneStats st_i[4], st_p[4]; - uint8_t *pk_i = nhf3_encode(src, w, h, ch, stride, NULL, pred, decorrelate, &sz_i, st_i); - uint8_t *pk_p = nhf3_encode(src, w, h, ch, stride, prev, p_pred, decorrelate, &sz_p, st_p); + uint8_t *pk_i = nhf3_encode(src, w, h, ch, stride, NULL, pred, decorrelate, NHF3_EFFORT_BEST, &sz_i, st_i); + uint8_t *pk_p = nhf3_encode(src, w, h, ch, stride, prev, p_pred, decorrelate, NHF3_EFFORT_BEST, &sz_p, st_p); if (pk_p && (!pk_i || sz_p < sz_i)) { free(pk_i); @@ -1337,7 +1377,16 @@ NHF3_API int nhf3_decode(const uint8_t *in, size_t n, uint8_t *dst, int w, int h if (plane_err[c] != NHF3_OK) { free(planes); return plane_err[c]; } } - if (pred != NHF3_PRED_NONE) { + /* With OpenMP, large frames run one thread per plane; otherwise the three + * colour planes share one interleaved pass. */ + int med3 = pred == NHF3_PRED_MED && ch >= 3; +#ifdef _OPENMP + if (cnt >= 64000) med3 = 0; +#endif + if (med3) { + nhf3_spatial_inverse_med3(planes, planes + cnt, planes + 2 * (size_t)cnt, pw, ph); + if (ch == 4) spatial_inverse(planes + 3 * (size_t)cnt, pw, ph, pred); + } else if (pred != NHF3_PRED_NONE) { #ifdef _OPENMP #pragma omp parallel for if(cnt >= 64000) #endif @@ -1462,17 +1511,79 @@ PHF_API int phf_probe_memory(const uint8_t *data, size_t size, phf_info_t *out_i return PHF_OK; } -PHF_API uint8_t* phf_encode_memory(const uint8_t *pixels, uint32_t width, uint32_t height, uint8_t channels, size_t *out_size) { +/* AUTO with BEST also encodes the palette mode up to this many colours; AUTO with + * FAST uses it, without trying MED, up to the smaller limit. On the test images the + * palette mode was never smaller beyond a few thousand colours and got much slower. */ +#define PHF_AUTO_PALETTE_TRY 4096u +#define PHF_AUTO_PALETTE_FAST 256u + +/* Number of distinct colours, or limit + 1 as soon as there are more. */ +static uint32_t count_colors_upto(const uint8_t *px, uint32_t n, int ch, uint32_t limit) { + uint32_t cap = 16; + while (cap < 2 * (limit + 1)) cap <<= 1; + uint32_t *keys = (uint32_t *)malloc((size_t)cap * sizeof(uint32_t)); + uint8_t *used = (uint8_t *)calloc(cap, 1); + if (!keys || !used) { free(keys); free(used); return limit + 1; } + uint32_t count = 0, prev_key = 0; + for (uint32_t i = 0; i < n && count <= limit; i++) { + const uint8_t *p = px + (size_t)i * ch; + uint32_t key = (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) + | (ch == 4 ? (uint32_t)p[3] << 24 : 0); + if (i && key == prev_key) continue; /* runs are common in graphics */ + prev_key = key; + uint32_t h = (key * 2654435761u) & (cap - 1); + while (used[h] && keys[h] != key) h = (h + 1) & (cap - 1); + if (!used[h]) { used[h] = 1; keys[h] = key; count++; } + } + free(keys); + free(used); + return count; +} + +/* Encodes the NHF3 payload in the requested mode; *palette tells which mode won. */ +static uint8_t *encode_payload(const uint8_t *pixels, uint32_t w, uint32_t h, int ch, + phf_mode_t mode, int effort, size_t *out_size, int *palette) { + int try_med = mode != PHF_MODE_PALETTE, try_pal = mode != PHF_MODE_MED; + if (mode == PHF_MODE_AUTO) { + uint32_t limit = effort == NHF3_EFFORT_FAST ? PHF_AUTO_PALETTE_FAST : PHF_AUTO_PALETTE_TRY; + try_pal = count_colors_upto(pixels, w * h, ch, limit) <= limit; + if (try_pal && effort == NHF3_EFFORT_FAST) try_med = 0; + } + size_t sm = 0, sp = 0; + uint8_t *med = try_med ? nhf3_encode(pixels, (int)w, (int)h, ch, ch, NULL, NHF3_PRED_MED, 1, effort, &sm, NULL) : NULL; + uint8_t *pal = try_pal ? nhf3_encode(pixels, (int)w, (int)h, ch, ch, NULL, NHF3_PRED_PIXEL, 0, effort, &sp, NULL) : NULL; + if (pal && (!med || sp < sm)) { + nhf3_free(med); + *palette = 1; + *out_size = sp; + return pal; + } + nhf3_free(pal); + *palette = 0; + *out_size = sm; + return med; +} + +PHF_API uint8_t* phf_encode_memory(const uint8_t *pixels, uint32_t width, uint32_t height, uint8_t channels, + const phf_encode_options_t *options, size_t *out_size) { if (!pixels || width == 0 || height == 0 || (channels != 3 && channels != 4) || !out_size) { return NULL; } if (width > PHF_MAX_DIM || height > PHF_MAX_DIM || (uint64_t)width * height > PHF_MAX_PIXELS) { return NULL; } + phf_mode_t mode = options ? options->mode : PHF_MODE_AUTO; + phf_effort_t effort = options ? options->effort : PHF_EFFORT_BEST; + if ((mode != PHF_MODE_AUTO && mode != PHF_MODE_MED && mode != PHF_MODE_PALETTE) + || (effort != PHF_EFFORT_BEST && effort != PHF_EFFORT_FAST)) { + return NULL; + } size_t payload_sz = 0; - int decorrelate = 1; /* channels is always 3 or 4 here */ - uint8_t *payload = nhf3_encode(pixels, (int)width, (int)height, (int)channels, (int)channels, NULL, NHF3_PRED_MED, decorrelate, &payload_sz, NULL); + int palette = 0; + uint8_t *payload = encode_payload(pixels, width, height, channels, mode, + effort == PHF_EFFORT_FAST ? NHF3_EFFORT_FAST : NHF3_EFFORT_BEST, + &payload_sz, &palette); if (!payload) return NULL; if (payload_sz > 0xFFFFFFFFu) { nhf3_free(payload); return NULL; } @@ -1487,8 +1598,8 @@ PHF_API uint8_t* phf_encode_memory(const uint8_t *pixels, uint32_t width, uint32 memcpy(phf_buf, "PHF1", 4); phf_buf[4] = (uint8_t)PHF_VERSION; phf_buf[5] = channels; - phf_buf[6] = (uint8_t)((channels == 4 ? PHF_FLAG_ALPHA : 0) | PHF_FLAG_MED - | (decorrelate ? PHF_FLAG_DECORR : 0) | PHF_FLAG_CRC32); + phf_buf[6] = (uint8_t)((channels == 4 ? PHF_FLAG_ALPHA : 0) | PHF_FLAG_CRC32 + | (palette ? 0 : PHF_FLAG_MED | PHF_FLAG_DECORR)); phf_buf[7] = 0; /* reserved */ put_le32(phf_buf + 8, width); put_le32(phf_buf + 12, height); @@ -1536,10 +1647,11 @@ PHF_API int phf_decode_memory(const uint8_t *phf_data, size_t phf_size, uint8_t return phf_map_nhf3_error(nhf3_decode(payload, info.payload_size, dst_pixels, (int)width, (int)height, (int)stride)); } -PHF_API int phf_write_file(const char *filepath, const uint8_t *pixels, uint32_t width, uint32_t height, uint8_t channels) { +PHF_API int phf_write_file(const char *filepath, const uint8_t *pixels, uint32_t width, uint32_t height, + uint8_t channels, const phf_encode_options_t *options) { if (!filepath || !pixels) return PHF_ERR_ARG; size_t out_sz = 0; - uint8_t *phf_bytes = phf_encode_memory(pixels, width, height, channels, &out_sz); + uint8_t *phf_bytes = phf_encode_memory(pixels, width, height, channels, options, &out_sz); if (!phf_bytes) return PHF_ERR_ARG; FILE *fp = fopen(filepath, "wb"); diff --git a/libphf/src/phf_simd.h b/libphf/src/phf_simd.h index 37c68a0..ce6d666 100644 --- a/libphf/src/phf_simd.h +++ b/libphf/src/phf_simd.h @@ -176,6 +176,41 @@ static inline void nhf3_spatial_inverse_med(uint8_t *buf, int w, int h) { } } +/* Three planes in one pass: their serial dependency chains are independent, so an + * out-of-order core overlaps them (about 2x faster than three separate passes on + * Apple M3; little gain on in-order cores). Same output as nhf3_spatial_inverse_med. */ +#define NHF3_MED_STEP(row, prow, a, x) \ + do { \ + int b_ = (prow)[x], c_ = (prow)[(x) - 1]; \ + int mn_ = (a) < b_ ? (a) : b_; \ + int mx_ = (a) ^ b_ ^ mn_; \ + int g_ = (a) + b_ - c_; \ + g_ = g_ < mn_ ? mn_ : g_; \ + g_ = g_ > mx_ ? mx_ : g_; \ + (a) = (uint8_t)((row)[x] + g_); \ + (row)[x] = (uint8_t)(a); \ + } while (0) + +static inline void nhf3_spatial_inverse_med3(uint8_t *p0, uint8_t *p1, uint8_t *p2, int w, int h) { + if (w <= 0 || h <= 0) return; + for (int x = 1; x < w; x++) { + p0[x] = (uint8_t)(p0[x] + p0[x - 1]); + p1[x] = (uint8_t)(p1[x] + p1[x - 1]); + p2[x] = (uint8_t)(p2[x] + p2[x - 1]); + } + for (int y = 1; y < h; y++) { + uint8_t *r0 = p0 + (size_t)y * w, *r1 = p1 + (size_t)y * w, *r2 = p2 + (size_t)y * w; + const uint8_t *q0 = r0 - w, *q1 = r1 - w, *q2 = r2 - w; + int a0 = (uint8_t)(r0[0] + q0[0]), a1 = (uint8_t)(r1[0] + q1[0]), a2 = (uint8_t)(r2[0] + q2[0]); + r0[0] = (uint8_t)a0; r1[0] = (uint8_t)a1; r2[0] = (uint8_t)a2; + for (int x = 1; x < w; x++) { + NHF3_MED_STEP(r0, q0, a0, x); + NHF3_MED_STEP(r1, q1, a1, x); + NHF3_MED_STEP(r2, q2, a2, x); + } + } +} + static inline void nhf3_spatial_inverse_left(uint8_t *buf, int w, int h) { if (w <= 0 || h <= 0) return; for (int x = 1; x < w; x++) buf[x] = (uint8_t)(buf[x] + buf[x - 1]); diff --git a/tests/test_phf.c b/tests/test_phf.c index 6414d84..a6c78a3 100644 --- a/tests/test_phf.c +++ b/tests/test_phf.c @@ -40,6 +40,19 @@ static void fill(uint8_t *p, int w, int h, int ch, int kind) { } } +/* Every mode/effort combination; index 0 is the NULL default. */ +static const phf_encode_options_t OPTS[] = { + { PHF_MODE_AUTO, PHF_EFFORT_BEST }, { PHF_MODE_AUTO, PHF_EFFORT_FAST }, + { PHF_MODE_MED, PHF_EFFORT_BEST }, { PHF_MODE_MED, PHF_EFFORT_FAST }, + { PHF_MODE_PALETTE, PHF_EFFORT_BEST }, { PHF_MODE_PALETTE, PHF_EFFORT_FAST }, +}; +#define N_OPTS (sizeof(OPTS) / sizeof(OPTS[0])) +static const phf_encode_options_t MED_BEST = { PHF_MODE_MED, PHF_EFFORT_BEST }; +static const phf_encode_options_t PAL_BEST = { PHF_MODE_PALETTE, PHF_EFFORT_BEST }; + +/* NHF3 predictor of an encoded PHF file: 2 = MED, 3 = palette. */ +static int file_pred(const uint8_t *e) { return (e[PHF_HEADER_SIZE + 5] >> 1) & 3; } + static uint32_t le32(const uint8_t *p) { return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); } @@ -49,41 +62,102 @@ static void test_roundtrip(void) { {31,33}, {64,64}, {257,129}, {640,480} }; for (size_t si = 0; si < sizeof(sizes) / sizeof(sizes[0]); si++) for (int ch = 3; ch <= 4; ch++) - for (int k = 0; k < K_COUNT; k++) { - int w = sizes[si][0], h = sizes[si][1]; - size_t n = (size_t)w * h * ch; - uint8_t *img = malloc(n), *dec = malloc(n), *d4 = malloc((size_t)w * h * 4); - fill(img, w, h, ch, k); - size_t osz = 0; - uint8_t *enc = phf_encode_memory(img, (uint32_t)w, (uint32_t)h, (uint8_t)ch, &osz); - CHECK(enc != NULL, "encode %dx%d ch%d kind%d", w, h, ch, k); - if (!enc) { free(img); free(dec); free(d4); continue; } - - int rc = phf_decode_memory(enc, osz, dec, (uint32_t)w, (uint32_t)h, (uint8_t)ch); - CHECK(rc == PHF_OK && memcmp(dec, img, n) == 0, - "roundtrip %dx%d ch%d kind%d rc=%d", w, h, ch, k, rc); - - rc = phf_decode_memory(enc, osz, dec, (uint32_t)w, (uint32_t)h, 0); - CHECK(rc == PHF_OK && memcmp(dec, img, n) == 0, "stride 0 %dx%d ch%d", w, h, ch); - - rc = phf_decode_memory(enc, osz, d4, (uint32_t)w, (uint32_t)h, 4); - int ok = (rc == PHF_OK); - for (size_t i = 0; ok && i < (size_t)w * h; i++) - for (int c = 0; c < 4; c++) { - uint8_t want = c < ch ? img[i * ch + c] : 255; - if (d4[i * 4 + c] != want) { ok = 0; break; } - } - CHECK(ok, "stride 4 %dx%d ch%d kind%d rc=%d", w, h, ch, k, rc); - - phf_free(enc); free(img); free(dec); free(d4); - } + for (int k = 0; k < K_COUNT; k++) + for (size_t oi = 0; oi < N_OPTS; oi++) { + int w = sizes[si][0], h = sizes[si][1]; + size_t n = (size_t)w * h * ch; + uint8_t *img = malloc(n), *dec = malloc(n), *d4 = malloc((size_t)w * h * 4); + fill(img, w, h, ch, k); + size_t osz = 0; + uint8_t *enc = phf_encode_memory(img, (uint32_t)w, (uint32_t)h, (uint8_t)ch, + oi ? &OPTS[oi] : NULL, &osz); + CHECK(enc != NULL, "encode %dx%d ch%d kind%d opt%zu", w, h, ch, k, oi); + if (!enc) { free(img); free(dec); free(d4); continue; } + + int rc = phf_decode_memory(enc, osz, dec, (uint32_t)w, (uint32_t)h, (uint8_t)ch); + CHECK(rc == PHF_OK && memcmp(dec, img, n) == 0, + "roundtrip %dx%d ch%d kind%d opt%zu rc=%d", w, h, ch, k, oi, rc); + + rc = phf_decode_memory(enc, osz, dec, (uint32_t)w, (uint32_t)h, 0); + CHECK(rc == PHF_OK && memcmp(dec, img, n) == 0, "stride 0 %dx%d ch%d opt%zu", w, h, ch, oi); + + /* RGB into RGBA must come out opaque in every mode. */ + memset(d4, 0, (size_t)w * h * 4); + rc = phf_decode_memory(enc, osz, d4, (uint32_t)w, (uint32_t)h, 4); + int ok = (rc == PHF_OK); + for (size_t i = 0; ok && i < (size_t)w * h; i++) + for (int c = 0; c < 4; c++) { + uint8_t want = c < ch ? img[i * ch + c] : 255; + if (d4[i * 4 + c] != want) { ok = 0; break; } + } + CHECK(ok, "stride 4 %dx%d ch%d kind%d opt%zu rc=%d", w, h, ch, k, oi, rc); + + phf_free(enc); free(img); free(dec); free(d4); + } +} + +static void test_modes(void) { + enum { W = 96, H = 64 }; + uint8_t img[W * H * 4]; + size_t a, b; + + /* NULL and a zeroed struct are the same default. */ + fill(img, W, H, 3, K_GRADIENT); + phf_encode_options_t zero; + memset(&zero, 0, sizeof(zero)); + uint8_t *e1 = phf_encode_memory(img, W, H, 3, NULL, &a), *e2 = phf_encode_memory(img, W, H, 3, &zero, &b); + CHECK(e1 && e2 && a == b && memcmp(e1, e2, a) == 0, "NULL options == zeroed options"); + phf_free(e1); phf_free(e2); + + /* AUTO: 4 colours scattered at random -> palette (both efforts); noise -> MED. */ + static const uint8_t four[4][3] = { {250, 10, 10}, {10, 200, 30}, {20, 20, 230}, {240, 240, 240} }; + for (int fast = 0; fast <= 1; fast++) { + const phf_encode_options_t o = { PHF_MODE_AUTO, fast ? PHF_EFFORT_FAST : PHF_EFFORT_BEST }; + for (int i = 0; i < W * H; i++) memcpy(img + i * 3, four[rnd() % 4], 3); + uint8_t *e = phf_encode_memory(img, W, H, 3, &o, &a); + CHECK(e && file_pred(e) == 3, "AUTO fast=%d picks palette for 4 scattered colours", fast); + CHECK(e && e[6] == PHF_FLAG_CRC32, "palette flags = 0x%02x", e ? e[6] : 0); + phf_free(e); + fill(img, W, H, 3, K_NOISE); + e = phf_encode_memory(img, W, H, 3, &o, &a); + CHECK(e && file_pred(e) == 2, "AUTO fast=%d picks MED for noise", fast); + phf_free(e); + } + + /* AUTO + BEST never loses to either forced mode. */ + for (int k = 0; k < K_COUNT; k++) { + fill(img, W, H, 3, k); + size_t s_auto, s_med, s_pal; + const phf_encode_options_t pal = { PHF_MODE_PALETTE, PHF_EFFORT_BEST }; + uint8_t *x = phf_encode_memory(img, W, H, 3, NULL, &s_auto); + uint8_t *y = phf_encode_memory(img, W, H, 3, &MED_BEST, &s_med); + uint8_t *z = phf_encode_memory(img, W, H, 3, &pal, &s_pal); + CHECK(x && y && z && s_auto <= s_med && s_auto <= s_pal, "AUTO kind%d: %zu vs MED %zu / palette %zu", + k, s_auto, s_med, s_pal); + phf_free(x); phf_free(y); phf_free(z); + } + + /* RGBA palette keeps alpha and flags it. */ + fill(img, W, H, 4, K_XOR_ALPHA); + uint8_t *e = phf_encode_memory(img, W, H, 4, &PAL_BEST, &a); + uint8_t out[W * H * 4]; + CHECK(e && file_pred(e) == 3 && e[6] == (PHF_FLAG_ALPHA | PHF_FLAG_CRC32), "RGBA palette header"); + CHECK(e && phf_decode_memory(e, a, out, W, H, 4) == PHF_OK && memcmp(out, img, sizeof(out)) == 0, + "RGBA palette roundtrip"); + phf_free(e); + + /* Invalid options are rejected. */ + phf_encode_options_t bad = { (phf_mode_t)7, PHF_EFFORT_BEST }; + CHECK(phf_encode_memory(img, W, H, 3, &bad, &a) == NULL, "invalid mode"); + bad.mode = PHF_MODE_MED; bad.effort = (phf_effort_t)9; + CHECK(phf_encode_memory(img, W, H, 3, &bad, &a) == NULL, "invalid effort"); } static void test_header(void) { uint8_t img[8 * 8 * 4]; fill(img, 8, 8, 4, K_GRADIENT); size_t osz; - uint8_t *e3 = phf_encode_memory(img, 8, 8, 3, &osz); + uint8_t *e3 = phf_encode_memory(img, 8, 8, 3, &MED_BEST, &osz); CHECK(memcmp(e3, "PHF1", 4) == 0 && e3[4] == PHF_VERSION && e3[5] == 3, "magic/version/channels"); CHECK(e3[6] == (PHF_FLAG_MED | PHF_FLAG_DECORR | PHF_FLAG_CRC32), "RGB flags = 0x%02x", e3[6]); CHECK(e3[7] == 0, "reserved byte"); @@ -97,7 +171,7 @@ static void test_header(void) { CHECK(phf_probe_memory(NULL, 20, &info) == PHF_ERR_ARG, "probe NULL"); phf_free(e3); - uint8_t *e4 = phf_encode_memory(img, 8, 8, 4, &osz); + uint8_t *e4 = phf_encode_memory(img, 8, 8, 4, &MED_BEST, &osz); CHECK(e4[6] == (PHF_FLAG_ALPHA | PHF_FLAG_MED | PHF_FLAG_DECORR | PHF_FLAG_CRC32), "RGBA flags = 0x%02x", e4[6]); phf_free(e4); } @@ -105,14 +179,14 @@ static void test_header(void) { static void test_limits_and_errors(void) { uint8_t px[4] = { 0 }, out[64 * 64 * 4]; size_t osz; - CHECK(phf_encode_memory(px, 0, 1, 3, &osz) == NULL, "width 0"); - CHECK(phf_encode_memory(px, 1, 1, 2, &osz) == NULL, "channels 2"); - CHECK(phf_encode_memory(px, PHF_MAX_DIM + 1, 1, 3, &osz) == NULL, "width > 65535"); - CHECK(phf_encode_memory(px, 1, 1, 3, NULL) == NULL, "NULL out_size"); + CHECK(phf_encode_memory(px, 0, 1, 3, NULL, &osz) == NULL, "width 0"); + CHECK(phf_encode_memory(px, 1, 1, 2, NULL, &osz) == NULL, "channels 2"); + CHECK(phf_encode_memory(px, PHF_MAX_DIM + 1, 1, 3, NULL, &osz) == NULL, "width > 65535"); + CHECK(phf_encode_memory(px, 1, 1, 3, NULL, NULL) == NULL, "NULL out_size"); uint8_t img[64 * 64 * 3]; fill(img, 64, 64, 3, K_GRADIENT); - uint8_t *e = phf_encode_memory(img, 64, 64, 3, &osz); + uint8_t *e = phf_encode_memory(img, 64, 64, 3, &MED_BEST, &osz); uint8_t *m = malloc(osz); CHECK(phf_decode_memory(e, osz, out, 63, 64, 3) == PHF_ERR_SIZE, "dimension mismatch"); @@ -152,10 +226,11 @@ static void test_fuzz(void) { uint8_t img[W * H * 4], dst[W * H * 4]; int accepted_crc = 0, total = 0; for (int kind = 0; kind < K_COUNT; kind++) - for (int ch = 3; ch <= 4; ch++) { + for (int ch = 3; ch <= 4; ch++) + for (int pal = 0; pal <= 1; pal++) { fill(img, W, H, ch, kind); size_t osz; - uint8_t *e = phf_encode_memory(img, W, H, (uint8_t)ch, &osz); + uint8_t *e = phf_encode_memory(img, W, H, (uint8_t)ch, pal ? &PAL_BEST : &MED_BEST, &osz); uint8_t *m = malloc(osz); for (int it = 0; it < ITERS; it++) { memcpy(m, e, osz); @@ -191,7 +266,7 @@ static void test_file_io(void) { uint8_t img[33 * 17 * 4]; fill(img, 33, 17, 4, K_XOR_ALPHA); const char *path = "test_phf_tmp.phf"; - CHECK(phf_write_file(path, img, 33, 17, 4) == PHF_OK, "phf_write_file"); + CHECK(phf_write_file(path, img, 33, 17, 4, NULL) == PHF_OK, "phf_write_file"); phf_info_t info; uint8_t *px = phf_read_file(path, &info); CHECK(px && info.width == 33 && info.height == 17 && info.channels == 4 && memcmp(px, img, sizeof(img)) == 0, @@ -203,6 +278,7 @@ static void test_file_io(void) { int main(int argc, char **argv) { test_roundtrip(); + test_modes(); test_header(); test_limits_and_errors(); test_fuzz();