feat(ota): Download gzip-compressed app images when the release has one

.changes/ota-gzip.md

@@ -0,0 +1,9 @@
+---
+kind: added
+---
+Download gzip-compressed app images during OTA updates
+
+## Release Note
+Firmware updates download about a third less when the release has a gzip copy of the app binary, which helps most on slow WiFi. Releases without one update as before.
+
+- Tries the app binary URL + `.gz` first (gzip with a 4 KB window, `wbits` 12, decompressed with espressif/zlib) and falls back to the plain image on any failure; `OPENSHOCK_OTA_GZIP` turns it off

components/ota/CMakeLists.txt

@@ -8,13 +8,17 @@
 idf_component_register(
   SRCS          "src/OtaUpdateManager.cpp"
                 "src/PartitionUtils.cpp"
+                "src/GzipInflater.cpp"
   INCLUDE_DIRS  "include"
   REQUIRES      common                                          # enums/FirmwareBootType.h, enums/OtaUpdateChannel.h, SemVer.h in public header
   PRIV_REQUIRES core captive_portal config                      # gateway/wifi/WSGateway/AppHooks; CaptivePortal::ForceClose; Config::
                 crypto http logging osjson temporal             # Hashing, HTTPRequestManager, json
                 www                                             # the www partition the staticfs artifact goes into
                 app_update esp_event esp_netif esp_wifi         # esp_ota_ops.h, WiFi/IP event handlers
                 esp_partition                                   # flashing partitions from a URL
+                heap                                            # heap_caps (the gzip decoder's memory, free-block check)
+                espressif__zlib                                 # gzip app images; always required (requirements can't
+                                                                # depend on Kconfig), linked only with OPENSHOCK_OTA_GZIP
 )
 
 # Match core: IDF compiles application components with -Werror, the legacy build did not.

components/ota/Kconfig

@@ -0,0 +1,20 @@
+menu "OpenShock OTA"
+
+    config OPENSHOCK_OTA_GZIP
+        bool "Try the gzip app image (the binary URL + .gz) first"
+        default y
+        help
+            Decompressed with zlib (espressif/zlib), compressed with a 4 KB window. The decoder and its
+            window are allocated before the downloads start; without the memory the plain image is
+            downloaded instead.
+
+    config OPENSHOCK_OTA_GZIP_MIN_FREE_BLOCK
+        int "Largest free block to keep after allocating the gzip decoder (bytes)"
+        range 16384 131072
+        default 40960
+        help
+            The decoder is freed again, and the plain image downloaded, unless this much contiguous
+            internal RAM is left for the TLS connections to the download host. Unused without
+            OPENSHOCK_OTA_GZIP; not hidden behind it, so the decoder compiles either way.
+
+endmenu

components/ota/idf_component.yml

@@ -0,0 +1,3 @@
+# zlib decompresses gzip app images during OTA (src/GzipInflater.cpp).
+dependencies:
+  espressif/zlib: "^1.3.2"

components/ota/include/OtaUpdateManager.h

@@ -19,8 +19,8 @@ namespace OpenShock::OtaUpdateManager {
 
   struct FirmwareRelease {
     std::string appBinaryUrl;
-    uint8_t appBinaryHash[32];
-    std::string wwwBinaryUrl;  // the `staticfs` artifact: the www image (www.bin)
+    uint8_t appBinaryHash[32];  // of the image, so it also checks what `appBinaryUrl` + ".gz" decompresses to
+    std::string wwwBinaryUrl;   // the `staticfs` artifact: the www image (www.bin)
     uint8_t wwwBinaryHash[32];
   };
 

components/ota/src/GzipInflater.cpp

@@ -0,0 +1,227 @@
+#include <freertos/FreeRTOS.h>
+
+#include "GzipInflater.h"
+
+const char* const TAG = "GzipInflater";
+
+#include "Logging.h"
+
+#include <esp_heap_caps.h>
+#include <zlib.h>
+
+#include <new>
+
+using namespace OpenShock;
+
+// zlib's inflate allocates its state (struct inflate_state) at init and its window on the first output. Both come
+// out of the arena, so the arena must hold both: the window exactly, the state with some headroom since its size is
+// internal to zlib. The constructor checks the window still fits after init, so a short arena fails up front rather
+// than mid-download.
+static constexpr std::size_t kWindowSize     = std::size_t {1} << GzipInflater::kWindowBits;
+static constexpr std::size_t kStateReserve   = 8 * 1024;
+static constexpr std::size_t kArenaSize      = kStateReserve + kWindowSize;
+static constexpr std::size_t kArenaAlignment = alignof(std::max_align_t);
+
+struct GzipInflater::Storage {
+  z_stream stream;
+  std::size_t arenaUsed;
+  std::size_t outputUsed;
+  uint8_t output[kOutputSize];
+  alignas(kArenaAlignment) uint8_t arena[kArenaSize];
+};
+
+// Bump allocation out of the arena; zfree is a no-op and the whole arena goes back in release().
+void* GzipInflater::zalloc(void* opaque, unsigned items, unsigned size)
+{
+  auto* storage = static_cast<Storage*>(opaque);
+
+  if (size != 0 && items > SIZE_MAX / size) {
+    return Z_NULL;
+  }
+  std::size_t length = static_cast<std::size_t>(items) * size;
+  std::size_t start  = (storage->arenaUsed + kArenaAlignment - 1) & ~(kArenaAlignment - 1);
+  if (start > kArenaSize || length > kArenaSize - start) {
+    OS_LOGE(TAG, "zlib asked for %zu bytes, %zu of %zu left", length, start > kArenaSize ? 0 : kArenaSize - start, kArenaSize);
+    return Z_NULL;
+  }
+
+  storage->arenaUsed = start + length;
+  return storage->arena + start;
+}
+
+void GzipInflater::zfree(void* /*opaque*/, void* /*address*/)
+{
+}
+
+GzipInflater::GzipInflater() noexcept
+  : m_storage(nullptr)
+  , m_done(false)
+  , m_failed(true)
+{
+  if constexpr (kEnabled) {
+    allocate();
+  }
+}
+
+void GzipInflater::allocate() noexcept
+{
+  void* memory = heap_caps_malloc(sizeof(Storage), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
+  if (memory == nullptr) {
+    OS_LOGW(TAG, "Failed to allocate %zu bytes for the gzip decoder", sizeof(Storage));
+    return;
+  }
+
+  Storage* storage       = new (memory) Storage;
+  storage->arenaUsed     = 0;
+  storage->outputUsed    = 0;
+  storage->stream        = {};
+  storage->stream.zalloc = &GzipInflater::zalloc;
+  storage->stream.zfree  = &GzipInflater::zfree;
+  storage->stream.opaque = storage;
+
+  // 16 + wbits: a gzip wrapper (header, CRC-32 and size trailer), no zlib or raw stream
+  int ret = inflateInit2(&storage->stream, 16 + kWindowBits);
+  if (ret != Z_OK) {
+    OS_LOGE(TAG, "inflateInit2 failed (%d)", ret);
+    storage->~Storage();
+    heap_caps_free(memory);
+    return;
+  }
+
+  // The window is allocated on the first output; make sure it will fit
+  std::size_t windowStart = (storage->arenaUsed + kArenaAlignment - 1) & ~(kArenaAlignment - 1);
+  if (windowStart > kArenaSize || kArenaSize - windowStart < kWindowSize) {
+    OS_LOGE(TAG, "zlib's state took %zu bytes, no room left for its %zu byte window", storage->arenaUsed, kWindowSize);
+    inflateEnd(&storage->stream);
+    storage->~Storage();
+    heap_caps_free(memory);
+    return;
+  }
+
+  m_storage = storage;
+  m_failed  = false;
+
+  // Only worth holding if the TLS connections that follow still fit
+  std::size_t largestFree = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
+  if (largestFree < CONFIG_OPENSHOCK_OTA_GZIP_MIN_FREE_BLOCK) {
+    OS_LOGW(TAG, "Only %zu bytes contiguous left after the gzip decoder, not keeping it", largestFree);
+    release();
+    return;
+  }
+
+  OS_LOGI(TAG, "Allocated %zu bytes for the gzip decoder, %zu bytes contiguous left", sizeof(Storage), largestFree);
+}
+
+GzipInflater::~GzipInflater()
+{
+  release();
+}
+
+void GzipInflater::release() noexcept
+{
+  // Discarded without gzip (it never allocated), which keeps inflateEnd out of the link
+  if constexpr (kEnabled) {
+    if (m_storage == nullptr) {
+      return;
+    }
+    inflateEnd(&m_storage->stream);
+    m_storage->~Storage();
+    heap_caps_free(m_storage);
+    m_storage = nullptr;
+    m_done    = false;
+    m_failed  = true;
+  }
+}
+
+std::size_t GzipInflater::AllocationSize() noexcept
+{
+  return sizeof(Storage);
+}
+
+bool GzipInflater::reset() noexcept
+{
+  if (m_storage == nullptr) {
+    return false;
+  }
+
+  // Keeps the state and window zlib already allocated
+  int ret = inflateReset(&m_storage->stream);
+  if (ret != Z_OK) {
+    OS_LOGE(TAG, "inflateReset failed (%d)", ret);
+    m_failed = true;
+    return false;
+  }
+
+  m_storage->outputUsed = 0;
+  m_done                = false;
+  m_failed              = false;
+  return true;
+}
+
+GzipInflater::Result GzipInflater::feed(std::span<const uint8_t> input, Sink sink) noexcept
+{
+  if (m_storage == nullptr || m_failed) {
+    return Result::Error;
+  }
+
+  if (m_done) {
+    if (input.empty()) {
+      return Result::Done;
+    }
+    // A second member (concatenated gzip) or junk; neither is something the release tooling produces
+    OS_LOGE(TAG, "Data after the end of the gzip stream");
+    m_failed = true;
+    return Result::Error;
+  }
+
+  z_stream& stream = m_storage->stream;
+  stream.next_in   = const_cast<Bytef*>(input.data());  // zlib's API is not const-correct; it only reads the input
+  stream.avail_in  = static_cast<uInt>(input.size());
+
+  while (true) {
+    std::size_t& outputUsed = m_storage->outputUsed;
+    stream.next_out         = m_storage->output + outputUsed;
+    stream.avail_out        = static_cast<uInt>(kOutputSize - outputUsed);
+
+    int ret    = inflate(&stream, Z_NO_FLUSH);
+    outputUsed = kOutputSize - stream.avail_out;
+
+    // Z_BUF_ERROR only means no progress was possible this call, checked below
+    if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) {
+      OS_LOGE(TAG, "Failed to decompress (%d: %s)", ret, stream.msg != nullptr ? stream.msg : "no message");
+      m_failed = true;
+      return Result::Error;
+    }
+
+    bool outputFull = stream.avail_out == 0;
+    if (outputFull || (ret == Z_STREAM_END && outputUsed > 0)) {
+      if (!sink(std::span<const uint8_t>(m_storage->output, outputUsed))) {
+        m_failed = true;
+        return Result::Error;
+      }
+      outputUsed = 0;
+    }
+
+    if (ret == Z_STREAM_END) {
+      if (stream.avail_in != 0) {
+        OS_LOGE(TAG, "Data after the end of the gzip stream");
+        m_failed = true;
+        return Result::Error;
+      }
+      m_done = true;
+      return Result::Done;
+    }
+
+    // A full buffer may have left output pending in zlib, so go round again even when the input is used up
+    if (!outputFull && stream.avail_in == 0) {
+      return Result::NeedMoreInput;
+    }
+
+    if (ret == Z_BUF_ERROR && !outputFull) {
+      // Room for output and input left, yet no progress: not a state a well-formed stream gets zlib into
+      OS_LOGE(TAG, "zlib made no progress");
+      m_failed = true;
+      return Result::Error;
+    }
+  }
+}

components/ota/src/GzipInflater.h

@@ -0,0 +1,92 @@
+#pragma once
+
+#include "util/FunctionRef.h"
+
+#include <sdkconfig.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <span>
+
+namespace OpenShock {
+  /// @brief Streaming gzip decoder: zlib's inflate, with all of its memory allocated up front.
+  ///
+  /// The stream must be compressed with a window of at most kWindowBits (gzip/zlib `wbits` <= 12, a 4 KB window);
+  /// zlib rejects a stream that refers back further than that ("invalid distance too far back").
+  ///
+  /// Construction allocates one block that zlib's allocations are served from (its state and window) plus the
+  /// output buffer, and keeps it only if CONFIG_OPENSHOCK_OTA_GZIP_MIN_FREE_BLOCK of contiguous internal RAM is left
+  /// for the TLS connections that follow. So the caller learns whether the memory is there before it asks for a
+  /// compressed download. release() gives it back early, e.g. when the server did not send gzip. The caller owns the
+  /// lifetime; nothing is pooled.
+  ///
+  /// Without CONFIG_OPENSHOCK_OTA_GZIP it never allocates and is never valid. Guard the gzip paths with
+  /// `if constexpr (GzipInflater::kEnabled)`: they stay compiled and type-checked, but emit no code, so nothing
+  /// reaches zlib and the linker drops it.
+  ///
+  /// Not thread-safe.
+  class GzipInflater {
+  public:
+    // The one #if: Kconfig leaves a disabled bool undefined, so it can't be used in `if constexpr` directly
+#if CONFIG_OPENSHOCK_OTA_GZIP
+    static constexpr bool kEnabled = true;
+#else
+    static constexpr bool kEnabled = false;
+#endif
+
+    static constexpr int kWindowBits         = 12;
+    static constexpr std::size_t kOutputSize = 4096;  // the sink gets pieces of this size, the last one shorter
+
+    /// @brief Receives decompressed output in order. Returning false aborts decoding.
+    using Sink = Util::FunctionRef<bool(std::span<const uint8_t>)>;
+
+    enum class Result : uint8_t {
+      NeedMoreInput,  // all input consumed, the stream is not finished
+      Done,           // the stream ended and zlib verified its CRC-32 and size
+      Error,          // malformed stream, trailing data, or the sink refused
+    };
+
+    /// @brief Whether a body starting with this byte is gzip (the first magic byte). A firmware image starts with
+    ///        0xE9, so one byte tells the two apart; a body that starts like gzip but is not fails in feed().
+    [[nodiscard]] static constexpr bool StartsLikeGzip(uint8_t firstByte) noexcept { return firstByte == 0x1F; }
+
+    GzipInflater(const GzipInflater&)            = delete;
+    GzipInflater& operator=(const GzipInflater&) = delete;
+    GzipInflater(GzipInflater&&)                 = delete;
+    GzipInflater& operator=(GzipInflater&&)      = delete;
+
+    GzipInflater() noexcept;
+    ~GzipInflater();
+
+    /// @brief Whether the memory is held, i.e. the constructor succeeded and release() was not called.
+    [[nodiscard]] bool valid() const noexcept { return m_storage != nullptr; }
+
+    /// @brief Frees the memory now instead of at destruction. The inflater is unusable afterwards.
+    void release() noexcept;
+
+    /// @brief Bytes held by a valid inflater.
+    [[nodiscard]] static std::size_t AllocationSize() noexcept;
+
+    /// @brief Starts a new stream. Must be called before the first feed() of every stream.
+    [[nodiscard]] bool reset() noexcept;
+
+    /// @brief Decodes the next piece of the compressed stream. After Error every call returns Error, and after Done
+    ///        any further input does, until reset().
+    [[nodiscard]] Result feed(std::span<const uint8_t> input, Sink sink) noexcept;
+
+    /// @brief Whether the stream has ended and verified.
+    [[nodiscard]] bool finished() const noexcept { return m_done; }
+
+  private:
+    struct Storage;
+
+    void allocate() noexcept;
+
+    static void* zalloc(void* opaque, unsigned items, unsigned size);
+    static void zfree(void* opaque, void* address);
+
+    Storage* m_storage;
+    bool m_done;
+    bool m_failed;
+  };
+}  // namespace OpenShock