From e38c73cfe40c5c2e6ea7ed296a56b444ca58b1cd Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Mon, 9 Nov 2026 15:27:35 +0100 Subject: [PATCH 1/2] feat(ota): Download gzip-compressed app images when the release has one Try the app binary URL + ".gz" first and fall back to the plain image on any failure (a 404 for releases without one, a bad stream, a hash mismatch). - GzipInflater wraps espressif/zlib's inflate; one block of internal RAM holds zlib's state and window (4 KB, wbits 12) and the output buffer, allocated before the downloads connect and kept only if OPENSHOCK_OTA_GZIP_MIN_FREE_BLOCK is left for TLS - A body that isn't gzip is flashed as is and the decoder released; the SHA-256 is checked on the decompressed image, and writes past the end of the partition are refused - OPENSHOCK_OTA_GZIP turns it off: one #if for GzipInflater::kEnabled, the gzip paths behind if constexpr, and zlib is garbage-collected out of the link --- .changes/ota-gzip.md | 9 + components/ota/CMakeLists.txt | 4 + components/ota/Kconfig | 20 ++ components/ota/idf_component.yml | 3 + components/ota/include/OtaUpdateManager.h | 2 +- components/ota/src/GzipInflater.cpp | 225 ++++++++++++++++++++++ components/ota/src/GzipInflater.h | 92 +++++++++ components/ota/src/OtaUpdateManager.cpp | 41 +++- components/ota/src/PartitionUtils.cpp | 103 ++++++++-- components/ota/src/PartitionUtils.h | 13 +- dependencies.lock | 13 +- 11 files changed, 499 insertions(+), 26 deletions(-) create mode 100644 .changes/ota-gzip.md create mode 100644 components/ota/Kconfig create mode 100644 components/ota/idf_component.yml create mode 100644 components/ota/src/GzipInflater.cpp create mode 100644 components/ota/src/GzipInflater.h diff --git a/.changes/ota-gzip.md b/.changes/ota-gzip.md new file mode 100644 index 00000000..6fbe3bcf --- /dev/null +++ b/.changes/ota-gzip.md @@ -0,0 +1,9 @@ +--- +kind: added +--- +Download gzip-compressed app images during OTA updates + +## Release Note +Firmware updates download about a third less when the release has a gzip copy of the app binary, which helps most on slow WiFi. Releases without one update as before. + +- Tries the app binary URL + `.gz` first (gzip with a 4 KB window, `wbits` 12, decompressed with espressif/zlib) and falls back to the plain image on any failure; `OPENSHOCK_OTA_GZIP` turns it off diff --git a/components/ota/CMakeLists.txt b/components/ota/CMakeLists.txt index 313d12ea..0a64c501 100644 --- a/components/ota/CMakeLists.txt +++ b/components/ota/CMakeLists.txt @@ -8,6 +8,7 @@ idf_component_register( SRCS "src/OtaUpdateManager.cpp" "src/PartitionUtils.cpp" + "src/GzipInflater.cpp" INCLUDE_DIRS "include" REQUIRES common # enums/FirmwareBootType.h, enums/OtaUpdateChannel.h, SemVer.h in public header PRIV_REQUIRES core captive_portal config # gateway/wifi/WSGateway/AppHooks; CaptivePortal::ForceClose; Config:: @@ -15,6 +16,9 @@ idf_component_register( www # the www partition the staticfs artifact goes into app_update esp_event esp_netif esp_wifi # esp_ota_ops.h, WiFi/IP event handlers esp_partition # flashing partitions from a URL + heap # heap_caps (the gzip decoder's memory, free-block check) + espressif__zlib # gzip app images; always required (requirements can't + # depend on Kconfig), linked only with OPENSHOCK_OTA_GZIP ) # Match core: IDF compiles application components with -Werror, the legacy build did not. diff --git a/components/ota/Kconfig b/components/ota/Kconfig new file mode 100644 index 00000000..304afe40 --- /dev/null +++ b/components/ota/Kconfig @@ -0,0 +1,20 @@ +menu "OpenShock OTA" + + config OPENSHOCK_OTA_GZIP + bool "Try the gzip app image (the binary URL + .gz) first" + default y + help + Decompressed with zlib (espressif/zlib), compressed with a 4 KB window. The decoder and its + window are allocated before the downloads start; without the memory the plain image is + downloaded instead. + + config OPENSHOCK_OTA_GZIP_MIN_FREE_BLOCK + int "Largest free block to keep after allocating the gzip decoder (bytes)" + range 16384 131072 + default 40960 + help + The decoder is freed again, and the plain image downloaded, unless this much contiguous + internal RAM is left for the TLS connections to the download host. Unused without + OPENSHOCK_OTA_GZIP; not hidden behind it, so the decoder compiles either way. + +endmenu diff --git a/components/ota/idf_component.yml b/components/ota/idf_component.yml new file mode 100644 index 00000000..e0598c73 --- /dev/null +++ b/components/ota/idf_component.yml @@ -0,0 +1,3 @@ +# zlib decompresses gzip app images during OTA (src/GzipInflater.cpp). +dependencies: + espressif/zlib: "^1.3.2" diff --git a/components/ota/include/OtaUpdateManager.h b/components/ota/include/OtaUpdateManager.h index a896ae39..26160de4 100644 --- a/components/ota/include/OtaUpdateManager.h +++ b/components/ota/include/OtaUpdateManager.h @@ -19,7 +19,7 @@ namespace OpenShock::OtaUpdateManager { struct FirmwareRelease { std::string appBinaryUrl; - uint8_t appBinaryHash[32]; + uint8_t appBinaryHash[32]; // of the image, so it also checks what `appBinaryUrl` + ".gz" decompresses to std::string wwwBinaryUrl; // the `staticfs` artifact: the www image (www.bin) uint8_t wwwBinaryHash[32]; }; diff --git a/components/ota/src/GzipInflater.cpp b/components/ota/src/GzipInflater.cpp new file mode 100644 index 00000000..87c23838 --- /dev/null +++ b/components/ota/src/GzipInflater.cpp @@ -0,0 +1,225 @@ +#include + +#include "GzipInflater.h" + +const char* const TAG = "GzipInflater"; + +#include "Logging.h" + +#include +#include + +#include + +using namespace OpenShock; + +// zlib's inflate allocates its state (struct inflate_state) at init and its window on the first output. Both come +// out of the arena, so the arena must hold both: the window exactly, the state with some headroom since its size is +// internal to zlib. The constructor checks the window still fits after init, so a short arena fails up front rather +// than mid-download. +static constexpr std::size_t kWindowSize = std::size_t {1} << GzipInflater::kWindowBits; +static constexpr std::size_t kStateReserve = 8 * 1024; +static constexpr std::size_t kArenaSize = kStateReserve + kWindowSize; +static constexpr std::size_t kArenaAlignment = alignof(std::max_align_t); + +struct GzipInflater::Storage { + z_stream stream; + std::size_t arenaUsed; + std::size_t outputUsed; + uint8_t output[kOutputSize]; + alignas(kArenaAlignment) uint8_t arena[kArenaSize]; +}; + +// Bump allocation out of the arena; zfree is a no-op and the whole arena goes back in release(). +void* GzipInflater::zalloc(void* opaque, unsigned items, unsigned size) +{ + auto* storage = static_cast(opaque); + + if (size != 0 && items > SIZE_MAX / size) { + return Z_NULL; + } + std::size_t length = static_cast(items) * size; + std::size_t start = (storage->arenaUsed + kArenaAlignment - 1) & ~(kArenaAlignment - 1); + if (start > kArenaSize || length > kArenaSize - start) { + OS_LOGE(TAG, "zlib asked for %zu bytes, %zu of %zu left", length, start > kArenaSize ? 0 : kArenaSize - start, kArenaSize); + return Z_NULL; + } + + storage->arenaUsed = start + length; + return storage->arena + start; +} + +void GzipInflater::zfree(void* /*opaque*/, void* /*address*/) { } + +GzipInflater::GzipInflater() noexcept + : m_storage(nullptr) + , m_done(false) + , m_failed(true) +{ + if constexpr (kEnabled) { + allocate(); + } +} + +void GzipInflater::allocate() noexcept +{ + void* memory = heap_caps_malloc(sizeof(Storage), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (memory == nullptr) { + OS_LOGW(TAG, "Failed to allocate %zu bytes for the gzip decoder", sizeof(Storage)); + return; + } + + Storage* storage = new (memory) Storage; + storage->arenaUsed = 0; + storage->outputUsed = 0; + storage->stream = {}; + storage->stream.zalloc = &GzipInflater::zalloc; + storage->stream.zfree = &GzipInflater::zfree; + storage->stream.opaque = storage; + + // 16 + wbits: a gzip wrapper (header, CRC-32 and size trailer), no zlib or raw stream + int ret = inflateInit2(&storage->stream, 16 + kWindowBits); + if (ret != Z_OK) { + OS_LOGE(TAG, "inflateInit2 failed (%d)", ret); + storage->~Storage(); + heap_caps_free(memory); + return; + } + + // The window is allocated on the first output; make sure it will fit + std::size_t windowStart = (storage->arenaUsed + kArenaAlignment - 1) & ~(kArenaAlignment - 1); + if (windowStart > kArenaSize || kArenaSize - windowStart < kWindowSize) { + OS_LOGE(TAG, "zlib's state took %zu bytes, no room left for its %zu byte window", storage->arenaUsed, kWindowSize); + inflateEnd(&storage->stream); + storage->~Storage(); + heap_caps_free(memory); + return; + } + + m_storage = storage; + m_failed = false; + + // Only worth holding if the TLS connections that follow still fit + std::size_t largestFree = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (largestFree < CONFIG_OPENSHOCK_OTA_GZIP_MIN_FREE_BLOCK) { + OS_LOGW(TAG, "Only %zu bytes contiguous left after the gzip decoder, not keeping it", largestFree); + release(); + return; + } + + OS_LOGI(TAG, "Allocated %zu bytes for the gzip decoder, %zu bytes contiguous left", sizeof(Storage), largestFree); +} + +GzipInflater::~GzipInflater() +{ + release(); +} + +void GzipInflater::release() noexcept +{ + // Discarded without gzip (it never allocated), which keeps inflateEnd out of the link + if constexpr (kEnabled) { + if (m_storage == nullptr) { + return; + } + inflateEnd(&m_storage->stream); + m_storage->~Storage(); + heap_caps_free(m_storage); + m_storage = nullptr; + m_done = false; + m_failed = true; + } +} + +std::size_t GzipInflater::AllocationSize() noexcept +{ + return sizeof(Storage); +} + +bool GzipInflater::reset() noexcept +{ + if (m_storage == nullptr) { + return false; + } + + // Keeps the state and window zlib already allocated + int ret = inflateReset(&m_storage->stream); + if (ret != Z_OK) { + OS_LOGE(TAG, "inflateReset failed (%d)", ret); + m_failed = true; + return false; + } + + m_storage->outputUsed = 0; + m_done = false; + m_failed = false; + return true; +} + +GzipInflater::Result GzipInflater::feed(std::span input, Sink sink) noexcept +{ + if (m_storage == nullptr || m_failed) { + return Result::Error; + } + + if (m_done) { + if (input.empty()) { + return Result::Done; + } + // A second member (concatenated gzip) or junk; neither is something the release tooling produces + OS_LOGE(TAG, "Data after the end of the gzip stream"); + m_failed = true; + return Result::Error; + } + + z_stream& stream = m_storage->stream; + stream.next_in = const_cast(input.data()); // zlib's API is not const-correct; it only reads the input + stream.avail_in = static_cast(input.size()); + + while (true) { + std::size_t& outputUsed = m_storage->outputUsed; + stream.next_out = m_storage->output + outputUsed; + stream.avail_out = static_cast(kOutputSize - outputUsed); + + int ret = inflate(&stream, Z_NO_FLUSH); + outputUsed = kOutputSize - stream.avail_out; + + // Z_BUF_ERROR only means no progress was possible this call, checked below + if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) { + OS_LOGE(TAG, "Failed to decompress (%d: %s)", ret, stream.msg != nullptr ? stream.msg : "no message"); + m_failed = true; + return Result::Error; + } + + bool outputFull = stream.avail_out == 0; + if (outputFull || (ret == Z_STREAM_END && outputUsed > 0)) { + if (!sink(std::span(m_storage->output, outputUsed))) { + m_failed = true; + return Result::Error; + } + outputUsed = 0; + } + + if (ret == Z_STREAM_END) { + if (stream.avail_in != 0) { + OS_LOGE(TAG, "Data after the end of the gzip stream"); + m_failed = true; + return Result::Error; + } + m_done = true; + return Result::Done; + } + + // A full buffer may have left output pending in zlib, so go round again even when the input is used up + if (!outputFull && stream.avail_in == 0) { + return Result::NeedMoreInput; + } + + if (ret == Z_BUF_ERROR && !outputFull) { + // Room for output and input left, yet no progress: not a state a well-formed stream gets zlib into + OS_LOGE(TAG, "zlib made no progress"); + m_failed = true; + return Result::Error; + } + } +} diff --git a/components/ota/src/GzipInflater.h b/components/ota/src/GzipInflater.h new file mode 100644 index 00000000..a58fc70a --- /dev/null +++ b/components/ota/src/GzipInflater.h @@ -0,0 +1,92 @@ +#pragma once + +#include "util/FunctionRef.h" + +#include + +#include +#include +#include + +namespace OpenShock { + /// @brief Streaming gzip decoder: zlib's inflate, with all of its memory allocated up front. + /// + /// The stream must be compressed with a window of at most kWindowBits (gzip/zlib `wbits` <= 12, a 4 KB window); + /// zlib rejects a stream that refers back further than that ("invalid distance too far back"). + /// + /// Construction allocates one block that zlib's allocations are served from (its state and window) plus the + /// output buffer, and keeps it only if CONFIG_OPENSHOCK_OTA_GZIP_MIN_FREE_BLOCK of contiguous internal RAM is left + /// for the TLS connections that follow. So the caller learns whether the memory is there before it asks for a + /// compressed download. release() gives it back early, e.g. when the server did not send gzip. The caller owns the + /// lifetime; nothing is pooled. + /// + /// Without CONFIG_OPENSHOCK_OTA_GZIP it never allocates and is never valid. Guard the gzip paths with + /// `if constexpr (GzipInflater::kEnabled)`: they stay compiled and type-checked, but emit no code, so nothing + /// reaches zlib and the linker drops it. + /// + /// Not thread-safe. + class GzipInflater { + public: + // The one #if: Kconfig leaves a disabled bool undefined, so it can't be used in `if constexpr` directly +#if CONFIG_OPENSHOCK_OTA_GZIP + static constexpr bool kEnabled = true; +#else + static constexpr bool kEnabled = false; +#endif + + static constexpr int kWindowBits = 12; + static constexpr std::size_t kOutputSize = 4096; // the sink gets pieces of this size, the last one shorter + + /// @brief Receives decompressed output in order. Returning false aborts decoding. + using Sink = Util::FunctionRef)>; + + enum class Result : uint8_t { + NeedMoreInput, // all input consumed, the stream is not finished + Done, // the stream ended and zlib verified its CRC-32 and size + Error, // malformed stream, trailing data, or the sink refused + }; + + /// @brief Whether a body starting with this byte is gzip (the first magic byte). A firmware image starts with + /// 0xE9, so one byte tells the two apart; a body that starts like gzip but is not fails in feed(). + [[nodiscard]] static constexpr bool StartsLikeGzip(uint8_t firstByte) noexcept { return firstByte == 0x1F; } + + GzipInflater(const GzipInflater&) = delete; + GzipInflater& operator=(const GzipInflater&) = delete; + GzipInflater(GzipInflater&&) = delete; + GzipInflater& operator=(GzipInflater&&) = delete; + + GzipInflater() noexcept; + ~GzipInflater(); + + /// @brief Whether the memory is held, i.e. the constructor succeeded and release() was not called. + [[nodiscard]] bool valid() const noexcept { return m_storage != nullptr; } + + /// @brief Frees the memory now instead of at destruction. The inflater is unusable afterwards. + void release() noexcept; + + /// @brief Bytes held by a valid inflater. + [[nodiscard]] static std::size_t AllocationSize() noexcept; + + /// @brief Starts a new stream. Must be called before the first feed() of every stream. + [[nodiscard]] bool reset() noexcept; + + /// @brief Decodes the next piece of the compressed stream. After Error every call returns Error, and after Done + /// any further input does, until reset(). + [[nodiscard]] Result feed(std::span input, Sink sink) noexcept; + + /// @brief Whether the stream has ended and verified. + [[nodiscard]] bool finished() const noexcept { return m_done; } + + private: + struct Storage; + + void allocate() noexcept; + + static void* zalloc(void* opaque, unsigned items, unsigned size); + static void zfree(void* opaque, void* address); + + Storage* m_storage; + bool m_done; + bool m_failed; + }; +} // namespace OpenShock diff --git a/components/ota/src/OtaUpdateManager.cpp b/components/ota/src/OtaUpdateManager.cpp index dad9a88c..3d664948 100644 --- a/components/ota/src/OtaUpdateManager.cpp +++ b/components/ota/src/OtaUpdateManager.cpp @@ -8,6 +8,7 @@ const char* const TAG = "OtaUpdateManager"; #include "BoardDescriptor.h" #include "captiveportal/Manager.h" #include "GatewayConnectionManager.h" +#include "GzipInflater.h" #include "Hashing.h" #include "http/ContentTypes.h" #include "http/HTTPRequestManager.h" @@ -215,7 +216,23 @@ static bool otaum_slot_matches_this_board(const esp_partition_t* slot, const cha return true; } -static bool otaum_flash_app_partition(OpenShock::HTTP::Client& client, const esp_partition_t* partition, const char* remoteUrl, const uint8_t (&remoteHash)[32]) +/// @brief The gzip image sits next to the plain one: the same URL with ".gz" on the path (before any query). +static std::string otaum_gzip_url_for(std::string_view url) +{ + std::size_t pathEnd = url.find_first_of("?#"); + if (pathEnd == std::string_view::npos) { + pathEnd = url.size(); + } + + std::string gzipUrl; + gzipUrl.reserve(url.size() + 3); + gzipUrl.append(url.substr(0, pathEnd)).append(".gz").append(url.substr(pathEnd)); + return gzipUrl; +} + +/// @param inflater Holds the memory to decompress the gzip image with; released once the gzip attempt is over, so the +/// plain fallback and everything after the download run without it. +static bool otaum_flash_app_partition(OpenShock::HTTP::Client& client, const esp_partition_t* partition, const std::string& remoteUrl, const uint8_t (&remoteHash)[32], OpenShock::GzipInflater& inflater) { OS_LOGD(TAG, "Flashing app partition"); @@ -231,7 +248,21 @@ static bool otaum_flash_app_partition(OpenShock::HTTP::Client& client, const esp return true; }; - if (!OpenShock::FlashPartitionFromUrl(client, partition, remoteUrl, remoteHash, onProgress)) { + // Not every release has a gzip image, and the manifest does not say: try it, and on any failure (a 404 for a release + // without one, or anything else) fall back to the plain image. Only the inactive slot was written by then. + bool flashed = false; + if constexpr (OpenShock::GzipInflater::kEnabled) { + if (inflater.valid()) { + std::string gzipUrl = otaum_gzip_url_for(remoteUrl); + flashed = OpenShock::FlashPartitionFromUrl(client, partition, gzipUrl.c_str(), remoteHash, onProgress, std::nullopt, &inflater); + if (!flashed) { + OS_LOGW(TAG, "No usable gzip app image, downloading the uncompressed one"); + } + } + inflater.release(); + } + + if (!flashed && !OpenShock::FlashPartitionFromUrl(client, partition, remoteUrl.c_str(), remoteHash, onProgress)) { OS_LOGE(TAG, "Failed to flash app partition"); _sendFailureMessage("Failed to flash app partition"); return false; @@ -509,6 +540,10 @@ static void otaum_updatetask(void* arg) continue; } + // Allocated before the downloads connect to the artifact host, so whether to fetch the gzip image is decided on + // memory already held rather than found missing mid-download. Kept only if enough is left for TLS. + OpenShock::GzipInflater inflater; // a never-valid stub without CONFIG_OPENSHOCK_OTA_GZIP + // Flash the app and www partitions. An unchanged www image (the same hash) is not written again. std::array currentWwwHash {}; const bool wwwCurrent = Www::ImageSha256(wwwPartition, currentWwwHash) && memcmp(currentWwwHash.data(), release.wwwBinaryHash, 32) == 0; @@ -517,7 +552,7 @@ static void otaum_updatetask(void* arg) } else if (!otaum_flash_www_partition(client, wwwPartition, release.wwwBinaryUrl.c_str(), release.wwwBinaryHash)) { continue; } - if (!otaum_flash_app_partition(client, appPartition, release.appBinaryUrl.c_str(), release.appBinaryHash)) { + if (!otaum_flash_app_partition(client, appPartition, release.appBinaryUrl, release.appBinaryHash, inflater)) { continue; } diff --git a/components/ota/src/PartitionUtils.cpp b/components/ota/src/PartitionUtils.cpp index d1eec4e0..0f5c0b17 100644 --- a/components/ota/src/PartitionUtils.cpp +++ b/components/ota/src/PartitionUtils.cpp @@ -2,6 +2,7 @@ const char* const TAG = "PartitionUtils"; +#include "GzipInflater.h" #include "Hashing.h" #include "http/ContentTypes.h" #include "http/HTTPRequestManager.h" @@ -22,7 +23,8 @@ bool OpenShock::FlashPartitionFromUrl( const char* remoteUrl, const uint8_t (&remoteHash)[32], Util::FunctionRef progressCallback, - std::optional)>> firstChunkCheck + std::optional)>> firstChunkCheck, + GzipInflater* inflater ) { OpenShock::SHA256 sha256; @@ -31,24 +33,32 @@ bool OpenShock::FlashPartitionFromUrl( return false; } - std::size_t contentLength = 0; - std::size_t contentWritten = 0; + std::size_t contentLength = 0; // of the body as sent: the compressed size for gzip + std::size_t received = 0; // body bytes received, what progress is measured in + std::size_t contentWritten = 0; // bytes written to the partition std::size_t erasedTo = 0; int64_t lastProgress = 0; - // Where the time goes (us), logged once at the end: whatever isn't flash, hashing or progress reporting is the - // network (TLS reads and waiting for data), which can't overlap the flash work in this single-task loop. + // Where the time goes (us), logged once at the end: whatever isn't flash, hashing, inflating or progress reporting + // is the network (TLS reads and waiting for data), which can't overlap the flash work in this single-task loop. struct { int64_t erase, write, hash, progress; } spent = {}; + // Decided by the first byte of the body, when an inflater was handed in. Every test of it goes with kEnabled: the + // compiler can't prove it stays false once the download callback has it by reference. + bool gzip = false; + int64_t inflateSpent = 0; // us, excluding the flash work done from inside the inflater + auto sizeValidator = [partition, &contentLength, progressCallback, &lastProgress, &spent](std::size_t size) -> bool { + // A gzip body larger than the partition is not a sensible image either; the decompressed size is bounded by + // writeOut if (size > partition->size) { OS_LOGE(TAG, "Remote partition binary is too large"); return false; } - // Erased incrementally by dataWriter, just ahead of the data + // Erased incrementally by writeOut, just ahead of the data contentLength = size; lastProgress = OpenShock::millis(); @@ -58,15 +68,23 @@ bool OpenShock::FlashPartitionFromUrl( return true; }; - auto dataWriter = [partition, &sha256, &contentLength, &contentWritten, &erasedTo, progressCallback, &lastProgress, &firstChunkCheck, &spent](std::size_t offset, const uint8_t* data, std::size_t length) -> bool { + + // Writes the next piece of the image (decompressed, for gzip) to the partition and hashes it + auto writeOut = [partition, &sha256, &contentWritten, &erasedTo, &firstChunkCheck, &spent](std::span chunk) -> bool { // Before the first erase: a refused first chunk leaves the partition as it was - if (offset == 0 && firstChunkCheck && !(*firstChunkCheck)(std::span(data, length))) { + if (contentWritten == 0 && firstChunkCheck && !(*firstChunkCheck)(chunk)) { OS_LOGE(TAG, "The download is not what the partition takes; nothing was erased"); return false; } + // Content-Length only bounds a plain body; a decompressed one is bounded here + if (chunk.size() > partition->size - contentWritten) { + OS_LOGE(TAG, "The image is larger than the partition"); + return false; + } + int64_t t0 = OpenShock::micros(); - while (erasedTo < offset + length) { + while (erasedTo < contentWritten + chunk.size()) { // The step is a multiple of the erase size, and partitions end on an erase-size boundary std::size_t eraseLength = std::min(kEraseStep, partition->size - erasedTo); if (eraseLength == 0 || esp_partition_erase_range(partition, erasedTo, eraseLength) != ESP_OK) { @@ -77,31 +95,70 @@ bool OpenShock::FlashPartitionFromUrl( } int64_t t1 = OpenShock::micros(); - if (esp_partition_write(partition, offset, data, length) != ESP_OK) { + if (esp_partition_write(partition, contentWritten, chunk.data(), chunk.size()) != ESP_OK) { OS_LOGE(TAG, "Failed to write to partition"); return false; } int64_t t2 = OpenShock::micros(); - if (!sha256.update(data, length)) { + if (!sha256.update(chunk.data(), chunk.size())) { OS_LOGE(TAG, "Failed to update SHA256 hash"); return false; } int64_t t3 = OpenShock::micros(); - contentWritten += length; + contentWritten += chunk.size(); + + spent.erase += t1 - t0; + spent.write += t2 - t1; + spent.hash += t3 - t2; + + return true; + }; + + auto dataWriter = [&](std::size_t offset, const uint8_t* data, std::size_t length) -> bool { + if (length == 0) { + return true; + } + + received += length; + + if constexpr (GzipInflater::kEnabled) { + if (offset == 0 && inflater != nullptr && inflater->valid()) { + if (GzipInflater::StartsLikeGzip(data[0])) { + if (!inflater->reset()) { + return false; + } + gzip = true; + } else { + // Asked for gzip, got something else (e.g. a CDN that decoded it): flash it as is, the hash still decides + OS_LOGW(TAG, "The body is not gzip, flashing it as is"); + inflater->release(); + } + } + } + + if (GzipInflater::kEnabled && gzip) { + int64_t t0 = OpenShock::micros(); + int64_t sinkSpent = spent.erase + spent.write + spent.hash; + if (inflater->feed(std::span(data, length), writeOut) == GzipInflater::Result::Error) { + OS_LOGE(TAG, "Failed to decompress the download"); + return false; + } + inflateSpent += (OpenShock::micros() - t0) - (spent.erase + spent.write + spent.hash - sinkSpent); + } else if (!writeOut(std::span(data, length))) { // written in order, at contentWritten + return false; + } int64_t now = OpenShock::millis(); if (now - lastProgress >= 500) { // Send progress every 500ms lastProgress = now; - progressCallback(contentWritten, contentLength, static_cast(contentWritten) / static_cast(contentLength)); + int64_t t0 = OpenShock::micros(); + // A chunked body has no Content-Length to measure against + progressCallback(received, contentLength, contentLength > 0 ? static_cast(received) / static_cast(contentLength) : 0.0f); + spent.progress += OpenShock::micros() - t0; } - spent.erase += t1 - t0; - spent.write += t2 - t1; - spent.hash += t3 - t2; - spent.progress += OpenShock::micros() - t3; - return true; }; @@ -125,6 +182,16 @@ bool OpenShock::FlashPartitionFromUrl( int64_t total = OpenShock::micros() - downloadStart; int64_t other = total - spent.erase - spent.write - spent.hash - spent.progress; + + if (GzipInflater::kEnabled && gzip) { + if (!inflater->finished()) { + OS_LOGE(TAG, "The gzip stream ended early"); + return false; + } + other -= inflateSpent; + OS_LOGI(TAG, "Downloaded %zu bytes of gzip, inflating took %lld ms", received, inflateSpent / 1000); + } + OS_LOGI( TAG, "Flashed %zu bytes in %lld ms (%lld KB/s): erase %lld ms, write %lld ms, hash %lld ms, progress %lld ms, network+other %lld ms", diff --git a/components/ota/src/PartitionUtils.h b/components/ota/src/PartitionUtils.h index 5abc544f..5b6b2238 100644 --- a/components/ota/src/PartitionUtils.h +++ b/components/ota/src/PartitionUtils.h @@ -14,19 +14,26 @@ namespace OpenShock { namespace HTTP { class Client; } + class GzipInflater; /// @brief Streams `remoteUrl` into `partition`, erasing just ahead of the data, and checks the SHA-256 of what was /// downloaded against `remoteHash`. - /// @param progressCallback Invoked with (written, total, fraction) while flashing; only used during this call, never stored. + /// @param progressCallback Invoked with (body bytes received, Content-Length, fraction) while flashing; only used during + /// this call, never stored. /// @param firstChunkCheck Invoked with the first chunk of the body before anything is erased: returning false aborts /// the flash with the partition untouched, so a download that is not the kind of image the partition takes - /// never destroys what is there. Only used during this call, never stored. + /// never destroys what is there. Only used during this call, never stored. For a gzip body it sees the + /// first decompressed piece. + /// @param inflater When given and valid, a body that starts like gzip is decompressed through it and `remoteHash` + /// is checked against the decompressed image. Any other body is flashed as is and the inflater released, + /// so its memory is back before the bulk of the download. Only used during this call, never stored. bool FlashPartitionFromUrl( HTTP::Client& client, const esp_partition_t* partition, const char* remoteUrl, const uint8_t (&remoteHash)[32], Util::FunctionRef progressCallback, - std::optional)>> firstChunkCheck = std::nullopt + std::optional)>> firstChunkCheck = std::nullopt, + GzipInflater* inflater = nullptr ); } // namespace OpenShock diff --git a/dependencies.lock b/dependencies.lock index 1afe5857..402c2b81 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -26,6 +26,16 @@ dependencies: registry_url: https://components.espressif.com/ type: service version: 2.1.0 + espressif/zlib: + component_hash: 61b7c41a6c7091085555a3cef58722af4bd9078e8708ee00a0e78efb997daf7c + dependencies: + - name: idf + require: private + version: '>=5.1' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.3.2~2 idf: source: type: idf @@ -34,6 +44,7 @@ direct_dependencies: - espressif/esp_websocket_client - espressif/jsmn - espressif/json_generator -manifest_hash: fdbd37009830b0ba06da199c3db943a10aaa1182b311045c502121c502d74dd5 +- espressif/zlib +manifest_hash: 65cd11c51ee06606759647c78d779d3e24fa082fc83a99f682546dac12494364 target: esp32s3 version: 3.0.0 From 31cff9c0f198472f34cfd240ae6f5dc0ea652edf Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Mon, 9 Nov 2026 15:42:34 +0100 Subject: [PATCH 2/2] style(ota): clang-format the gzip OTA changes --- components/ota/include/OtaUpdateManager.h | 2 +- components/ota/src/GzipInflater.cpp | 18 ++++++++++-------- components/ota/src/PartitionUtils.cpp | 2 +- components/ota/src/PartitionUtils.h | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/components/ota/include/OtaUpdateManager.h b/components/ota/include/OtaUpdateManager.h index 26160de4..bd791409 100644 --- a/components/ota/include/OtaUpdateManager.h +++ b/components/ota/include/OtaUpdateManager.h @@ -20,7 +20,7 @@ namespace OpenShock::OtaUpdateManager { struct FirmwareRelease { std::string appBinaryUrl; uint8_t appBinaryHash[32]; // of the image, so it also checks what `appBinaryUrl` + ".gz" decompresses to - std::string wwwBinaryUrl; // the `staticfs` artifact: the www image (www.bin) + std::string wwwBinaryUrl; // the `staticfs` artifact: the www image (www.bin) uint8_t wwwBinaryHash[32]; }; diff --git a/components/ota/src/GzipInflater.cpp b/components/ota/src/GzipInflater.cpp index 87c23838..bb4e5af9 100644 --- a/components/ota/src/GzipInflater.cpp +++ b/components/ota/src/GzipInflater.cpp @@ -17,9 +17,9 @@ using namespace OpenShock; // out of the arena, so the arena must hold both: the window exactly, the state with some headroom since its size is // internal to zlib. The constructor checks the window still fits after init, so a short arena fails up front rather // than mid-download. -static constexpr std::size_t kWindowSize = std::size_t {1} << GzipInflater::kWindowBits; -static constexpr std::size_t kStateReserve = 8 * 1024; -static constexpr std::size_t kArenaSize = kStateReserve + kWindowSize; +static constexpr std::size_t kWindowSize = std::size_t {1} << GzipInflater::kWindowBits; +static constexpr std::size_t kStateReserve = 8 * 1024; +static constexpr std::size_t kArenaSize = kStateReserve + kWindowSize; static constexpr std::size_t kArenaAlignment = alignof(std::max_align_t); struct GzipInflater::Storage { @@ -49,7 +49,9 @@ void* GzipInflater::zalloc(void* opaque, unsigned items, unsigned size) return storage->arena + start; } -void GzipInflater::zfree(void* /*opaque*/, void* /*address*/) { } +void GzipInflater::zfree(void* /*opaque*/, void* /*address*/) +{ +} GzipInflater::GzipInflater() noexcept : m_storage(nullptr) @@ -69,10 +71,10 @@ void GzipInflater::allocate() noexcept return; } - Storage* storage = new (memory) Storage; - storage->arenaUsed = 0; - storage->outputUsed = 0; - storage->stream = {}; + Storage* storage = new (memory) Storage; + storage->arenaUsed = 0; + storage->outputUsed = 0; + storage->stream = {}; storage->stream.zalloc = &GzipInflater::zalloc; storage->stream.zfree = &GzipInflater::zfree; storage->stream.opaque = storage; diff --git a/components/ota/src/PartitionUtils.cpp b/components/ota/src/PartitionUtils.cpp index 0f5c0b17..c67c0724 100644 --- a/components/ota/src/PartitionUtils.cpp +++ b/components/ota/src/PartitionUtils.cpp @@ -48,7 +48,7 @@ bool OpenShock::FlashPartitionFromUrl( // Decided by the first byte of the body, when an inflater was handed in. Every test of it goes with kEnabled: the // compiler can't prove it stays false once the download callback has it by reference. bool gzip = false; - int64_t inflateSpent = 0; // us, excluding the flash work done from inside the inflater + int64_t inflateSpent = 0; // us, excluding the flash work done from inside the inflater auto sizeValidator = [partition, &contentLength, progressCallback, &lastProgress, &spent](std::size_t size) -> bool { // A gzip body larger than the partition is not a sensible image either; the decompressed size is bounded by diff --git a/components/ota/src/PartitionUtils.h b/components/ota/src/PartitionUtils.h index 5b6b2238..e5f8a7f2 100644 --- a/components/ota/src/PartitionUtils.h +++ b/components/ota/src/PartitionUtils.h @@ -34,6 +34,6 @@ namespace OpenShock { const uint8_t (&remoteHash)[32], Util::FunctionRef progressCallback, std::optional)>> firstChunkCheck = std::nullopt, - GzipInflater* inflater = nullptr + GzipInflater* inflater = nullptr ); } // namespace OpenShock