From 6b9d809915dead6bd6b7bed2fb0c97b589e7aef4 Mon Sep 17 00:00:00 2001 From: Alex Sorafumo Date: Thu, 1 Oct 2026 16:47:00 +1000 Subject: [PATCH 1/3] fix(signage): bound media cache storage, bandwidth and memory - The cache budget comes from navigator.storage.estimate() and persistent storage is requested. Files in the current playlist are never evicted, and a file that cannot fit streams instead. The fixed 512 MB budget evicted files it had just downloaded, so every sync downloaded them again. - A full store (QuotaExceededError or a blob write DataError) evicts, retries once, then streams the file. It retried every 5 minutes. - A failed store read no longer downloads the file again. Replaced, duplicate and empty records are deleted. - Downloads have no overall deadline while data arrives, so large files on slow links can finish. - Downloads stream into a Blob instead of an array of chunks. When the browser cannot build the Blob on small storage, files up to 50 MB retry once in memory and larger files stream. - The sync checks the cache again after its stagger delay. - The service worker no longer stores S3 media a second time. --- apps/signage/DEBUGGING.md | 24 +- apps/signage/USER_STORIES.md | 13 +- apps/signage/ngsw-config.json | 3 - apps/signage/src/app/media-cache.service.ts | 729 ++++++++++++++---- .../src/tests/media-cache.service.spec.ts | 436 ++++++++++- 5 files changed, 1019 insertions(+), 186 deletions(-) diff --git a/apps/signage/DEBUGGING.md b/apps/signage/DEBUGGING.md index c5673361883..765caeeba02 100644 --- a/apps/signage/DEBUGGING.md +++ b/apps/signage/DEBUGGING.md @@ -32,7 +32,7 @@ questions without needing to reproduce anything. | `playlists.takeover` | The override playlist, its media and when it ends | | `active_media` | What the background playlist currently resolves to | | `upcoming_schedules` | Every scheduled run in the next month, soonest first | -| `media_cache` | Per file `status`, `size`, `owners`; plus totals, budget, `failed_sync_attempts` | +| `media_cache` | Per file `status`, `size`, `owners`; plus totals, budget (`limit_bytes`), `too_large`, `failed_sync_attempts` | | `watchdog` | Heartbeats for `poll` / `schedule` / `playback`, which are `stalled`, the last fatal error, and the recovery count and throttle state | | `players` | Per player: `state`, `item_index`, `progress_percent`, `playing`, `queue`, `mid_play_through` | @@ -88,6 +88,7 @@ makes the display request use `?preview=true`. | Paused and does not resume | Pause and resume messages are obeyed only from the parent frame. Check what embeds the player and `players[].state` | | Plugin cut short, or held long | A play-through plugin advances on `finished`, or after a limit. Look for `did not report finished in time` in the console | | Blank screen, no `window.signage` | The application did not start. Look for `Application failed to start` in the console; it reloads with a backoff | +| Media always streams | `media_cache.too_large` — the file does not fit in `limit_bytes`; see [Media cache storage](#media-cache-storage) | ## Recovery watchdog @@ -189,6 +190,27 @@ recovered and you want to know what from. | `sessionStorage["SIGNAGE.boot_failures"]` | Consecutive failed starts, for the backoff | | IndexedDB `SignageMedia` → `files` | The cached media files themselves | +## Media cache storage + +The cache budget (`limit_bytes`) is 80% of the storage quota, less the usage +outside the cache. To see the values, run +`await navigator.storage.estimate()`. If the browser cannot supply them, the +budget is 512 MB. At startup the app requests persistent storage. To see the +result, run `await navigator.storage.persisted()`. + +- The cache never removes media that the current playlist uses. +- Media that cannot fit is not downloaded. Its URL shows in `too_large` and it + plays from the network. The cache tries it again only when more space is + available, or after a reload. +- If a write fails because storage is full (`QuotaExceededError` or + `DataError`), the cache removes the files that the playlist does not use and + tries one more time. +- On a small profile volume, the browser's blob storage can fill before the + disk does. The console then shows `Browser blob storage is full`. Files up to + 50 MB are downloaded one more time into memory. Larger files go into + `too_large` until the next reload. +- The service worker does not cache media. Cache Storage holds only the app. + ## Resetting ``` diff --git a/apps/signage/USER_STORIES.md b/apps/signage/USER_STORIES.md index 0129228e020..1e4b348d58d 100644 --- a/apps/signage/USER_STORIES.md +++ b/apps/signage/USER_STORIES.md @@ -450,7 +450,10 @@ The Signage app is a kiosk-style digital signage player. It bootstraps a device **Acceptance Criteria:** - Non-webpage and non-plugin media URLs are requested for local caching. -- Media files are stored in IndexedDB in the `SignageMedia` database. +- Media files are stored in IndexedDB in the `SignageMedia` database. The service worker does not keep a second copy. +- Downloads stream to storage. They do not keep the full file in memory. +- If the browser cannot store a streamed download, the cache downloads files up to 50 MB one more time into memory. Larger files play from the network, and the cache does not try them again until the app reloads. +- A download has no total time limit. It stops when no data arrives for 60 seconds. - Cache metadata is persisted in localStorage under `PlaceOS.SIGNAGE.cached_files`. - Cache status moves through preparing, downloading, storing, and cached states. - Upload API media requests apply a short-lived authentication cookie before fetching. @@ -469,7 +472,13 @@ The Signage app is a kiosk-style digital signage player. It bootstraps a device - When display configuration changes, the app requests caching for current media URLs. - Cached URLs that are no longer referenced by the display are invalidated. -- Cache pruning keeps the current display's priority URLs first and enforces a per-owner storage limit. +- The cache budget is 80% of the storage that the browser gives the app, less the storage used outside the cache. If the browser cannot supply this value, the budget is 512 MB. +- At startup, the app asks the browser for persistent storage. +- Pruning never removes media in the current request. It removes files of other displays first (root players only), then the largest files. +- Media that cannot fit in the budget is not downloaded. It plays from the network. The cache tries it again only when more space is available. +- If storage becomes full during a write (`QuotaExceededError`, or a `DataError` from a failed blob write), the cache removes the files that the request does not need and tries one more time. If the write fails again, the media plays from the network. +- If the cache database cannot be read, the cache does not download the file again. The sync tries again later. +- Stored files that no cache entry uses (duplicates, empty files, and replaced files) are deleted. - Embedded signage players avoid pruning files owned by other displays. - Failed cache requests schedule a retry after 15 seconds. - Media currently preparing, downloading, or storing waits for a final cached or invalidated state before playback tries to use it. diff --git a/apps/signage/ngsw-config.json b/apps/signage/ngsw-config.json index ec5831cc08b..6662b665bb9 100644 --- a/apps/signage/ngsw-config.json +++ b/apps/signage/ngsw-config.json @@ -23,9 +23,6 @@ "files": [ "/assets/**", "/*.(eot|svg|cur|jpg|png|webp|gif|otf|ttf|woff|woff2|ani)" - ], - "urls": [ - "https://*.amazonaws.com/**/*.*" ] } } diff --git a/apps/signage/src/app/media-cache.service.ts b/apps/signage/src/app/media-cache.service.ts index 1daa221d6b7..1b51ddbe0c9 100644 --- a/apps/signage/src/app/media-cache.service.ts +++ b/apps/signage/src/app/media-cache.service.ts @@ -17,7 +17,16 @@ const DB_VERSION = 1; const DB_STORE = 'files'; const UPLOADS_PATH = '/api/engine/v2/uploads'; const STAGGER_DELAY_MS = 500; // Delay between uncached resource requests -const DEFAULT_OWNER_CACHE_LIMIT_BYTES = 512 * 1024 * 1024; +/** Cache budget used when the browser cannot report its storage */ +const FALLBACK_CACHE_LIMIT_BYTES = 512 * 1024 * 1024; +/** + * Share of the storage available to this origin that the cache may fill. The + * rest is headroom for the app itself, database overhead, and an estimate that + * lags behind recent writes. + */ +const STORAGE_BUDGET_SHARE = 0.8; +/** Most URLs remembered as too large to cache, so the list stays bounded */ +const MAX_TOO_LARGE_URLS = 200; /** * How long a single database request may take before it counts as failed. A * request that never settles would otherwise hold the whole cache sync - and @@ -26,11 +35,30 @@ const DEFAULT_OWNER_CACHE_LIMIT_BYTES = 512 * 1024 * 1024; const DB_OPERATION_TIMEOUT_MS = 30 * SECONDS; /** Minimum spacing between attempts to reopen a broken database connection */ const DB_RECONNECT_INTERVAL_MS = 30 * SECONDS; -/** How long a download may go without receiving any data before it is abandoned */ +/** + * How long a download may go without receiving any data before it is + * abandoned. A download that keeps receiving data has no overall deadline, so + * a large file on a slow link can always finish. + */ const DOWNLOAD_STALL_MS = 60 * SECONDS; -/** Longest a single download may run, however slowly it is progressing */ +/** + * Shortest deadline for a download that cannot be watched for progress (no + * streaming support). Longer files get time at `MIN_DOWNLOAD_BYTES_PER_SECOND`. + */ const DOWNLOAD_TIMEOUT_MS = 15 * MINUTES; -/** Longest anything waits on an in-progress download to reach a final state */ +/** Slowest link assumed when sizing that deadline: about 1 Mbps */ +const MIN_DOWNLOAD_BYTES_PER_SECOND = 128 * 1024; +/** + * Largest file read into memory when the browser cannot build a Blob from a + * stream. That read holds about twice the file at its peak: about 100 MB here, + * which a 2 GB player can spare beside the video it is playing. Larger files + * play from the network instead. + */ +const IN_MEMORY_DOWNLOAD_LIMIT_BYTES = 50 * 1024 * 1024; +/** + * Longest a caller waits on a download that is already in progress before it + * stops waiting. The download itself carries on. + */ const DOWNLOAD_WAIT_MS = DOWNLOAD_TIMEOUT_MS + DB_OPERATION_TIMEOUT_MS; /** Lifetime of the cookie that lets media elements stream protected uploads */ const DIRECT_URL_COOKIE_SECONDS = 60 * 60; @@ -55,7 +83,9 @@ export interface CacheItem { } export interface CacheRequestOptions { + /** Bytes the whole cache may hold. Defaults to the storage budget. */ max_size?: number; + /** Whether files of other owners may be evicted to make room */ prune_other_owners?: boolean; } @@ -72,6 +102,51 @@ interface DownloadResult { file: File | null; /** Whether the file is now in the cache store */ stored: boolean; + /** The file does not fit in storage, so it plays from the network */ + no_room?: boolean; +} + +/** How much a download may store, and how to make room for it */ +interface CacheFit { + /** Largest file that may be stored */ + max_bytes: number; + /** + * Evict entries the request does not need until `bytes` more fit in the + * budget. `Infinity` evicts every entry the request may evict. + */ + make_room?: (bytes: number) => Promise; +} + +/** A response body stream, typed as the browser hands it over */ +type ResponseBody = NonNullable; + +/** A file that does not fit in the storage the cache may use */ +class NoRoomError extends Error { + /** + * @param bytes Room the file needs before it is worth trying again. + * `Infinity` means not until the player reloads. + */ + constructor(public readonly bytes: number) { + super(`Media needs ${bytes} bytes of storage`); + } +} + +/** + * The browser could not build a Blob from a download that arrived fine, + * usually because its blob storage is full. On a small profile volume that + * limit can be far below the free space. + */ +class BlobStorageError extends Error {} + +/** + * Whether a write failed because storage is full. Chrome reports a disk that + * fills during a blob write as a `DataError` ("Failed to write blobs"), not a + * quota error. The cache's keys are always valid, so no other `DataError` is + * expected from a write. + */ +function isStorageFullError(error: unknown) { + const name = (error as DOMException | null)?.name; + return name === 'QuotaExceededError' || name === 'DataError'; } function isLoadingStatus(status: CacheItemStatus) { @@ -129,6 +204,118 @@ function withTimeout( }); } +/** + * Read a response body into a Blob. Fails if no data arrives for + * `DOWNLOAD_STALL_MS`, or once the body grows past `max_bytes`. Chunks pass + * straight through into the Blob instead of being collected first, so the + * file is held once, and the browser can keep a large one on disk. + * + * The browser builds that Blob in its own blob storage, which has a limit of + * its own. When the Blob cannot be built while the network is fine, this + * rejects with a `BlobStorageError`. + */ +function streamToBlob( + body: ResponseBody, + type: string, + max_bytes: number, + abort: () => void, +) { + return new Promise((resolve, reject) => { + const reader = body.getReader(); + let timer: ReturnType; + let received = 0; + // Set when the download itself failed, not the Blob it feeds + let source_error: unknown = null; + const fail = (error: unknown) => { + clearTimeout(timer); + abort(); + // Also stops reading where the request cannot be aborted + reader.cancel(error).catch(() => undefined); + reject(error); + }; + const watch = () => { + clearTimeout(timer); + timer = setTimeout( + () => fail(new Error('Download stalled')), + DOWNLOAD_STALL_MS, + ); + }; + const counted = new ReadableStream({ + // Only a failed read or an oversized body counts as the source + // failing. `close` and `enqueue` throw once the Blob side has + // given up, and that is a blob storage failure. + pull: async (controller) => { + const { done, value } = await reader.read().catch((e) => { + source_error = e || new Error('Download failed'); + throw source_error; + }); + if (done) return controller.close(); + received += value.byteLength; + if (received > max_bytes) { + source_error = new NoRoomError(received); + throw source_error; + } + watch(); + controller.enqueue(value); + }, + cancel: (reason) => reader.cancel(reason), + }); + watch(); + new Response( + counted, + type ? { headers: { 'content-type': type } } : undefined, + ) + .blob() + .then( + (blob) => { + clearTimeout(timer); + resolve(blob); + }, + (e) => fail(source_error || new BlobStorageError(`${e}`)), + ); + }); +} + +/** + * Read a response body into memory, then into a Blob. Holds about twice the + * file at its peak, so it is only a fallback for small files when the browser + * cannot build a Blob from a stream. A file past `max_bytes`, or one the + * browser cannot hold, fails with a `NoRoomError` that is not retried. + */ +async function readToBlob( + body: ResponseBody, + type: string, + max_bytes: number, + abort: () => void, +) { + const reader = body.getReader(); + const chunks: BlobPart[] = []; + let received = 0; + // Ends when the body does, stalls, or passes `max_bytes` + for (;;) { + const { done, value } = await withTimeout( + reader.read(), + DOWNLOAD_STALL_MS, + 'Download stalled', + abort, + ); + if (done) break; + received += value.byteLength; + if (received > max_bytes) { + abort(); + reader.cancel().catch(() => undefined); + throw new NoRoomError(Infinity); + } + chunks.push(value); + } + try { + return new Blob(chunks, { type }); + } catch (e) { + log.warn(`Unable to hold downloaded media in memory. ${e}`); + throw new NoRoomError(Infinity); + } +} + /** * Local store of media files for offline playback. * @@ -152,6 +339,13 @@ export class MediaCacheService extends AsyncHandler { private readonly _unverified_ids = new Set(); /** Downloads currently running, keyed by URL, so they are never duplicated */ private readonly _downloads = new Map>(); + /** + * URLs that do not fit in storage, with the room they need before they + * are tried again. They play from the network until then. + */ + private readonly _too_large = new Map(); + /** Bytes the cache may hold, as last worked out from the storage estimate */ + private _budget_bytes = FALLBACK_CACHE_LIMIT_BYTES; private _last_reconnect = 0; private get _cache_index() { @@ -162,17 +356,26 @@ export class MediaCacheService extends AsyncHandler { super(); this._loadCacheMetadata(); this._connectDatabase(); + this._requestPersistentStorage(); effect(() => { this._file_cache_index(); this._saveCacheMetadata(); }); } + /** + * Cache the files in `url_list`, most important first. Other entries are + * evicted to make room for them, but entries in the list never are. A + * file that cannot fit is not downloaded; it plays from the network. + * Resolves true when a file failed in a way that is worth retrying. + */ public async requestFilesToCache( url_list: string[], owner = '', options: CacheRequestOptions = {}, ): Promise { + const budget = options.max_size ?? (await this._storageBudget()); + const prune_others = !!options.prune_other_owners; let failures = false; let uncached_count = 0; for (const url of url_list) { @@ -185,33 +388,51 @@ export class MediaCacheService extends AsyncHandler { await this._addOwner(existing, owner); continue; } - } else if ( - existing.status === 'cached' && - (await this._hasStoredFile(existing, url)) - ) { - await this._addOwner(existing, owner); - continue; + } else if (existing.status === 'cached') { + // A store that cannot be read says nothing about the + // file. Downloading it again would orphan the old copy. + const stored = await this._hasStoredFile( + existing, + url, + ).catch(() => null); + if (stored === null) { + failures = true; + continue; + } + if (stored) { + await this._addOwner(existing, owner); + continue; + } } } + const room = + budget - this._pinnedBytes(owner, url_list, prune_others); + if (!this._mayFit(url, room)) continue; // Stagger requests for uncached resources to avoid overwhelming the network if (uncached_count > 0) await delay(STAGGER_DELAY_MS); uncached_count++; - const { stored } = await this._cacheFile(url, owner); - if (!stored) failures = true; - await this.pruneCache( - owner, - url_list, - options.max_size, - options.prune_other_owners, - ); + // Playback may have cached the file, or found it too large, during + // the delay + const latest = this._cacheItem(url); + if (latest?.status === 'cached') { + await this._addOwner(latest, owner); + continue; + } + if (!this._mayFit(url, room)) continue; + const { stored, no_room } = await this._cacheFile(url, owner, { + max_bytes: room, + make_room: (bytes) => + this.pruneCache( + owner, + url_list, + budget - bytes, + prune_others, + ), + }); + if (!stored && !no_room) failures = true; } this._file_cache_index.set([...this._cache_index]); - await this.pruneCache( - owner, - url_list, - options.max_size, - options.prune_other_owners, - ); + await this.pruneCache(owner, url_list, budget, prune_others); return failures; } @@ -226,8 +447,9 @@ export class MediaCacheService extends AsyncHandler { * The file for a URL, from the cache when it has it and downloaded when it * does not. Unlike `requestFilesToCache` this hands back a download that * could not be stored, so a broken database never stops media playing. - * Waits at most `wait_ms` for a download that is already in progress and - * returns null if it has not finished by then. + * Resolves null for a file too large to cache, which plays from the + * network instead. Waits at most `wait_ms` for a download that is already + * in progress and returns null if it has not finished by then. */ public async fetchFile( url: string, @@ -248,7 +470,10 @@ export class MediaCacheService extends AsyncHandler { ); if (file) return file; } - const { file } = await this._cacheFile(url, owner); + if (this._too_large.has(url)) return null; + const { file } = await this._cacheFile(url, owner, { + max_bytes: this._budget_bytes, + }); return file; } @@ -279,8 +504,9 @@ export class MediaCacheService extends AsyncHandler { file_count: files.length, cached_count: files.filter((_) => _.status === 'cached').length, total_bytes: files.reduce((total, _) => total + _.size, 0), - limit_bytes: DEFAULT_OWNER_CACHE_LIMIT_BYTES, + limit_bytes: this._budget_bytes, downloads_in_flight: this._downloads.size, + too_large: [...this._too_large.keys()], files, }; } @@ -298,11 +524,11 @@ export class MediaCacheService extends AsyncHandler { /** * Whether a file is still being prepared/downloaded/stored, or has not yet * been registered for caching (i.e. queued). Returns false once the file is - * cached or has been invalidated. + * cached, has been invalidated, or is too large to cache. */ public isLoadingFile(url: string): boolean { const item = this._cacheItem(url); - if (!item) return true; + if (!item) return !this._too_large.has(url); return isLoadingStatus(item.status); } @@ -335,64 +561,43 @@ export class MediaCacheService extends AsyncHandler { return this._storedFile(cache_item, url); } + /** + * Evict cached files until the whole cache fits in `max_size` bytes. + * Files in `priority_urls` are never evicted, nor are files shared with + * other owners unless `prune_other_owners` is set. Other owners' files go + * first, then the largest. + */ public async pruneCache( owner = '', priority_urls: string[] = [], - max_size = DEFAULT_OWNER_CACHE_LIMIT_BYTES, + max_size = this._budget_bytes, prune_other_owners = false, ) { - if (!this._cache_db_ready || max_size <= 0) return; + if (!this._cache_db_ready) return; // Sizes are tracked on the index, so the common case - comfortably - // under budget - costs nothing. Reading every record back out of the - // store to add up its size would pull every cached video into memory. - const candidates = this._cache_index.filter( - (item) => - item.status === 'cached' && - (!owner || - cacheOwners(item).includes(owner) || - prune_other_owners), - ); - // Metadata written before sizes were recorded needs one pass over the - // store to fill them in; after that this stays in memory. - if (candidates.some((item) => !(item.size > 0))) { + // under budget - costs nothing. Metadata written before sizes were + // recorded needs one pass over the store to fill them in. + if ( + this._cache_index.some( + (item) => item.status === 'cached' && !(item.size > 0), + ) + ) { await this._recoverCachedSizes(); } - const owner_items = candidates - .map((item) => { - const owners = cacheOwners(item); - return { - item, - owners, - size: item.size || 0, - priority: priority_urls.indexOf(item.url), - owner_priority: !owner || owners.includes(owner) ? 1 : 0, - }; - }) - .filter((_) => _.size > 0); - let total_size = owner_items.reduce( - (total, item) => total + item.size, - 0, - ); + let total_size = this._cachedBytes(); if (total_size <= max_size) return; - const eviction_list = owner_items.sort((a, b) => { - const a_priority = - a.priority >= 0 ? a.priority : Number.MAX_SAFE_INTEGER; - const b_priority = - b.priority >= 0 ? b.priority : Number.MAX_SAFE_INTEGER; - if (a.owner_priority !== b.owner_priority) { - return a.owner_priority - b.owner_priority; - } - if (a_priority !== b_priority) return b_priority - a_priority; - return b.size - a.size; - }); - for (const { item, owners, size } of eviction_list) { + const eviction_list = this._evictable( + owner, + priority_urls, + prune_other_owners, + ); + for (const item of eviction_list) { if (total_size <= max_size) break; - const is_owner_file = owner && owners.includes(owner); - await this.invalidateFile( - item.url, - is_owner_file ? owner : '', - ).catch(() => undefined); - total_size -= size; + const removed = await this.invalidateFile(item.url).then( + () => true, + () => false, + ); + if (removed) total_size -= item.size || 0; } } @@ -406,6 +611,7 @@ export class MediaCacheService extends AsyncHandler { } log.debug(`Cleared all cached resources.`); this._file_cache_index.set([]); + this._too_large.clear(); } public async invalidateFile(url: string, owner = '') { @@ -463,7 +669,11 @@ export class MediaCacheService extends AsyncHandler { * Download a URL into the cache, sharing the download with any other * caller asking for the same URL at the same time. */ - private _cacheFile(url: string, owner: string): Promise { + private _cacheFile( + url: string, + owner: string, + fit: CacheFit, + ): Promise { const in_flight = this._downloads.get(url); if (in_flight) return in_flight; const cache_item: CacheItem = { @@ -475,16 +685,21 @@ export class MediaCacheService extends AsyncHandler { on_change: new Subject(), }; // One entry per URL: a stale duplicate left behind would be found - // before this one and reported missing on every lookup. + // before this one and reported missing on every lookup. The records + // of the entries replaced go too, as nothing would point at them. + const replaced = this._cache_index.filter((_) => _.url === url); this._file_cache_index.set([ ...this._cache_index.filter((_) => _.url !== url), cache_item, ]); - const download = this._downloadAndStore(url, cache_item).finally(() => { - if (this._downloads.get(url) === download) { - this._downloads.delete(url); - } - }); + this._deleteRecords(replaced.map((_) => _.id)); + const download = this._downloadAndStore(url, cache_item, fit).finally( + () => { + if (this._downloads.get(url) === download) { + this._downloads.delete(url); + } + }, + ); this._downloads.set(url, download); return download; } @@ -492,41 +707,96 @@ export class MediaCacheService extends AsyncHandler { private async _downloadAndStore( url: string, cache_item: CacheItem, + fit: CacheFit = { max_bytes: Infinity }, ): Promise { let file: File | null = null; try { cacheStatus(cache_item, 'downloading'); // If not an API call, just load the image if (url.includes(UPLOADS_PATH)) this.applyAuthenticationCookie(); - const blob = await this._download(url); + const blob = await this._download(url, fit.max_bytes); if (blob.size <= 0) { log.error(`Downloaded resource is empty.`, url); throw new Error('Downloaded media file is empty'); } cacheStatus(cache_item, 'storing'); - // Create a File object (or you can use the blob directly) + // Wraps the blob without copying it file = new File([blob], cache_item.id, { type: blob.type }); - await this._storeFile(cache_item, file, url); + await fit.make_room?.(file.size); + try { + await this._storeFile(cache_item, file, url); + } catch (e) { + // The budget is an estimate, so storage can still run out. + // Evict everything the request does not need and try once + // more; past that the file plays from the network. + if (!isStorageFullError(e) || !fit.make_room) throw e; + log.warn(`Storage is full. Evicting media to retry.`, url); + await fit.make_room(Infinity); + try { + await this._storeFile(cache_item, file, url); + } catch (retry_error) { + if (!isStorageFullError(retry_error)) throw retry_error; + throw new NoRoomError(fit.max_bytes + file.size); + } + } cache_item.size = file.size; + this._too_large.delete(url); log.debug(`Cached resource.`, [cache_item.id, url]); cacheStatus(cache_item, 'cached'); this._file_cache_index.set([...this._cache_index]); return { file, stored: true }; } catch (e) { - log.error(`Error downloading resource.`, url, e); + const no_room = e instanceof NoRoomError; + if (no_room) { + log.warn( + `Media does not fit in storage. It will play from the network.`, + url, + e, + ); + this._markTooLarge(url, e.bytes); + } else { + log.error(`Error downloading resource.`, url, e); + } if (cache_item.status !== 'invalidated') { this._markInvalidated(cache_item); } - return { file, stored: false }; + return { file, stored: false, no_room }; + } + } + + /** + * Fetch a URL, giving up if the response stops arriving or turns out + * larger than `max_bytes`. A download that hangs would otherwise leave + * its cache entry loading forever, with the player and every later cache + * sync waiting behind it. One that is still receiving data has no + * deadline, so a large file on a slow link can finish. If the browser + * cannot build a Blob from the stream, a small file is downloaded once + * more into memory; a large one fails with a `NoRoomError`. + */ + private async _download(url: string, max_bytes: number): Promise { + try { + return await this._fetchBlob(url, max_bytes, false); + } catch (e) { + if (!(e instanceof BlobStorageError)) throw e; + log.warn( + `Browser blob storage is full. Retrying in memory.`, + url, + e, + ); + return this._fetchBlob(url, max_bytes, true); } } /** - * Fetch a URL, giving up if the response stops arriving. A download that - * hangs would otherwise leave its cache entry loading forever, with the - * player and every later cache sync waiting behind it. + * One attempt at `_download`. `in_memory` reads the body into memory + * instead of streaming it into a Blob, for files up to + * `IN_MEMORY_DOWNLOAD_LIMIT_BYTES`. */ - private async _download(url: string): Promise { + private async _fetchBlob( + url: string, + max_bytes: number, + in_memory: boolean, + ): Promise { const controller = typeof AbortController === 'function' ? new AbortController() @@ -542,34 +812,169 @@ export class MediaCacheService extends AsyncHandler { log.error(`Error fetching resource. ${response.status}`, url); throw new Error(`Request failed with status ${response.status}`); } - const reader = response.body?.getReader?.(); - if (!reader) { - return withTimeout( - response.blob(), - DOWNLOAD_TIMEOUT_MS, - 'Timed out downloading resource', - abort, - ); + // Refuse a file that cannot fit before downloading any of it + const length = Number(response.headers?.get?.('content-length')) || 0; + if (length > max_bytes) { + abort(); + throw new NoRoomError(length); } - const deadline = Date.now() + DOWNLOAD_TIMEOUT_MS; - const chunks: BlobPart[] = []; - for (;;) { - const remaining = deadline - Date.now(); - if (remaining <= 0) { + const type = response.headers?.get?.('content-type') || ''; + const body = response.body; + if (in_memory) { + if (length > IN_MEMORY_DOWNLOAD_LIMIT_BYTES) { abort(); - throw new Error('Timed out downloading resource'); + throw new NoRoomError(Infinity); } - const { done, value } = await withTimeout( - reader.read(), - Math.min(DOWNLOAD_STALL_MS, remaining), - 'Download stalled', + return readToBlob( + body, + type, + Math.min(max_bytes, IN_MEMORY_DOWNLOAD_LIMIT_BYTES), abort, ); - if (done) break; - if (value) chunks.push(value); } - const type = response.headers?.get?.('content-type') || ''; - return new Blob(chunks, { type }); + if (typeof body?.getReader === 'function') { + return streamToBlob(body, type, max_bytes, abort); + } + // Without streams progress cannot be watched, so allow for the whole + // file arriving at a slow but workable rate. + const blob = await withTimeout( + response.blob(), + Math.max( + DOWNLOAD_TIMEOUT_MS, + (length / MIN_DOWNLOAD_BYTES_PER_SECOND) * SECONDS, + ), + 'Timed out downloading resource', + abort, + ); + if (blob.size > max_bytes) throw new NoRoomError(blob.size); + return blob; + } + + /** + * Whether a download of `url` could fit in `room` bytes. A URL refused + * before is only tried again once there is the room it needed. + */ + private _mayFit(url: string, room: number) { + if (room >= (this._too_large.get(url) ?? 1)) return true; + if (!this._too_large.has(url)) this._markTooLarge(url, 1); + return false; + } + + private _markTooLarge(url: string, bytes: number) { + this._too_large.delete(url); + this._too_large.set(url, bytes); + if (this._too_large.size > MAX_TOO_LARGE_URLS) { + const [oldest] = this._too_large.keys(); + this._too_large.delete(oldest); + } + } + + private _cachedBytes() { + return this._cache_index + .filter((_) => _.status === 'cached') + .reduce((total, _) => total + (_.size || 0), 0); + } + + /** + * Cached entries a request may evict, in the order to evict them: files + * of other owners first, then the largest. Entries the request lists are + * never evicted, and neither are files shared with owners it may not + * touch. + */ + private _evictable( + owner: string, + priority_urls: string[], + prune_other_owners: boolean, + ) { + const own = (item: CacheItem) => + !owner || cacheOwners(item).includes(owner) ? 1 : 0; + return this._cache_index + .filter( + (item) => + item.status === 'cached' && + !priority_urls.includes(item.url) && + (!owner || + prune_other_owners || + cacheOwners(item).every((_) => _ === owner)), + ) + .sort((a, b) => own(a) - own(b) || (b.size || 0) - (a.size || 0)); + } + + /** Bytes held by cached entries that a request may not evict */ + private _pinnedBytes( + owner: string, + priority_urls: string[], + prune_other_owners: boolean, + ) { + const evictable = this._evictable( + owner, + priority_urls, + prune_other_owners, + ).reduce((total, _) => total + (_.size || 0), 0); + return this._cachedBytes() - evictable; + } + + /** + * Bytes the cache may hold: a share of the storage this origin may use, + * less what the app holds outside the cache. Falls back to a fixed budget + * when the browser cannot report its storage. + */ + private async _storageBudget() { + const storage = + typeof navigator === 'undefined' ? undefined : navigator.storage; + if (typeof storage?.estimate !== 'function') { + this._budget_bytes = FALLBACK_CACHE_LIMIT_BYTES; + return this._budget_bytes; + } + const estimate = await withTimeout( + storage.estimate(), + DB_OPERATION_TIMEOUT_MS, + 'Storage estimate timed out', + ).catch((e) => { + log.warn(`Unable to estimate storage. ${e}`); + return null; + }); + if (!(estimate?.quota > 0)) { + this._budget_bytes = FALLBACK_CACHE_LIMIT_BYTES; + return this._budget_bytes; + } + const other_usage = Math.max( + 0, + (estimate.usage || 0) - this._cachedBytes(), + ); + this._budget_bytes = Math.max( + 0, + Math.floor((estimate.quota - other_usage) * STORAGE_BUDGET_SHARE), + ); + return this._budget_bytes; + } + + /** + * Ask the browser not to clear stored media when the device runs low on + * space. A cleared cache leaves the player downloading everything again. + */ + private _requestPersistentStorage() { + const storage = + typeof navigator === 'undefined' ? undefined : navigator.storage; + if (typeof storage?.persist !== 'function') return; + storage.persist().then( + (granted) => + log.debug( + `Persistent storage ${granted ? 'granted' : 'denied'}.`, + ), + (e) => log.warn(`Unable to request persistent storage. ${e}`), + ); + } + + /** Delete store records that no entry points at. Failures are logged. */ + private _deleteRecords(ids: string[]) { + return Promise.all( + ids.map((id) => + this._write((store) => store.delete(id), 'delete').catch((e) => + log.warn(`Unable to delete orphaned media. ${e}`, id), + ), + ), + ); } /** @@ -617,21 +1022,18 @@ export class MediaCacheService extends AsyncHandler { /** * Whether the file behind a cache entry is still in the store. Uses a key * count rather than reading the record, so confirming a cached playlist - * does not pull every one of its files into memory. + * does not pull every one of its files into memory. Rejects when the store + * cannot be read, as that says nothing about whether the file is there. */ private async _hasStoredFile(cache_item: CacheItem, url: string) { if (!(cache_item.size > 0)) { // Size unknown - metadata written by an older build. Read the // record once to recover it; later checks are cheap. - const file = await this._storedFile(cache_item, url).catch( - () => null, - ); + const file = await this._storedFile(cache_item, url); if (file) this._setCachedSize(cache_item, file.size); return !!file; } - const exists = await this._storedFileExists(cache_item.id).catch( - () => false, - ); + const exists = await this._storedFileExists(cache_item.id); if (!exists) { this._markMissing(cache_item, url); return false; @@ -701,49 +1103,68 @@ export class MediaCacheService extends AsyncHandler { * Rebuild the cached entries from what the store actually holds. The store * is authoritative: persisted metadata is only a head start until it has * answered, and any entry it does not hold is dropped so nothing keeps - * looking for a file that is not there. + * looking for a file that is not there. Records no entry can use - empty, + * without a URL, or a second copy of a URL - are deleted, as nothing else + * would ever remove them. */ private async _loadCacheMetadataFromStore() { const records = await this._storedFileRecords().catch(() => null); if (!records) return; - const stored_items = records - .filter((record) => record.url && record.file?.size > 0) - .map((record) => ({ - id: record.name, - url: record.url, - owner: record.owner || '', - owners: cacheOwners(record), - size: record.file.size, - status: 'cached' as const, - on_change: new Subject(), - })); - const stored_ids = new Set(stored_items.map((_) => _.id)); - // Keep entries still in progress, and files cached by this session - // that the store snapshot may predate. Only entries restored from - // persisted metadata and never seen in the store are dropped. - const kept_items = this._cache_index.filter( - (item) => - item.status !== 'cached' || - stored_ids.has(item.id) || - !this._unverified_ids.has(item.id), + const usable = records.filter( + (record) => record.url && record.file?.size > 0, ); + const stored_ids = new Set(usable.map((_) => _.name)); + // Keep entries still in progress, and files cached by this session + // that the store snapshot may predate. Entries restored from persisted + // metadata and never seen in the store are dropped, and so are failed + // entries for a URL the store holds a usable file for. + const kept_items = this._cache_index.filter((item) => { + if (item.status === 'cached') { + return ( + stored_ids.has(item.id) || + !this._unverified_ids.has(item.id) + ); + } + if (item.status === 'invalidated') { + return !usable.some((_) => _.url === item.url); + } + return true; + }); const dropped = this._cache_index.length - kept_items.length; if (dropped > 0) { log.warn( `Dropped ${dropped} cached entries that have no stored file.`, ); } + const stored_items: CacheItem[] = []; + const orphan_ids: string[] = []; + for (const record of records) { + const kept = kept_items.find((_) => _.url === record.url); + if (kept?.id === record.name) continue; + if ( + !stored_ids.has(record.name) || + kept || + stored_items.some((_) => _.url === record.url) + ) { + orphan_ids.push(record.name); + continue; + } + stored_items.push({ + id: record.name, + url: record.url, + owner: record.owner || '', + owners: cacheOwners(record), + size: record.file.size, + status: 'cached', + on_change: new Subject(), + }); + } this._unverified_ids.clear(); - this._file_cache_index.set([ - ...kept_items, - ...stored_items.filter( - (stored) => - !kept_items.some( - (item) => - item.id === stored.id || item.url === stored.url, - ), - ), - ]); + this._file_cache_index.set([...kept_items, ...stored_items]); + if (orphan_ids.length) { + log.warn(`Deleting ${orphan_ids.length} orphaned media records.`); + await this._deleteRecords(orphan_ids); + } } private _storedFileRecords(): Promise { diff --git a/apps/signage/src/tests/media-cache.service.spec.ts b/apps/signage/src/tests/media-cache.service.spec.ts index bfd39a94cd6..ef68b1c8a5b 100644 --- a/apps/signage/src/tests/media-cache.service.spec.ts +++ b/apps/signage/src/tests/media-cache.service.spec.ts @@ -490,6 +490,9 @@ describe('MediaCacheService', () => { await expect(spectator.service.getFile('/blank.png')).resolves.toEqual( expect.any(File), ); + // The blank record is replaced, not left behind + expect(stored_files.has('blank-file')).toBe(false); + expect(stored_files.size).toBe(1); }); it('should reject empty downloads instead of storing blank files', async () => { @@ -532,8 +535,9 @@ describe('MediaCacheService', () => { const cache_promise = spectator.service.requestFilesToCache([ '/waiting.png', ]); - await Promise.resolve(); - await Promise.resolve(); + for (let i = 0; i < 20 && !fetch_spy.mock.calls.length; i++) { + await Promise.resolve(); + } expect(fetch_spy).toHaveBeenCalledWith( '/waiting.png', @@ -639,7 +643,7 @@ describe('MediaCacheService', () => { ]); }); - it('should keep earlier priority files when the owner cache is over size', async () => { + it('should not download a priority file that cannot fit', async () => { const fetch_spy = vi .fn() .mockResolvedValueOnce({ @@ -659,12 +663,14 @@ describe('MediaCacheService', () => { value: fetch_spy, }); - await spectator.service.requestFilesToCache( + const has_failures = await spectator.service.requestFilesToCache( ['/first.png', '/second.png', '/third.png'], 'display-1', { max_size: 8 }, ); + expect(has_failures).toBe(false); + expect(fetch_spy).toHaveBeenCalledTimes(2); expect(spectator.service.availableFiles('display-1')).toEqual([ '/first.png', '/second.png', @@ -673,6 +679,11 @@ describe('MediaCacheService', () => { '/first.png', '/second.png', ]); + // Plays from the network without waiting on a download + expect(spectator.service.isLoadingFile('/third.png')).toBe(false); + expect(spectator.service.cacheState().too_large).toEqual([ + '/third.png', + ]); }); it('should evict non-priority owner files before active playlist files', async () => { @@ -832,6 +843,19 @@ describe('MediaCacheService', () => { Promise.resolve(new Blob(['image'], { type: 'image/png' })), } as Response); + const streamed_response = ( + body: ReadableStream, + length = 0, + ) => + ({ + ok: true, + headers: new Headers({ + 'content-type': 'image/png', + ...(length ? { 'content-length': `${length}` } : {}), + }), + body, + }) as Response; + const cached_entry = (id: string, url: string) => ({ id, url, @@ -1027,19 +1051,15 @@ describe('MediaCacheService', () => { it('should abandon a download that stops sending data', async () => { vi.useFakeTimers(); try { - const cancel = vi.fn().mockResolvedValue(undefined); Object.defineProperty(globalThis, 'fetch', { configurable: true, - value: vi.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'image/png' }, - body: { - getReader: () => ({ - read: () => new Promise(() => undefined), - cancel, + value: vi.fn().mockResolvedValue( + streamed_response( + new ReadableStream({ + pull: () => new Promise(() => undefined), }), - }, - }), + ), + ), }); const cache_promise = spectator.service.requestFilesToCache([ @@ -1060,19 +1080,17 @@ describe('MediaCacheService', () => { const chunks = [new Uint8Array([1, 2]), new Uint8Array([3])]; Object.defineProperty(globalThis, 'fetch', { configurable: true, - value: vi.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'image/png' }, - body: { - getReader: () => ({ - read: async () => - chunks.length - ? { done: false, value: chunks.shift() } - : { done: true, value: undefined }, - cancel: vi.fn(), + value: vi.fn().mockResolvedValue( + streamed_response( + new ReadableStream({ + pull: (controller) => { + const chunk = chunks.shift(); + if (chunk) controller.enqueue(chunk); + else controller.close(); + }, }), - }, - }), + ), + ), }); const file = await spectator.service.fetchFile('/chunked.png'); @@ -1082,6 +1100,372 @@ describe('MediaCacheService', () => { expect(spectator.service.isCachedFile('/chunked.png')).toBe(true); }); + it('should let a slow download finish while data keeps arriving', async () => { + vi.useFakeTimers(); + try { + // 40 chunks, 30 seconds apart: 20 minutes in all + let sent = 0; + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: vi.fn().mockResolvedValue( + streamed_response( + new ReadableStream({ + pull: async (controller) => { + await new Promise((resolve) => + setTimeout(resolve, 30_000), + ); + controller.enqueue(new Uint8Array([1])); + if (++sent >= 40) controller.close(); + }, + }), + ), + ), + }); + + const cache_promise = spectator.service.requestFilesToCache([ + '/slow.mp4', + ]); + await vi.advanceTimersByTimeAsync(21 * 60_000); + + await expect(cache_promise).resolves.toBe(false); + expect(spectator.service.cacheState().files[0].size).toBe(40); + } finally { + vi.useRealTimers(); + } + }); + + it('should not download a file whose length cannot fit', async () => { + const body = new ReadableStream(); + const fetch_spy = vi + .fn() + .mockResolvedValue(streamed_response(body, 100)); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: fetch_spy, + }); + + const has_failures = await spectator.service.requestFilesToCache( + ['/huge.mp4'], + 'display-1', + { max_size: 50 }, + ); + await spectator.service.requestFilesToCache( + ['/huge.mp4'], + 'display-1', + { max_size: 50 }, + ); + + // Refused on its headers, and not asked for again + expect(has_failures).toBe(false); + expect(fetch_spy).toHaveBeenCalledTimes(1); + expect(body.locked).toBe(false); + expect(spectator.service.isLoadingFile('/huge.mp4')).toBe(false); + // Playback streams it from the server instead + await expect( + spectator.service.fetchFile('/huge.mp4', 'display-1'), + ).resolves.toBeNull(); + expect(fetch_spy).toHaveBeenCalledTimes(1); + }); + + it('should evict files outside the playlist to make room before storing', async () => { + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: good_fetch(), + }); + stored_files.set('stale', { + name: 'stale', + url: '/stale.png', + owner: 'display-1', + owners: ['display-1'], + file: new File(['12345'], 'stale'), + }); + spectator.service['_file_cache_index'].set([ + cached_entry('stale', '/stale.png'), + ]); + + const has_failures = await spectator.service.requestFilesToCache( + ['/new.png'], + 'display-1', + { max_size: 8 }, + ); + + expect(has_failures).toBe(false); + expect(spectator.service.availableFiles()).toEqual(['/new.png']); + expect([...stored_files.values()].map((_) => _.url)).toEqual([ + '/new.png', + ]); + }); + + it('should evict and retry once when storage is full', async () => { + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: good_fetch(), + }); + stored_files.set('stale', { + name: 'stale', + url: '/stale.png', + owner: 'display-1', + owners: ['display-1'], + file: new File(['12345'], 'stale'), + }); + spectator.service['_file_cache_index'].set([ + cached_entry('stale', '/stale.png'), + ]); + const store_file = spectator.service['_storeFile'].bind( + spectator.service, + ); + spectator.service['_storeFile'] = vi + .fn() + .mockRejectedValueOnce( + new DOMException('Storage is full', 'QuotaExceededError'), + ) + .mockImplementation(store_file); + + const has_failures = await spectator.service.requestFilesToCache( + ['/new.png'], + 'display-1', + ); + + expect(has_failures).toBe(false); + expect(spectator.service.availableFiles()).toEqual(['/new.png']); + expect([...stored_files.values()].map((_) => _.url)).toEqual([ + '/new.png', + ]); + }); + + // Chrome reports a disk filling during a blob write as a DataError + it.each(['QuotaExceededError', 'DataError'])( + 'should stop retrying a file that storage has no room for (%s)', + async (error_name) => { + const fetch_spy = good_fetch(); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: fetch_spy, + }); + spectator.service['_storeFile'] = vi + .fn() + .mockRejectedValue( + new DOMException('Storage is full', error_name), + ); + + const has_failures = + await spectator.service.requestFilesToCache( + ['/full.png'], + 'display-1', + ); + await spectator.service.requestFilesToCache( + ['/full.png'], + 'display-1', + ); + + // Not a failure: a retry would download it again to no end + expect(has_failures).toBe(false); + expect(fetch_spy).toHaveBeenCalledTimes(1); + expect(spectator.service.isLoadingFile('/full.png')).toBe( + false, + ); + }, + ); + + describe('when the browser cannot build a Blob from a stream', () => { + const stream_of = (bytes: number) => + new ReadableStream({ + start: (controller) => { + controller.enqueue(new Uint8Array(bytes)); + controller.close(); + }, + }); + + beforeEach(() => { + // Chrome rejects like this when its blob storage is full + vi.stubGlobal( + 'Response', + class { + blob() { + return Promise.reject( + new TypeError('Failed to fetch'), + ); + } + }, + ); + }); + + afterEach(() => vi.unstubAllGlobals()); + + it('should download a small file again into memory', async () => { + const fetch_spy = vi.fn(() => + Promise.resolve(streamed_response(stream_of(3), 3)), + ); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: fetch_spy, + }); + + const has_failures = + await spectator.service.requestFilesToCache(['/small.png']); + + expect(has_failures).toBe(false); + expect(fetch_spy).toHaveBeenCalledTimes(2); + expect(spectator.service.isCachedFile('/small.png')).toBe(true); + expect(spectator.service.cacheState().files[0].size).toBe(3); + }); + + it('should stream a large file from the network without retrying', async () => { + // Too large to read into memory on a low-memory player + const length = 100 * 1024 * 1024; + const fetch_spy = vi.fn(() => + Promise.resolve(streamed_response(stream_of(3), length)), + ); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: fetch_spy, + }); + + const has_failures = + await spectator.service.requestFilesToCache(['/large.mp4']); + await spectator.service.requestFilesToCache(['/large.mp4']); + + // The fallback is refused on its headers, and the file is + // not asked for again + expect(has_failures).toBe(false); + expect(fetch_spy).toHaveBeenCalledTimes(2); + expect(spectator.service.isLoadingFile('/large.mp4')).toBe( + false, + ); + await expect( + spectator.service.fetchFile('/large.mp4'), + ).resolves.toBeNull(); + expect(fetch_spy).toHaveBeenCalledTimes(2); + }); + }); + + it('should not download a file that playback cached during the stagger delay', async () => { + vi.useFakeTimers(); + try { + const fetch_spy = good_fetch(); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: fetch_spy, + }); + + const cache_promise = spectator.service.requestFilesToCache( + ['/first.png', '/second.png'], + 'display-1', + ); + // The first file is cached; the sync waits before the second + await vi.advanceTimersByTimeAsync(100); + await spectator.service.fetchFile('/second.png', 'display-1'); + await vi.advanceTimersByTimeAsync(1_000); + + await expect(cache_promise).resolves.toBe(false); + expect(fetch_spy).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it('should not download a file that playback found too large during the stagger delay', async () => { + vi.useFakeTimers(); + try { + const fetch_spy = good_fetch(); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: fetch_spy, + }); + + const cache_promise = spectator.service.requestFilesToCache( + ['/first.png', '/second.png'], + 'display-1', + ); + // The first file is cached; the sync waits before the second + await vi.advanceTimersByTimeAsync(100); + spectator.service['_markTooLarge']('/second.png', 1e15); + await vi.advanceTimersByTimeAsync(1_000); + + await expect(cache_promise).resolves.toBe(false); + expect(fetch_spy).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + + it('should not download a file again when the store cannot be read', async () => { + const fetch_spy = good_fetch(); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: fetch_spy, + }); + stored_files.set('kept', { + name: 'kept', + url: '/kept.png', + owner: 'display-1', + file: new File(['image'], 'kept'), + }); + spectator.service['_file_cache_index'].set([ + cached_entry('kept', '/kept.png'), + ]); + spectator.service['_storedFileExists'] = vi + .fn() + .mockRejectedValue(new Error('read failed')); + + const has_failures = await spectator.service.requestFilesToCache( + ['/kept.png'], + 'display-1', + ); + + // Retried later rather than orphaning the stored copy + expect(has_failures).toBe(true); + expect(fetch_spy).not.toHaveBeenCalled(); + expect(spectator.service.isCachedFile('/kept.png')).toBe(true); + expect(stored_files.has('kept')).toBe(true); + }); + + it('should delete records no entry can use when loading the store', async () => { + const record = (name: string, url: string, content: string[]) => ({ + name, + url, + owner: 'display-1', + file: new File(content, name), + }); + stored_files.set('first', record('first', '/a.png', ['image'])); + stored_files.set('copy', record('copy', '/a.png', ['image'])); + stored_files.set('no-url', record('no-url', '', ['image'])); + stored_files.set('blank', record('blank', '/b.png', [])); + + await spectator.service['_loadCacheMetadataFromStore'](); + + expect([...stored_files.keys()]).toEqual(['first']); + expect(spectator.service.availableFiles()).toEqual(['/a.png']); + }); + + it('should size the budget from the storage estimate', async () => { + const persist = vi.fn().mockResolvedValue(true); + Object.defineProperty(navigator, 'storage', { + configurable: true, + value: { + estimate: () => + Promise.resolve({ quota: 1_000, usage: 110 }), + persist, + }, + }); + const service = TestBed.runInInjectionContext( + () => new MediaCacheService(), + ); + try { + service['_file_cache_index'].set([ + { ...cached_entry('a', '/a.png'), size: 10 }, + ]); + + // 80% of the quota, less 100 bytes used outside the cache + await expect(service['_storageBudget']()).resolves.toBe(720); + expect(service.cacheState().limit_bytes).toBe(720); + expect(persist).toHaveBeenCalledTimes(1); + } finally { + service.ngOnDestroy(); + delete (navigator as { storage?: StorageManager }).storage; + } + }); + it('should recreate the database when it cannot be opened', async () => { const open_requests: any[] = []; const delete_request: any = {}; From 9fb881c47a9449612f42096d2b71b42e5a4a5904 Mon Sep 17 00:00:00 2001 From: Alex Sorafumo Date: Fri, 2 Oct 2026 00:40:26 +1000 Subject: [PATCH 2/3] fix(signage): keep playback in the cache budget and fall back to network - Playback and cache syncs share one budget check. Playback never evicts and marks a file too large when storage is full. It allowed a download as large as the whole budget and retried full storage. - Evict before reading a body whose Content-Length needs the room. On a nearly full disk the browser can fail to build a Blob that would fit once older files are gone. - Template backgrounds play from the server when the cache has no copy. They went blank. - Tests: hand the service jsdom's Blob. Before Node 24, Response.blob() returns Node's Blob, which jsdom's File stores as "[object Blob]". --- apps/signage/USER_STORIES.md | 3 + apps/signage/src/app/media-cache.service.ts | 76 +++++++---- apps/signage/src/app/template.component.ts | 8 +- .../src/tests/media-cache.service.spec.ts | 122 ++++++++++++++++++ .../src/tests/template.component.spec.ts | 13 ++ 5 files changed, 194 insertions(+), 28 deletions(-) diff --git a/apps/signage/USER_STORIES.md b/apps/signage/USER_STORIES.md index 1e4b348d58d..206c05ed9a2 100644 --- a/apps/signage/USER_STORIES.md +++ b/apps/signage/USER_STORIES.md @@ -476,6 +476,9 @@ The Signage app is a kiosk-style digital signage player. It bootstraps a device - At startup, the app asks the browser for persistent storage. - Pruning never removes media in the current request. It removes files of other displays first (root players only), then the largest files. - Media that cannot fit in the budget is not downloaded. It plays from the network. The cache tries it again only when more space is available. +- When the file size is known, the cache removes files to make room before it reads the download. +- Playback downloads a missing file only if it fits in the space that is left in the budget. Playback never removes cached files. A full storage write during playback stops later playback downloads of that file. +- A template background that the cache cannot supply plays from the network. - If storage becomes full during a write (`QuotaExceededError`, or a `DataError` from a failed blob write), the cache removes the files that the request does not need and tries one more time. If the write fails again, the media plays from the network. - If the cache database cannot be read, the cache does not download the file again. The sync tries again later. - Stored files that no cache entry uses (duplicates, empty files, and replaced files) are deleted. diff --git a/apps/signage/src/app/media-cache.service.ts b/apps/signage/src/app/media-cache.service.ts index 1b51ddbe0c9..6e18f5ffe99 100644 --- a/apps/signage/src/app/media-cache.service.ts +++ b/apps/signage/src/app/media-cache.service.ts @@ -405,9 +405,8 @@ export class MediaCacheService extends AsyncHandler { } } } - const room = - budget - this._pinnedBytes(owner, url_list, prune_others); - if (!this._mayFit(url, room)) continue; + const fit = this._cacheFit(owner, url_list, budget, prune_others); + if (!this._mayFit(url, fit.max_bytes)) continue; // Stagger requests for uncached resources to avoid overwhelming the network if (uncached_count > 0) await delay(STAGGER_DELAY_MS); uncached_count++; @@ -418,17 +417,8 @@ export class MediaCacheService extends AsyncHandler { await this._addOwner(latest, owner); continue; } - if (!this._mayFit(url, room)) continue; - const { stored, no_room } = await this._cacheFile(url, owner, { - max_bytes: room, - make_room: (bytes) => - this.pruneCache( - owner, - url_list, - budget - bytes, - prune_others, - ), - }); + if (!this._mayFit(url, fit.max_bytes)) continue; + const { stored, no_room } = await this._cacheFile(url, owner, fit); if (!stored && !no_room) failures = true; } this._file_cache_index.set([...this._cache_index]); @@ -448,8 +438,10 @@ export class MediaCacheService extends AsyncHandler { * does not. Unlike `requestFilesToCache` this hands back a download that * could not be stored, so a broken database never stops media playing. * Resolves null for a file too large to cache, which plays from the - * network instead. Waits at most `wait_ms` for a download that is already - * in progress and returns null if it has not finished by then. + * network instead. Playback never evicts: the file must fit in what the + * budget has left, and only a sync makes more room. Waits at most + * `wait_ms` for a download that is already in progress and returns null + * if it has not finished by then. */ public async fetchFile( url: string, @@ -471,9 +463,15 @@ export class MediaCacheService extends AsyncHandler { if (file) return file; } if (this._too_large.has(url)) return null; - const { file } = await this._cacheFile(url, owner, { - max_bytes: this._budget_bytes, - }); + // Everything cached counts as needed, so nothing is evicted + const fit = this._cacheFit( + owner, + this._cache_index.map((_) => _.url), + this._budget_bytes, + false, + ); + if (!this._mayFit(url, fit.max_bytes)) return null; + const { file } = await this._cacheFile(url, owner, fit); return file; } @@ -714,7 +712,7 @@ export class MediaCacheService extends AsyncHandler { cacheStatus(cache_item, 'downloading'); // If not an API call, just load the image if (url.includes(UPLOADS_PATH)) this.applyAuthenticationCookie(); - const blob = await this._download(url, fit.max_bytes); + const blob = await this._download(url, fit); if (blob.size <= 0) { log.error(`Downloaded resource is empty.`, url); throw new Error('Downloaded media file is empty'); @@ -773,9 +771,9 @@ export class MediaCacheService extends AsyncHandler { * cannot build a Blob from the stream, a small file is downloaded once * more into memory; a large one fails with a `NoRoomError`. */ - private async _download(url: string, max_bytes: number): Promise { + private async _download(url: string, fit: CacheFit): Promise { try { - return await this._fetchBlob(url, max_bytes, false); + return await this._fetchBlob(url, fit, false); } catch (e) { if (!(e instanceof BlobStorageError)) throw e; log.warn( @@ -783,7 +781,7 @@ export class MediaCacheService extends AsyncHandler { url, e, ); - return this._fetchBlob(url, max_bytes, true); + return this._fetchBlob(url, fit, true); } } @@ -794,9 +792,10 @@ export class MediaCacheService extends AsyncHandler { */ private async _fetchBlob( url: string, - max_bytes: number, + fit: CacheFit, in_memory: boolean, ): Promise { + const { max_bytes } = fit; const controller = typeof AbortController === 'function' ? new AbortController() @@ -818,6 +817,10 @@ export class MediaCacheService extends AsyncHandler { abort(); throw new NoRoomError(length); } + // Make room before the body arrives. On a nearly full disk the + // browser can fail to build a large Blob that would fit once older + // files are gone. + if (length > 0) await fit.make_room?.(length); const type = response.headers?.get?.('content-type') || ''; const body = response.body; if (in_memory) { @@ -900,6 +903,31 @@ export class MediaCacheService extends AsyncHandler { .sort((a, b) => own(a) - own(b) || (b.size || 0) - (a.size || 0)); } + /** + * How much a download for `owner` may store within `budget`, and how it + * makes room: by evicting cached entries outside `priority_urls`. Shared + * by cache syncs and playback so both keep to the same budget. + */ + private _cacheFit( + owner: string, + priority_urls: string[], + budget: number, + prune_other_owners: boolean, + ): CacheFit { + return { + max_bytes: + budget - + this._pinnedBytes(owner, priority_urls, prune_other_owners), + make_room: (bytes) => + this.pruneCache( + owner, + priority_urls, + budget - bytes, + prune_other_owners, + ), + }; + } + /** Bytes held by cached entries that a request may not evict */ private _pinnedBytes( owner: string, diff --git a/apps/signage/src/app/template.component.ts b/apps/signage/src/app/template.component.ts index be61027eca8..6b5d0f85faa 100644 --- a/apps/signage/src/app/template.component.ts +++ b/apps/signage/src/app/template.component.ts @@ -87,10 +87,10 @@ function backgroundPlayerItem( .catch(() => null); } try { - return file ? URL.createObjectURL(file) : ''; - } catch { - return ''; - } + if (file) return URL.createObjectURL(file); + } catch {} + // Not cached (for example too large to cache): play from the server + return media_cache.directURL(media.media_url); }, isLoading: cacheable ? () => media_cache.isLoadingFile(media.media_url) diff --git a/apps/signage/src/tests/media-cache.service.spec.ts b/apps/signage/src/tests/media-cache.service.spec.ts index ef68b1c8a5b..a685d8cb8b0 100644 --- a/apps/signage/src/tests/media-cache.service.spec.ts +++ b/apps/signage/src/tests/media-cache.service.spec.ts @@ -7,6 +7,22 @@ import { Subject } from 'rxjs'; import { MediaCacheService } from '../app/media-cache.service'; +/** + * Before Node 24, `Response.blob()` returns Node's own Blob. jsdom's File does + * not take that as a part and stores the text "[object Blob]" instead. A + * browser has a single Blob type, so hand the service jsdom's. + */ +class JsdomResponse extends Response { + override async blob() { + const blob = await super.blob(); + if (blob instanceof Blob) return blob; + const node_blob = blob as unknown as Blob; + return new Blob([await node_blob.arrayBuffer()], { + type: node_blob.type, + }); + } +} + describe('MediaCacheService', () => { let spectator: SpectatorService; const stored_files = new Map< @@ -124,11 +140,13 @@ describe('MediaCacheService', () => { transaction: vi.fn(create_transaction), } as any; spectator.service['_cache_db_ready'] = Promise.resolve(); + vi.stubGlobal('Response', JsdomResponse); }); afterEach(() => { spectator.service.ngOnDestroy(); vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); it('should not read stored files when re-confirming a cached playlist', async () => { @@ -1196,6 +1214,110 @@ describe('MediaCacheService', () => { ]); }); + it('should evict files outside the playlist before reading a body that needs the room', async () => { + stored_files.set('stale', { + name: 'stale', + url: '/stale.png', + owner: 'display-1', + owners: ['display-1'], + file: new File(['12345'], 'stale'), + }); + spectator.service['_file_cache_index'].set([ + cached_entry('stale', '/stale.png'), + ]); + let stale_kept_while_reading: boolean; + // A zero high-water mark: pulled only once the body is read + const body = new ReadableStream( + { + pull: (controller) => { + stale_kept_while_reading = stored_files.has('stale'); + controller.enqueue(new Uint8Array(5)); + controller.close(); + }, + }, + { highWaterMark: 0 }, + ); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: vi.fn().mockResolvedValue(streamed_response(body, 5)), + }); + + await spectator.service.requestFilesToCache( + ['/new.png'], + 'display-1', + { max_size: 8 }, + ); + + expect(stale_kept_while_reading).toBe(false); + expect(spectator.service.availableFiles()).toEqual(['/new.png']); + }); + + it('should keep playback downloads inside what the budget has left', async () => { + stored_files.set('playing', { + name: 'playing', + url: '/playing.png', + owner: 'display-1', + owners: ['display-1'], + file: new File(['12345'], 'playing'), + }); + spectator.service['_file_cache_index'].set([ + cached_entry('playing', '/playing.png'), + ]); + spectator.service['_budget_bytes'] = 8; + const body = new ReadableStream({ + start: (controller) => { + controller.enqueue(new Uint8Array(5)); + controller.close(); + }, + }); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: vi.fn().mockResolvedValue(streamed_response(body, 5)), + }); + + const file = await spectator.service.fetchFile( + '/new.png', + 'display-1', + ); + + // Plays from the network; nothing cached is evicted for it + expect(file).toBeNull(); + expect([...stored_files.keys()]).toEqual(['playing']); + expect(spectator.service.availableFiles()).toEqual([ + '/playing.png', + ]); + }); + + it('should stop playback downloading a file that storage has no room for', async () => { + const fetch_spy = good_fetch(); + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: fetch_spy, + }); + spectator.service['_storeFile'] = vi + .fn() + .mockRejectedValue( + new DOMException('Storage is full', 'DataError'), + ); + + const first = await spectator.service.fetchFile( + '/full.png', + 'display-1', + ); + const second = await spectator.service.fetchFile( + '/full.png', + 'display-1', + ); + + // The download is still handed back once, then it streams + expect(first).toEqual(expect.any(File)); + expect(second).toBeNull(); + expect(fetch_spy).toHaveBeenCalledTimes(1); + expect(spectator.service.cacheState().too_large).toEqual([ + '/full.png', + ]); + }); + it('should evict and retry once when storage is full', async () => { Object.defineProperty(globalThis, 'fetch', { configurable: true, diff --git a/apps/signage/src/tests/template.component.spec.ts b/apps/signage/src/tests/template.component.spec.ts index a8a7f104b0a..a495e3eb6f5 100644 --- a/apps/signage/src/tests/template.component.spec.ts +++ b/apps/signage/src/tests/template.component.spec.ts @@ -120,6 +120,19 @@ describe('SignageTemplateComponent', () => { shell_frame = null; }); + it('plays the background from the server when the cache has no copy', async () => { + spectator = create_component({ + params: { template_id: 'template-1', system_id: 'display-1' }, + }); + await vi.waitFor(() => { + expect(spectator.component.background_playlist()).toHaveLength(1); + }); + + await expect( + spectator.component.background_playlist()[0].getURL(), + ).resolves.toBe('/api/background'); + }); + it('loads the template, background, and layout plugins', async () => { spectator = create_component({ params: { template_id: 'template-1', system_id: 'display-1' }, From 64745c9c8d646ddc13823ef4c12b53a931b33087 Mon Sep 17 00:00:00 2001 From: Alex Sorafumo Date: Fri, 2 Oct 2026 00:52:01 +1000 Subject: [PATCH 3/3] fix(signage): stop parallel downloads spending the same cache room A sync and playback could each work out the free room, download different files, and both store them over the budget. The room is now checked again just before storing, counting entries being stored, and a file claims its size before the store yields. The sync also works out the room again after its stagger delay. --- apps/signage/src/app/media-cache.service.ts | 23 +++++++++++++++- .../src/tests/media-cache.service.spec.ts | 27 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/apps/signage/src/app/media-cache.service.ts b/apps/signage/src/app/media-cache.service.ts index 6e18f5ffe99..ab08f65f4d2 100644 --- a/apps/signage/src/app/media-cache.service.ts +++ b/apps/signage/src/app/media-cache.service.ts @@ -115,6 +115,12 @@ interface CacheFit { * budget. `Infinity` evicts every entry the request may evict. */ make_room?: (bytes: number) => Promise; + /** + * Whether `bytes` more fit in the budget now. Checked again just before + * storing, as another download may have used the room since `max_bytes` + * was worked out. + */ + fits?: (bytes: number) => boolean; } /** A response body stream, typed as the browser hands it over */ @@ -405,7 +411,7 @@ export class MediaCacheService extends AsyncHandler { } } } - const fit = this._cacheFit(owner, url_list, budget, prune_others); + let fit = this._cacheFit(owner, url_list, budget, prune_others); if (!this._mayFit(url, fit.max_bytes)) continue; // Stagger requests for uncached resources to avoid overwhelming the network if (uncached_count > 0) await delay(STAGGER_DELAY_MS); @@ -417,6 +423,7 @@ export class MediaCacheService extends AsyncHandler { await this._addOwner(latest, owner); continue; } + fit = this._cacheFit(owner, url_list, budget, prune_others); if (!this._mayFit(url, fit.max_bytes)) continue; const { stored, no_room } = await this._cacheFile(url, owner, fit); if (!stored && !no_room) failures = true; @@ -721,6 +728,12 @@ export class MediaCacheService extends AsyncHandler { // Wraps the blob without copying it file = new File([blob], cache_item.id, { type: blob.type }); await fit.make_room?.(file.size); + if (fit.fits && !fit.fits(file.size)) { + throw new NoRoomError(file.size); + } + // Claim the room before the store yields, so a download storing + // at the same time counts it + cache_item.size = file.size; try { await this._storeFile(cache_item, file, url); } catch (e) { @@ -872,6 +885,13 @@ export class MediaCacheService extends AsyncHandler { } } + /** Bytes held by cached entries and those being stored */ + private _claimedBytes() { + return this._cache_index + .filter((_) => _.status === 'cached' || _.status === 'storing') + .reduce((total, _) => total + (_.size || 0), 0); + } + private _cachedBytes() { return this._cache_index .filter((_) => _.status === 'cached') @@ -925,6 +945,7 @@ export class MediaCacheService extends AsyncHandler { budget - bytes, prune_other_owners, ), + fits: (bytes) => this._claimedBytes() + bytes <= budget, }; } diff --git a/apps/signage/src/tests/media-cache.service.spec.ts b/apps/signage/src/tests/media-cache.service.spec.ts index a685d8cb8b0..ac12f5fed88 100644 --- a/apps/signage/src/tests/media-cache.service.spec.ts +++ b/apps/signage/src/tests/media-cache.service.spec.ts @@ -1288,6 +1288,33 @@ describe('MediaCacheService', () => { ]); }); + it('should not let two downloads spend the same room', async () => { + spectator.service['_budget_bytes'] = 8; + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + value: vi.fn().mockImplementation(async () => + streamed_response( + new ReadableStream({ + start: (controller) => { + controller.enqueue(new Uint8Array(5)); + controller.close(); + }, + }), + 5, + ), + ), + }); + + // Each fits on its own when it starts, but not both together + await Promise.all([ + spectator.service.fetchFile('/a.png', 'display-1'), + spectator.service.fetchFile('/b.png', 'display-1'), + ]); + + expect(stored_files.size).toBe(1); + expect(spectator.service.availableFiles()).toHaveLength(1); + }); + it('should stop playback downloading a file that storage has no room for', async () => { const fetch_spy = good_fetch(); Object.defineProperty(globalThis, 'fetch', {