From cc5bcfa590bf3f48f7c36ffd344a2fb49309c64c Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 17:26:46 +0100 Subject: [PATCH 01/21] fix(build): re-sign the compiled CLI ad hoc on macOS when Bun leaves an invalid signature On macOS 27 (arm64, Bun 1.3.13) the compiled binary keeps the linker's ad-hoc signature after Bun appends the payload. codesign --verify reports "code or signature have been modified" and the kernel kills the binary on launch with status 137. compile.sh now verifies the signature on darwin and re-signs ad hoc only when verification fails. --- .changeset/macos-compiled-binary-signature.md | 5 +++++ packages/podkit-cli/scripts/compile.sh | 8 ++++++++ 2 files changed, 13 insertions(+) create mode 100644 .changeset/macos-compiled-binary-signature.md diff --git a/.changeset/macos-compiled-binary-signature.md b/.changeset/macos-compiled-binary-signature.md new file mode 100644 index 00000000..ac273614 --- /dev/null +++ b/.changeset/macos-compiled-binary-signature.md @@ -0,0 +1,5 @@ +--- +'podkit': patch +--- + +Re-sign the compiled macOS binary ad hoc when Bun leaves an invalid signature behind, so the CLI no longer exits with status 137 on launch on Apple Silicon. diff --git a/packages/podkit-cli/scripts/compile.sh b/packages/podkit-cli/scripts/compile.sh index 8107c28b..0d61a199 100755 --- a/packages/podkit-cli/scripts/compile.sh +++ b/packages/podkit-cli/scripts/compile.sh @@ -172,4 +172,12 @@ PODKIT_COMPILE_VERSION="$VERSION" \ PODKIT_COMPILE_DEV_HOOKS="$DEV_HOOKS_DEFINE" \ bun scripts/compile-build.ts +# Bun can leave the linker's ad-hoc signature in place after appending the +# payload; arm64 macOS then SIGKILLs the binary on launch (exit 137). +if [ "$PLATFORM" = "darwin" ] && ! codesign --verify "$OUTFILE" 2>/dev/null; then + codesign --force --sign - "$OUTFILE" + codesign --verify "$OUTFILE" + echo "Re-signed ad hoc: $OUTFILE" +fi + echo "Compiled: $OUTFILE (v$VERSION, $BUILD_LABEL)" From 22dab378f96fe5b3ff554229ba7884c0fa312174 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 17:26:46 +0100 Subject: [PATCH 02/21] fix(core): treat an iTunesCDB-only library as an existing iPod database A Nano 7 synced by macOS carries iTunes Library.itlp and iTunesCDB but no iTunesDB; libgpod parses the CDB, yet IpodDatabase.hasDatabase only probed iTunesDB. device scan reported "iTunesDB not found" and sync refused with IPOD_NEEDS_INIT, pointing at device init for a device with 21 tracks. hasDatabase now accepts either file. --- .changeset/itunescdb-only-library.md | 6 ++++ .../podkit-core/src/ipod/database.test.ts | 35 +++++++++++++++++++ packages/podkit-core/src/ipod/database.ts | 18 ++++++---- 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 .changeset/itunescdb-only-library.md diff --git a/.changeset/itunescdb-only-library.md b/.changeset/itunescdb-only-library.md new file mode 100644 index 00000000..6b375d28 --- /dev/null +++ b/.changeset/itunescdb-only-library.md @@ -0,0 +1,6 @@ +--- +'@podkit/core': patch +'podkit': patch +--- + +Recognise an iPod library that only has the compressed `iTunesCDB`, as Apple software leaves it on an iPod nano 6th/7th generation. Such devices were reported as uninitialised, and `podkit device scan`/`sync` suggested `podkit device init`. diff --git a/packages/podkit-core/src/ipod/database.test.ts b/packages/podkit-core/src/ipod/database.test.ts index 9da9b2e0..b38b22e3 100644 --- a/packages/podkit-core/src/ipod/database.test.ts +++ b/packages/podkit-core/src/ipod/database.test.ts @@ -10,6 +10,8 @@ */ import { describe, it, expect } from 'bun:test'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { IpodDatabase, @@ -64,6 +66,39 @@ describe('IpodDatabase', () => { }); }); + describe('hasDatabase()', () => { + async function withDatabaseFiles(names: string[], check: (mount: string) => Promise) { + const mount = await mkdtemp(join(tmpdir(), 'podkit-hasdb-')); + try { + await mkdir(join(mount, 'iPod_Control', 'iTunes'), { recursive: true }); + for (const name of names) { + await writeFile(join(mount, 'iPod_Control', 'iTunes', name), ''); + } + await check(mount); + } finally { + await rm(mount, { recursive: true, force: true }); + } + } + + it('recognises a library that only has the compressed iTunesCDB', async () => { + await withDatabaseFiles(['iTunesCDB'], async (mount) => { + expect(await IpodDatabase.hasDatabase(mount)).toBe(true); + }); + }); + + it('recognises a plain iTunesDB', async () => { + await withDatabaseFiles(['iTunesDB'], async (mount) => { + expect(await IpodDatabase.hasDatabase(mount)).toBe(true); + }); + }); + + it('reports a blank device', async () => { + await withDatabaseFiles([], async (mount) => { + expect(await IpodDatabase.hasDatabase(mount)).toBe(false); + }); + }); + }); + describe('setDeviceName()', () => { it('throws DATABASE_CLOSED when the database has been closed', () => { const instance = makeInstance({}, true /* closed */); diff --git a/packages/podkit-core/src/ipod/database.ts b/packages/podkit-core/src/ipod/database.ts index c8bc8963..a06c4ed8 100644 --- a/packages/podkit-core/src/ipod/database.ts +++ b/packages/podkit-core/src/ipod/database.ts @@ -229,16 +229,20 @@ export class IpodDatabase implements IpodDatabaseInternal, PlaylistDatabaseInter * Checks if an iPod database exists at the given mount point. * * @param mountPoint - Path to check for iPod database - * @returns true if an iTunesDB exists at the path + * @returns true if an iTunesDB or a compressed iTunesCDB exists at the path */ static async hasDatabase(mountPoint: string): Promise { - try { - const itunesDbPath = `${mountPoint}/iPod_Control/iTunes/iTunesDB`; - await fs.promises.access(itunesDbPath); - return true; - } catch { - return false; + // Nano 6/7 libraries written by Apple software carry only iTunesCDB, + // which libgpod parses in preference to iTunesDB. + for (const name of ['iTunesCDB', 'iTunesDB']) { + try { + await fs.promises.access(`${mountPoint}/iPod_Control/iTunes/${name}`); + return true; + } catch { + // try the next name + } } + return false; } /** From a7754e4b0d247fc6cf6798a096e897c65d9d23e2 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 17:26:46 +0100 Subject: [PATCH 03/21] fix(doctor): pass the injected identity assessor into the repair pipeline runRepair ran its own refusal check with the injected assessIpodIdentity but called runRepairPipeline without deps, so the pipeline's second check assessed the real device path. The supported-device test therefore failed on any host without an iPod at /Volumes/iPod. The assessor is now forwarded through the pipeline's existing deps; both checks stay, and the test counts that both used the injected assessor. --- .../podkit-cli/src/commands/doctor-repair.test.ts | 12 +++++++++--- packages/podkit-cli/src/commands/doctor.ts | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/podkit-cli/src/commands/doctor-repair.test.ts b/packages/podkit-cli/src/commands/doctor-repair.test.ts index 21b3ac76..eb7e3835 100644 --- a/packages/podkit-cli/src/commands/doctor-repair.test.ts +++ b/packages/podkit-cli/src/commands/doctor-repair.test.ts @@ -275,6 +275,7 @@ describe('runRepair — refusal short-circuits IpodDatabase.open', () => { it('still calls IpodDatabase.open when the device is supported', async () => { let openCalls = 0; + let assessCalls = 0; const check = makeFakeCheck(); const fakeIpodDatabase = { @@ -300,12 +301,17 @@ describe('runRepair — refusal short-circuits IpodDatabase.open', () => { { music: {} } as ReturnType['config'], { loadCore: async () => fakeCore as unknown as typeof import('@podkit/core'), - assessIpodIdentity: async () => - ({ + assessIpodIdentity: async () => { + assessCalls++; + return { model: { unsupportedReason: undefined }, - }) as unknown as IpodIdentityAssessment, + } as unknown as IpodIdentityAssessment; + }, } ); expect(openCalls).toBe(1); + // Both refusal checks (before the open and inside the repair pipeline) + // use the injected assessor, so neither reads real hardware. + expect(assessCalls).toBe(2); }); }); diff --git a/packages/podkit-cli/src/commands/doctor.ts b/packages/podkit-cli/src/commands/doctor.ts index a1c26a11..5d59623c 100644 --- a/packages/podkit-cli/src/commands/doctor.ts +++ b/packages/podkit-cli/src/commands/doctor.ts @@ -1314,6 +1314,7 @@ export async function runRepair( dryRun, verbose, out, + ...(deps.assessIpodIdentity ? { deps: { assessIpodIdentity: deps.assessIpodIdentity } } : {}), }); } finally { for (const adapter of adapters) { From 22cad5a3b531cc90da107e58ed6bd91aab5b9a74 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 19:53:30 +0100 Subject: [PATCH 04/21] build(libgpod): vendor the open HashAB signer for iPod nano 6G/7G databases The nano 6th and 7th generation sign iTunesCDB and Locations.itdb.cbk with HashAB. libgpod 0.8.3 only computes it through an external blob loaded from LIBGPOD_BLOB_DIR, which podkit does not ship. dstaley/hashab is an Unlicense C implementation of the same calcHashAB function. The C sources, headers, licence and the 100 upstream reference vectors are copied byte for byte from the pinned commit; UPSTREAM.md records the source. Nothing builds them yet. --- tools/prebuild/vendor/hashab/LICENSE | 24 + tools/prebuild/vendor/hashab/UPSTREAM.md | 13 + tools/prebuild/vendor/hashab/src/byteorder.h | 14 + tools/prebuild/vendor/hashab/src/calcHashAB.c | 60 +++ tools/prebuild/vendor/hashab/src/calcHashAB.h | 12 + tools/prebuild/vendor/hashab/src/cipher.c | 112 ++++ tools/prebuild/vendor/hashab/src/cipher.h | 8 + .../vendor/hashab/src/cipher_tables.c | 449 ++++++++++++++++ .../vendor/hashab/src/cipher_tables.h | 19 + tools/prebuild/vendor/hashab/src/edonr.c | 133 +++++ tools/prebuild/vendor/hashab/src/edonr.h | 12 + tools/prebuild/vendor/hashab/src/expand.c | 56 ++ tools/prebuild/vendor/hashab/src/expand.h | 7 + tools/prebuild/vendor/hashab/src/hash.c | 166 ++++++ tools/prebuild/vendor/hashab/src/hash.h | 7 + tools/prebuild/vendor/hashab/src/reduce.c | 197 +++++++ tools/prebuild/vendor/hashab/src/reduce.h | 5 + tools/prebuild/vendor/hashab/test-data.json | 502 ++++++++++++++++++ 18 files changed, 1796 insertions(+) create mode 100644 tools/prebuild/vendor/hashab/LICENSE create mode 100644 tools/prebuild/vendor/hashab/UPSTREAM.md create mode 100644 tools/prebuild/vendor/hashab/src/byteorder.h create mode 100644 tools/prebuild/vendor/hashab/src/calcHashAB.c create mode 100644 tools/prebuild/vendor/hashab/src/calcHashAB.h create mode 100644 tools/prebuild/vendor/hashab/src/cipher.c create mode 100644 tools/prebuild/vendor/hashab/src/cipher.h create mode 100644 tools/prebuild/vendor/hashab/src/cipher_tables.c create mode 100644 tools/prebuild/vendor/hashab/src/cipher_tables.h create mode 100644 tools/prebuild/vendor/hashab/src/edonr.c create mode 100644 tools/prebuild/vendor/hashab/src/edonr.h create mode 100644 tools/prebuild/vendor/hashab/src/expand.c create mode 100644 tools/prebuild/vendor/hashab/src/expand.h create mode 100644 tools/prebuild/vendor/hashab/src/hash.c create mode 100644 tools/prebuild/vendor/hashab/src/hash.h create mode 100644 tools/prebuild/vendor/hashab/src/reduce.c create mode 100644 tools/prebuild/vendor/hashab/src/reduce.h create mode 100644 tools/prebuild/vendor/hashab/test-data.json diff --git a/tools/prebuild/vendor/hashab/LICENSE b/tools/prebuild/vendor/hashab/LICENSE new file mode 100644 index 00000000..fdddb29a --- /dev/null +++ b/tools/prebuild/vendor/hashab/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/tools/prebuild/vendor/hashab/UPSTREAM.md b/tools/prebuild/vendor/hashab/UPSTREAM.md new file mode 100644 index 00000000..af5e1aeb --- /dev/null +++ b/tools/prebuild/vendor/hashab/UPSTREAM.md @@ -0,0 +1,13 @@ +# HashAB source provenance + +Source: https://github.com/dstaley/hashab +Commit: f80d46432204c6238cad7d8ca3b3dd52ea66836b +License: The Unlicense (see LICENSE) + +`src/*.c`, `src/*.h`, `LICENSE` and `test-data.json` are copied unmodified +from that commit. The prebuilt `libhashab32.so` and the WASM/Zig build files +of the upstream repository are not included. Upstream compiles with +`-fwrapv`. + +`calcHashAB` signs with the eight-byte FireWire GUID of an iPod nano; it is +not a signer for iOS device identifiers. diff --git a/tools/prebuild/vendor/hashab/src/byteorder.h b/tools/prebuild/vendor/hashab/src/byteorder.h new file mode 100644 index 00000000..a2f78cac --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/byteorder.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +static inline uint32_t load32_le(const uint8_t* p) { + return (uint32_t)p[0] | (uint32_t)p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; +} + +static inline void store32_le(uint8_t* p, uint32_t value) { + p[0] = (uint8_t)value; + p[1] = (uint8_t)(value >> 8); + p[2] = (uint8_t)(value >> 16); + p[3] = (uint8_t)(value >> 24); +} diff --git a/tools/prebuild/vendor/hashab/src/calcHashAB.c b/tools/prebuild/vendor/hashab/src/calcHashAB.c new file mode 100644 index 00000000..6a809fdf --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/calcHashAB.c @@ -0,0 +1,60 @@ +#include "calcHashAB.h" + +#include "cipher.h" +#include "expand.h" +#include "reduce.h" + +#include +#include + +enum { HASHAB_SIGNATURE_SIZE = 57, HASHAB_RANDOM_SIZE = 23 }; + +static const uint8_t permutation_32[32] = {0x06, 0x18, 0x11, 0x07, 0x13, 0x0d, 0x0e, 0x09, 0x15, 0x1d, 0x02, 0x1f, 0x01, 0x04, 0x1c, 0x1a, + 0x10, 0x14, 0x0b, 0x1e, 0x03, 0x0a, 0x1b, 0x19, 0x05, 0x0f, 0x16, 0x00, 0x12, 0x17, 0x08, 0x0c}; + +static const uint8_t permutation_56[56] = {0x15, 0x1c, 0x06, 0x0c, 0x07, 0x1a, 0x05, 0x13, 0x08, 0x19, 0x03, 0x01, 0x2d, 0x1e, 0x10, 0x31, 0x1d, 0x14, 0x28, 0x27, 0x35, 0x00, 0x2f, 0x1b, 0x26, 0x0b, 0x0e, 0x02, + 0x23, 0x17, 0x24, 0x22, 0x12, 0x1f, 0x20, 0x04, 0x29, 0x25, 0x21, 0x09, 0x18, 0x0d, 0x32, 0x0f, 0x11, 0x2e, 0x33, 0x2b, 0x30, 0x2a, 0x36, 0x0a, 0x2c, 0x34, 0x16, 0x37}; + +void calcHashAB(uint8_t* target, const uint8_t* sha1, const uint8_t* uuid, const uint8_t* rnd_bytes) { + uint8_t stage1_input[80]; + uint8_t stage1_output[80]; + uint8_t expanded[190]; + uint8_t compression[16]; + uint8_t stage2_input[32]; + uint8_t stage2_output[32]; + if (target == NULL || sha1 == NULL || uuid == NULL || rnd_bytes == NULL) { + return; + } + + for (size_t i = 0; i < 8; i++) { + stage1_input[i] = (uint8_t)(uuid[i] * 0xed); + } + for (size_t i = 0; i < 20; i++) { + stage1_input[8 + i] = (uint8_t)(sha1[i] * 0xed); + } + for (size_t i = 0; i < HASHAB_RANDOM_SIZE; i++) { + stage1_input[28 + i] = (uint8_t)(rnd_bytes[i] * 0xed); + } + __builtin_memset(stage1_input + 51, 0xc1, sizeof(stage1_input) - 51); + + if (stage1_cipher(stage1_output, stage1_input, sizeof(stage1_input)) != 0 || expand(expanded, stage1_output + 20, UINT32_C(0xaeba51ec), stage1_output) != 0 || reduce(expanded, compression) != 0) { + __builtin_memset(target, 0, HASHAB_SIGNATURE_SIZE); + return; + } + + for (size_t i = 0; i < 32; i++) { + stage2_input[i] = stage1_output[44 + permutation_32[i]]; + } + + if (stage2_cipher(stage2_output, stage2_input, sizeof(stage2_input), compression) != 0) { + __builtin_memset(target, 0, HASHAB_SIGNATURE_SIZE); + return; + } + + target[0] = 3; + target[1] = 0; + for (size_t i = 0; i < 55; i++) { + size_t source = permutation_56[i]; + target[i + 2] = source < HASHAB_RANDOM_SIZE ? rnd_bytes[source] : (uint8_t)(stage2_output[source - HASHAB_RANDOM_SIZE] * 0x2d); + } +} diff --git a/tools/prebuild/vendor/hashab/src/calcHashAB.h b/tools/prebuild/vendor/hashab/src/calcHashAB.h new file mode 100644 index 00000000..bb9c8115 --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/calcHashAB.h @@ -0,0 +1,12 @@ +#pragma once +#include + +/** + * calcHashAB - Main hash calculation function. + * + * @param target Output buffer (57 bytes) + * @param sha1 Input SHA1 hash (20 bytes) + * @param uuid Input UUID (8 bytes) + * @param rnd_bytes Random bytes (23 bytes) + */ +void calcHashAB(uint8_t* target, const uint8_t* sha1, const uint8_t* uuid, const uint8_t* rnd_bytes); diff --git a/tools/prebuild/vendor/hashab/src/cipher.c b/tools/prebuild/vendor/hashab/src/cipher.c new file mode 100644 index 00000000..f08d9096 --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/cipher.c @@ -0,0 +1,112 @@ +#include "cipher.h" + +#include "byteorder.h" +#include "cipher_tables.h" + +enum { HASHAB_BLOCK_SIZE = 16, HASHAB_AES128_ROUNDS = 10 }; + +static uint8_t affine_byte(const uint8_t parameters[HASHAB_ROUND_AFFINE_PARAMETERS], uint8_t value) { + uint8_t result = parameters[0]; + + for (size_t bit = 0; bit < 8; bit++) { + if ((value & (uint8_t)(1 << bit)) != 0) { + result ^= parameters[bit + 1u]; + } + } + return result; +} + +static uint32_t profile_word(const cipher_profile* profile, size_t table_number, uint8_t index) { + uint8_t base = profile->round_base[index]; + uint32_t word = 0; + + for (size_t byte = 0; byte < 4; byte++) { + size_t lane = table_number * 4u + byte; + uint8_t value = lane == 0 ? base : affine_byte(profile->round_affine[lane - 1u], base); + + word |= (uint32_t)value << (byte * 8u); + } + return word; +} + +static void table_round(const cipher_profile* profile, const uint8_t input[HASHAB_BLOCK_SIZE], const uint8_t round_key[HASHAB_BLOCK_SIZE], uint8_t output[HASHAB_BLOCK_SIZE]) { + for (size_t word = 0; word < 4; word++) { + uint32_t value = load32_le(round_key + word * 4u); + + for (size_t table = 0; table < 4; table++) { + value ^= profile_word(profile, table, input[(word * 4u + table * 5u) & 15u]); + } + store32_le(output + word * 4u, value); + } +} + +static void final_round(const cipher_profile* profile, const uint8_t input[HASHAB_BLOCK_SIZE], const uint8_t round_key[HASHAB_BLOCK_SIZE], uint8_t output[HASHAB_BLOCK_SIZE]) { + for (size_t position = 0; position < HASHAB_BLOCK_SIZE; position++) { + size_t input_position = ((position & 12u) + (position & 3u) * 5u) & 15u; + uint8_t table_value = profile->final_map[input[input_position]]; + + output[position] = (uint8_t)((uint32_t)table_value * UINT32_C(0x81) + UINT32_C(0x7a)) ^ round_key[position]; + } +} + +static uint8_t inverse_map_byte(const uint8_t map[HASHAB_MAP_SIZE], uint8_t value) { + for (size_t index = 0; index < HASHAB_MAP_SIZE; index++) { + if (map[index] == value) { + return (uint8_t)index; + } + } + return 0; +} + +static int cipher_cbc(const cipher_profile* profile, uint8_t* output, const uint8_t* input, size_t length, const uint8_t initial_chain[HASHAB_BLOCK_SIZE]) { + uint8_t chaining[HASHAB_BLOCK_SIZE]; + uint8_t state[HASHAB_BLOCK_SIZE]; + uint8_t next[HASHAB_BLOCK_SIZE]; + + if (profile == NULL || output == NULL || input == NULL || initial_chain == NULL || length == 0 || length % HASHAB_BLOCK_SIZE != 0) { + return -1; + } + + __builtin_memcpy(chaining, initial_chain, sizeof(chaining)); + + for (size_t offset = 0; offset < length; offset += HASHAB_BLOCK_SIZE) { + for (size_t position = 0; position < HASHAB_BLOCK_SIZE; position++) { + state[position] = profile->input_map[input[offset + position]] ^ chaining[position] ^ profile->schedule[position]; + } + + for (size_t round = 1; round < HASHAB_AES128_ROUNDS; round++) { + table_round(profile, state, profile->schedule + round * HASHAB_BLOCK_SIZE, next); + __builtin_memcpy(state, next, sizeof(state)); + } + + final_round(profile, state, profile->schedule + HASHAB_AES128_ROUNDS * HASHAB_BLOCK_SIZE, next); + + for (size_t position = 0; position < HASHAB_BLOCK_SIZE; position++) { + chaining[position] = next[position]; + output[offset + position] = profile->output_map[next[position]]; + } + } + + return 0; +} + +int stage1_cipher(uint8_t* output, const uint8_t* input, size_t length) { + return cipher_cbc(&stage1_profile, output, input, length, stage1_initial_chain); +} + +int stage2_cipher(uint8_t* output, const uint8_t* input, size_t length, const uint8_t iv[16]) { + uint8_t initial_chain[HASHAB_BLOCK_SIZE]; + int result; + + if (iv == NULL) { + return -1; + } + + for (size_t position = 0; position < HASHAB_BLOCK_SIZE; position++) { + initial_chain[position] = inverse_map_byte(stage2_profile.output_map, iv[position]); + } + + result = cipher_cbc(&stage2_profile, output, input, length, initial_chain); + + return result; +} diff --git a/tools/prebuild/vendor/hashab/src/cipher.h b/tools/prebuild/vendor/hashab/src/cipher.h new file mode 100644 index 00000000..09585fba --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/cipher.h @@ -0,0 +1,8 @@ +#pragma once + +#include +#include + +int stage1_cipher(uint8_t* output, const uint8_t* input, size_t length); + +int stage2_cipher(uint8_t* output, const uint8_t* input, size_t length, const uint8_t iv[16]); diff --git a/tools/prebuild/vendor/hashab/src/cipher_tables.c b/tools/prebuild/vendor/hashab/src/cipher_tables.c new file mode 100644 index 00000000..97f73ab7 --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/cipher_tables.c @@ -0,0 +1,449 @@ +#include "cipher_tables.h" + +const cipher_profile stage1_profile = { + .input_map = + { + 0x6e, 0x04, 0x46, 0xb4, 0xdc, 0xcc, 0xf4, 0xf8, 0xc4, 0x29, 0xa7, 0x4a, 0x76, 0x7a, 0x42, 0x52, 0x3a, 0xc8, 0x8a, 0xe0, 0x58, 0xcf, 0xe8, 0x34, 0x90, 0x7d, 0xa4, 0x56, 0x75, 0x2e, 0x16, 0x9e, + 0x26, 0xcb, 0x0e, 0xe3, 0x94, 0x03, 0xbc, 0xb0, 0x93, 0x61, 0x68, 0x02, 0x21, 0x2d, 0x0a, 0x9d, 0x72, 0x9f, 0xc2, 0x2f, 0x97, 0x87, 0xbf, 0x7c, 0x5f, 0x35, 0xec, 0x01, 0x3d, 0xe1, 0x5e, 0xc9, + 0x71, 0x83, 0xc1, 0xab, 0xc3, 0x4b, 0x73, 0x7f, 0xdb, 0x36, 0x20, 0xcd, 0x69, 0x65, 0x5d, 0xd5, 0xbd, 0xd7, 0x95, 0x67, 0xdf, 0x48, 0xf7, 0x2b, 0x17, 0xfa, 0x23, 0x49, 0x6a, 0xa9, 0x91, 0x81, + 0x39, 0xd4, 0x89, 0x64, 0x8b, 0x1c, 0x3b, 0x37, 0x14, 0x7e, 0x77, 0x85, 0xa6, 0xaa, 0x15, 0x82, 0xf5, 0x18, 0xdd, 0x30, 0x88, 0x00, 0x38, 0x63, 0x40, 0xb2, 0x6b, 0x86, 0x22, 0xfe, 0xd9, 0x4e, + 0xf6, 0x9c, 0xde, 0x2c, 0x44, 0x54, 0x6c, 0x60, 0x5c, 0xb1, 0x3f, 0xd2, 0xee, 0xe2, 0xda, 0xca, 0xa2, 0x50, 0x12, 0x78, 0xc0, 0x57, 0x70, 0xac, 0x08, 0xe5, 0x3c, 0xce, 0xed, 0xb6, 0x8e, 0x06, + 0xbe, 0x53, 0x96, 0x7b, 0x0c, 0x9b, 0x24, 0x28, 0x0b, 0xf9, 0xf0, 0x9a, 0xb9, 0xb5, 0x92, 0x05, 0xea, 0x07, 0x5a, 0xb7, 0x0f, 0x1f, 0x27, 0xe4, 0xc7, 0xad, 0x74, 0x99, 0xa5, 0x79, 0xc6, 0x51, + 0xe9, 0x1b, 0x59, 0x33, 0x5b, 0xd3, 0xeb, 0xe7, 0x43, 0xae, 0xb8, 0x55, 0xf1, 0xfd, 0xc5, 0x4d, 0x25, 0x4f, 0x0d, 0xff, 0x47, 0xd0, 0x6f, 0xb3, 0x8f, 0x62, 0xbb, 0xd1, 0xf2, 0x31, 0x09, 0x19, + 0xa1, 0x4c, 0x11, 0xfc, 0x13, 0x84, 0xa3, 0xaf, 0x8c, 0xe6, 0xef, 0x1d, 0x3e, 0x32, 0x8d, 0x1a, 0x6d, 0x80, 0x45, 0xa8, 0x10, 0x98, 0xa0, 0xfb, 0xd8, 0x2a, 0xf3, 0x1e, 0xba, 0x66, 0x41, 0xd6, + }, + .final_map = + { + 0x13, 0x4c, 0xd8, 0x88, 0xbe, 0x65, 0x9c, 0x43, 0x9f, 0xf6, 0xf1, 0xe6, 0x07, 0x42, 0xc8, 0x04, 0xa8, 0xca, 0x05, 0x7a, 0x2c, 0x4a, 0xbc, 0xe7, 0x5a, 0xba, 0x21, 0x3c, 0xa2, 0x84, 0x53, 0xfd, + 0x80, 0xf0, 0x2e, 0xe3, 0x9b, 0x7c, 0x66, 0x82, 0xa0, 0xf2, 0x0a, 0x8e, 0xfb, 0xc9, 0xb4, 0xb1, 0xab, 0x61, 0xa3, 0x76, 0x5b, 0x63, 0x32, 0x10, 0x49, 0x97, 0x3e, 0xd6, 0x41, 0x38, 0x03, 0x87, + 0xb0, 0x2a, 0x79, 0xe5, 0x1c, 0x2b, 0xee, 0x75, 0xcd, 0x3d, 0x29, 0x86, 0x55, 0xaa, 0x16, 0xbf, 0xea, 0xb6, 0xa4, 0x6d, 0x33, 0x2d, 0x6b, 0xc6, 0x15, 0x74, 0x60, 0x30, 0x01, 0x54, 0x1d, 0xe1, + 0x12, 0x7b, 0xcf, 0xed, 0x6f, 0xae, 0x50, 0x09, 0x9a, 0x4f, 0x0f, 0x5d, 0xd0, 0x20, 0x96, 0x52, 0x69, 0x46, 0xff, 0xb7, 0x51, 0x81, 0xac, 0x4e, 0x92, 0x5e, 0x1b, 0x45, 0xd1, 0x62, 0x47, 0xbb, + 0xcc, 0xc2, 0x7d, 0xb8, 0xad, 0x70, 0x0c, 0x73, 0xeb, 0x18, 0xb5, 0xb3, 0xf7, 0x68, 0x9d, 0x31, 0x0b, 0xe0, 0x4b, 0x83, 0x6c, 0xdf, 0x71, 0x48, 0xc0, 0xc4, 0x3f, 0x17, 0xdd, 0x91, 0x34, 0x6e, + 0x00, 0x7f, 0x35, 0x3a, 0x85, 0xd5, 0xc7, 0xfc, 0xc1, 0xdc, 0xa6, 0xa9, 0x4d, 0x95, 0x19, 0x99, 0x67, 0xe2, 0x36, 0x22, 0x90, 0x77, 0x26, 0xd7, 0x64, 0x37, 0x8a, 0x1e, 0x6a, 0x9e, 0xf5, 0x5f, + 0x8b, 0x25, 0xfe, 0x27, 0x8d, 0x06, 0x2f, 0x93, 0xf3, 0x5c, 0x56, 0xf8, 0x78, 0xa7, 0xe4, 0xaf, 0xa5, 0xd3, 0xf4, 0xb9, 0x72, 0x1f, 0x59, 0x02, 0xe8, 0x0e, 0x23, 0xcb, 0xce, 0x0d, 0x89, 0xef, + 0x1a, 0xd4, 0x98, 0xde, 0x3b, 0x24, 0x28, 0xec, 0x94, 0xc5, 0xd2, 0xda, 0x58, 0xd9, 0x08, 0xe9, 0x7e, 0x40, 0x11, 0xfa, 0x44, 0xb2, 0x8c, 0x57, 0xa1, 0x8f, 0xbd, 0x14, 0x39, 0xdb, 0xc3, 0xf9, + }, + .output_map = + { + 0x9c, 0xd6, 0x26, 0x2c, 0x98, 0xda, 0x2a, 0x28, 0xdb, 0x95, 0x05, 0x8b, 0x57, 0x19, 0x89, 0x07, 0xc7, 0xc9, 0x59, 0x17, 0xcb, 0xc5, 0x55, 0x1b, 0x68, 0xea, 0x9a, 0xd8, 0xec, 0x66, 0x16, 0x5c, + 0x31, 0x3f, 0xaf, 0xe1, 0x3d, 0x33, 0xa3, 0xed, 0x72, 0x00, 0x50, 0x02, 0xfe, 0x74, 0xc4, 0x8e, 0x4e, 0x04, 0xb4, 0xbe, 0x42, 0x10, 0xc0, 0xb2, 0x2d, 0x63, 0xf3, 0x7d, 0xa1, 0xef, 0x7f, 0xf1, + 0x7b, 0xf5, 0xa5, 0xeb, 0xf7, 0x79, 0x29, 0x67, 0x3c, 0x36, 0xc6, 0x8c, 0x38, 0x3a, 0xca, 0x88, 0xc8, 0x8a, 0xfa, 0x78, 0x4c, 0x06, 0x76, 0xfc, 0x27, 0x69, 0xb9, 0xb7, 0x2b, 0x65, 0xb5, 0xbb, + 0x12, 0x60, 0xf0, 0x62, 0x9e, 0xd4, 0x64, 0xee, 0xd1, 0x9f, 0x4f, 0x41, 0xdd, 0x93, 0x43, 0x4d, 0x8d, 0x03, 0x53, 0x1d, 0x01, 0x8f, 0xdf, 0x91, 0xae, 0xa4, 0x14, 0x5e, 0xa2, 0xb0, 0x20, 0x52, + 0xe8, 0x6a, 0x1a, 0x58, 0x6c, 0xe6, 0x96, 0xdc, 0x47, 0x49, 0xd9, 0x97, 0x4b, 0x45, 0xd5, 0x9b, 0x5b, 0x15, 0x85, 0x0b, 0xd7, 0x99, 0x09, 0x87, 0x1c, 0x56, 0xa6, 0xac, 0x18, 0x5a, 0xaa, 0xa8, + 0xad, 0xe3, 0x73, 0xfd, 0x21, 0x6f, 0xff, 0x71, 0xce, 0x84, 0x34, 0x3e, 0xc2, 0x90, 0x40, 0x32, 0xf2, 0x80, 0xd0, 0x82, 0x7e, 0xf4, 0x44, 0x0e, 0xb1, 0xbf, 0x2f, 0x61, 0xbd, 0xb3, 0x23, 0x6d, + 0xa7, 0xe9, 0x39, 0x37, 0xab, 0xe5, 0x35, 0x3b, 0x48, 0x0a, 0x7a, 0xf8, 0xcc, 0x86, 0xf6, 0x7c, 0xbc, 0xb6, 0x46, 0x0c, 0xb8, 0xba, 0x4a, 0x08, 0xfb, 0x75, 0x25, 0x6b, 0x77, 0xf9, 0xa9, 0xe7, + 0x2e, 0x24, 0x94, 0xde, 0x22, 0x30, 0xa0, 0xd2, 0x0d, 0x83, 0xd3, 0x9d, 0x81, 0x0f, 0x5f, 0x11, 0x51, 0x1f, 0xcf, 0xc1, 0x5d, 0x13, 0xc3, 0xcd, 0x92, 0xe0, 0x70, 0xe2, 0x1e, 0x54, 0xe4, 0x6e, + }, + .round_base = + { + 0xb9, 0x97, 0x1b, 0xfa, 0x8b, 0xf8, 0x88, 0xf5, 0x35, 0x2c, 0xe3, 0xf7, 0x8d, 0xdc, 0xc0, 0xb8, 0x6d, 0xc9, 0x91, 0x7b, 0x3a, 0x37, 0x1f, 0xde, 0xec, 0xbf, 0x6b, 0xe1, 0xb5, 0x46, 0x83, 0x0c, + 0xef, 0xca, 0xae, 0xa6, 0x01, 0xdb, 0x09, 0xe6, 0xbc, 0xc3, 0x0d, 0xc7, 0xac, 0xe9, 0x59, 0xd9, 0x4d, 0x51, 0x9c, 0xd2, 0xc5, 0x58, 0x07, 0x99, 0x17, 0xa8, 0x75, 0xbb, 0xfc, 0x48, 0x31, 0x73, + 0xf0, 0x9a, 0x5b, 0x06, 0x76, 0xb3, 0x6a, 0xb4, 0x40, 0xc8, 0xba, 0x5a, 0x23, 0x64, 0x7f, 0xa2, 0x5e, 0x16, 0x15, 0x29, 0x2e, 0x13, 0x89, 0x60, 0x19, 0x9d, 0x78, 0x0e, 0x38, 0x0a, 0x5f, 0xaf, + 0x90, 0x52, 0xd4, 0xd7, 0xbd, 0x50, 0xa3, 0x2d, 0x28, 0x2a, 0x10, 0x65, 0x5d, 0x42, 0x81, 0xaa, 0x80, 0x9e, 0x98, 0x3f, 0x8a, 0xc6, 0xc4, 0x03, 0x6e, 0xd8, 0xff, 0x55, 0x74, 0x71, 0xb7, 0x96, + 0x69, 0x22, 0xf2, 0xb6, 0xed, 0x34, 0xad, 0x14, 0x77, 0xdf, 0x70, 0xd0, 0x05, 0xa9, 0xa1, 0x27, 0x24, 0x86, 0x1e, 0xcf, 0x00, 0x0f, 0x1d, 0x3e, 0x2b, 0x82, 0x5c, 0x56, 0x9b, 0x4e, 0xa7, 0x94, + 0x11, 0x66, 0x8e, 0x41, 0x6f, 0xdd, 0x49, 0x25, 0x02, 0xb2, 0xcd, 0x44, 0xbe, 0xe7, 0xf6, 0x08, 0x20, 0x8f, 0xe8, 0x4b, 0x67, 0xfb, 0x33, 0x92, 0xd1, 0xc1, 0xf3, 0xe2, 0xa0, 0x1c, 0x4a, 0xf1, + 0xda, 0xc2, 0xb1, 0x1a, 0x7a, 0xa4, 0x87, 0x47, 0xea, 0x4c, 0x45, 0x8c, 0x72, 0xe4, 0x2f, 0x79, 0x3c, 0x7d, 0x63, 0x9f, 0x3d, 0xcb, 0xcc, 0x18, 0x57, 0x39, 0x62, 0xe0, 0xfd, 0x84, 0xd3, 0x43, + 0xd6, 0xf4, 0x21, 0x26, 0x68, 0xeb, 0x93, 0xfe, 0xce, 0xab, 0x54, 0x12, 0xe5, 0x32, 0x04, 0x7e, 0x4f, 0xd5, 0xb0, 0x85, 0x7c, 0xf9, 0x53, 0x6c, 0x95, 0xee, 0x36, 0x30, 0x61, 0x3b, 0x0b, 0xa5, + }, + .round_affine = + { + { + 0xb0, + 0x73, + 0x5d, + 0x64, + 0x75, + 0xef, + 0x87, + 0x21, + 0x85, + }, + { + 0xe3, + 0x73, + 0x5d, + 0x64, + 0x75, + 0xef, + 0x87, + 0x21, + 0x85, + }, + { + 0xe1, + 0x72, + 0x5f, + 0x60, + 0x7d, + 0xff, + 0xa7, + 0x61, + 0x05, + }, + { + 0x2f, + 0x72, + 0x5f, + 0x60, + 0x7d, + 0xff, + 0xa7, + 0x61, + 0x05, + }, + { + 0xa4, + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + }, + { + 0x03, + 0x73, + 0x5d, + 0x64, + 0x75, + 0xef, + 0x87, + 0x21, + 0x85, + }, + { + 0xd4, + 0x73, + 0x5d, + 0x64, + 0x75, + 0xef, + 0x87, + 0x21, + 0x85, + }, + { + 0x4e, + 0x73, + 0x5d, + 0x64, + 0x75, + 0xef, + 0x87, + 0x21, + 0x85, + }, + { + 0x77, + 0x72, + 0x5f, + 0x60, + 0x7d, + 0xff, + 0xa7, + 0x61, + 0x05, + }, + { + 0x0c, + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + }, + { + 0x00, + 0x73, + 0x5d, + 0x64, + 0x75, + 0xef, + 0x87, + 0x21, + 0x85, + }, + { + 0xe9, + 0x73, + 0x5d, + 0x64, + 0x75, + 0xef, + 0x87, + 0x21, + 0x85, + }, + { + 0xeb, + 0x73, + 0x5d, + 0x64, + 0x75, + 0xef, + 0x87, + 0x21, + 0x85, + }, + { + 0x64, + 0x72, + 0x5f, + 0x60, + 0x7d, + 0xff, + 0xa7, + 0x61, + 0x05, + }, + { + 0xbd, + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + }, + }, + .schedule = { + 0x23, 0x13, 0xad, 0xee, 0xf1, 0xd1, 0xf0, 0x0b, 0xea, 0x2b, 0x5c, 0x70, 0xf0, 0x47, 0x0f, 0x9f, 0x5b, 0xfd, 0xe6, 0x16, 0xaa, 0x2c, 0x16, 0x1d, 0x40, 0x07, 0x4a, 0x6d, 0xb0, 0x40, + 0x45, 0xf2, 0x9a, 0x86, 0xbe, 0x6e, 0x30, 0xaa, 0xa8, 0x73, 0x70, 0xad, 0xe2, 0x1e, 0xc0, 0xed, 0xa7, 0xec, 0xf2, 0x3b, 0x34, 0x2a, 0xc2, 0x91, 0x9c, 0x59, 0xb2, 0x3c, 0x7e, 0x47, + 0x72, 0xd1, 0xd9, 0xab, 0xca, 0xbc, 0xcf, 0x5e, 0x08, 0x2d, 0x53, 0x07, 0xba, 0x11, 0x2d, 0x40, 0xc8, 0xc0, 0xf4, 0xeb, 0xc5, 0x09, 0xa1, 0x5c, 0xcd, 0x24, 0xf2, 0x5b, 0x77, 0x35, + 0xdf, 0x1b, 0xbf, 0xf5, 0x2b, 0xf0, 0x8c, 0x6a, 0x59, 0xee, 0x41, 0x4e, 0xab, 0xb5, 0x36, 0x7b, 0x74, 0xae, 0x89, 0x8e, 0x5f, 0x5e, 0x6c, 0x97, 0x44, 0x99, 0x2d, 0xd9, 0xef, 0x2c, + 0x1b, 0xa2, 0x9b, 0x82, 0x92, 0x2c, 0xc4, 0xdc, 0x8a, 0x8c, 0x7c, 0xa9, 0xa7, 0x55, 0x93, 0x85, 0xbc, 0xf7, 0x08, 0x07, 0x2e, 0xdb, 0xcc, 0xdb, 0xcc, 0x08, 0x82, 0xc9, 0x6b, 0x5d, + 0x11, 0x4c, 0xd7, 0xaa, 0x19, 0x4b, 0xf9, 0x71, 0xd5, 0x90, 0x66, 0x64, 0xd7, 0x0f, 0x0d, 0x39, 0xc6, 0x43, 0xda, 0x93, 0xdf, 0x08, 0x23, 0xe2, 0x0a, 0x98, + }}; + +const uint8_t stage1_initial_chain[16] = { + 0xfb, 0xf0, 0xe5, 0x88, 0x1e, 0x89, 0x74, 0xc9, 0x46, 0xb2, 0x5d, 0x56, 0x03, 0xe5, 0x7e, 0xe4, +}; + +const cipher_profile stage2_profile = { + .input_map = + { + 0x35, 0x0d, 0x3c, 0x1e, 0xb2, 0x30, 0x01, 0x23, 0xe5, 0x4a, 0x7b, 0x59, 0xf5, 0x77, 0x46, 0x64, 0xa2, 0x6d, 0x82, 0xe3, 0x0c, 0x50, 0xbf, 0xb4, 0x5b, 0x2a, 0xc5, 0xa4, 0x4b, 0x17, 0xf8, 0xf3, + 0x1c, 0xd3, 0x7f, 0x5d, 0x6c, 0xee, 0x28, 0x0a, 0x3b, 0x94, 0x38, 0x1a, 0x2b, 0xa9, 0x6f, 0x4d, 0x7c, 0x2e, 0xc1, 0x3d, 0xd2, 0x79, 0x96, 0x6a, 0x85, 0x69, 0x86, 0x7a, 0x95, 0x3e, 0xd1, 0x2d, + 0xc2, 0x90, 0xa1, 0x83, 0x45, 0xc7, 0xf6, 0xd4, 0x78, 0xd7, 0xe6, 0xc4, 0x02, 0x80, 0xb1, 0x93, 0x3f, 0xf0, 0x1f, 0x14, 0xfb, 0xa7, 0x48, 0x29, 0xc6, 0xb7, 0x58, 0x53, 0xbc, 0xe0, 0x0f, 0x6e, + 0x81, 0x4e, 0x88, 0xaa, 0x9b, 0x19, 0xb5, 0x97, 0xa6, 0x09, 0xcf, 0xed, 0xdc, 0x5e, 0xf2, 0xd0, 0xe1, 0xd9, 0x36, 0xca, 0x25, 0xe4, 0x0b, 0xf7, 0x18, 0x9e, 0x71, 0x8d, 0x62, 0xa3, 0x4c, 0xb0, + 0x5f, 0x67, 0x56, 0x74, 0xd8, 0x5a, 0x6b, 0x49, 0x8f, 0x20, 0x11, 0x33, 0x9f, 0x1d, 0x2c, 0x0e, 0xc8, 0x07, 0xe8, 0x89, 0x66, 0x3a, 0xd5, 0xde, 0x31, 0x40, 0xaf, 0xce, 0x21, 0x7d, 0x92, 0x99, + 0x76, 0xb9, 0x15, 0x37, 0x06, 0x84, 0x42, 0x60, 0x51, 0xfe, 0x52, 0x70, 0x41, 0xc3, 0x05, 0x27, 0x16, 0x44, 0xab, 0x57, 0xb8, 0x13, 0xfc, 0x00, 0xef, 0x03, 0xec, 0x10, 0xff, 0x54, 0xbb, 0x47, + 0xa8, 0xfa, 0xcb, 0xe9, 0x2f, 0xad, 0x9c, 0xbe, 0x12, 0xbd, 0x8c, 0xae, 0x68, 0xea, 0xdb, 0xf9, 0x55, 0x9a, 0x75, 0x7e, 0x91, 0xcd, 0x22, 0x43, 0xac, 0xdd, 0x32, 0x39, 0xd6, 0x8a, 0x65, 0x04, + 0xeb, 0x24, 0xe2, 0xc0, 0xf1, 0x73, 0xdf, 0xfd, 0xcc, 0x63, 0xa5, 0x87, 0xb6, 0x34, 0x98, 0xba, 0x8b, 0xb3, 0x5c, 0xa0, 0x4f, 0x8e, 0x61, 0x9d, 0x72, 0xf4, 0x1b, 0xe7, 0x08, 0xc9, 0x26, 0xda, + }, + .final_map = + { + 0xcd, 0x6e, 0x86, 0x8c, 0x3d, 0xf0, 0xef, 0x96, 0xdd, 0x85, 0x80, 0x69, 0xb2, 0xa0, 0x3f, 0x6b, 0x41, 0x8d, 0x1d, 0x2b, 0x1f, 0xb1, 0x6c, 0x0b, 0x76, 0x75, 0xa7, 0xc5, 0xec, 0xb3, 0x42, 0xe0, + 0xda, 0x4e, 0x67, 0xb7, 0xb8, 0x9b, 0x46, 0x35, 0xed, 0x2d, 0x88, 0xf6, 0x90, 0x1e, 0x27, 0xfb, 0xad, 0x6f, 0x21, 0x5f, 0x26, 0x62, 0x30, 0x99, 0x59, 0x71, 0x68, 0x02, 0x1a, 0xf7, 0x28, 0x92, + 0x18, 0x87, 0xeb, 0x0f, 0x73, 0x9c, 0x24, 0xdc, 0xb4, 0xd9, 0x5a, 0xfd, 0x40, 0xd0, 0x05, 0x91, 0x1c, 0xc9, 0xbf, 0x3e, 0xae, 0x9e, 0xe7, 0xbe, 0xc0, 0xfa, 0x51, 0xfc, 0x84, 0xa8, 0x3b, 0xf2, + 0xb0, 0x13, 0x10, 0x60, 0xf4, 0x55, 0xc3, 0x72, 0x2e, 0xc8, 0x19, 0x5c, 0xca, 0x22, 0x7d, 0xe4, 0xd1, 0x53, 0xf3, 0x8a, 0xce, 0xab, 0x93, 0x77, 0x04, 0x45, 0x08, 0x83, 0xba, 0xd6, 0xa1, 0x5d, + 0xaf, 0xbb, 0x48, 0x32, 0xff, 0xac, 0xa4, 0xa9, 0x8e, 0x29, 0x78, 0x98, 0x38, 0x17, 0x44, 0x06, 0xd5, 0x61, 0xbd, 0xe1, 0xf9, 0x9a, 0x50, 0x03, 0x7b, 0x3c, 0x0d, 0xa5, 0xbc, 0xd2, 0x0c, 0x66, + 0x94, 0xb5, 0x1b, 0xe8, 0xfe, 0xe3, 0x31, 0x56, 0x00, 0xc4, 0x7f, 0xee, 0x4d, 0x0e, 0xde, 0x79, 0xaa, 0xd8, 0x7e, 0xd3, 0x5b, 0x5e, 0x15, 0x49, 0xa6, 0x6d, 0x2f, 0x6a, 0x20, 0x97, 0x8f, 0x52, + 0x7c, 0xc6, 0x4f, 0x57, 0x4c, 0xf1, 0x12, 0x89, 0x58, 0x07, 0x63, 0x7a, 0x9d, 0x2c, 0x47, 0x70, 0x37, 0x3a, 0xc7, 0xa2, 0x25, 0x2a, 0x4a, 0xcc, 0x74, 0xb6, 0x95, 0x81, 0x0a, 0xc2, 0xd7, 0x14, + 0xdf, 0xd4, 0x82, 0xe9, 0xea, 0x8b, 0xc1, 0x39, 0x23, 0x11, 0x64, 0xb9, 0xcb, 0xf8, 0xe5, 0xe6, 0xf5, 0x43, 0x01, 0x33, 0x65, 0x9f, 0x36, 0xcf, 0x09, 0x16, 0xa3, 0xe2, 0x34, 0x4b, 0x54, 0xdb, + }, + .output_map = + { + 0x36, 0xd1, 0xff, 0x50, 0xae, 0xc9, 0x07, 0xd8, 0xcf, 0x20, 0xc6, 0x61, 0x97, 0x68, 0x7e, 0x99, 0x3a, 0x55, 0x7b, 0x4c, 0x82, 0x1d, 0xb3, 0x04, 0x4b, 0x1c, 0xca, 0xe5, 0x43, 0x94, 0x52, 0xed, + 0x40, 0x6f, 0xc1, 0xa6, 0x48, 0xf7, 0x39, 0x9e, 0xf1, 0xd6, 0xb0, 0xdf, 0xa9, 0x0e, 0x78, 0x27, 0xbc, 0x6b, 0xc5, 0x2a, 0xf4, 0x23, 0x0d, 0xf2, 0xf5, 0x5a, 0x2c, 0xdb, 0x7d, 0x62, 0x24, 0x53, + 0x30, 0x5f, 0x71, 0x56, 0xf8, 0xa7, 0x29, 0x8e, 0x41, 0x26, 0xc0, 0xef, 0xb9, 0x1e, 0xc8, 0x77, 0xac, 0x5b, 0x75, 0xda, 0xa4, 0xd3, 0xfd, 0xe2, 0x45, 0xaa, 0x3c, 0xeb, 0x8d, 0x72, 0x74, 0xa3, + 0x46, 0xe1, 0x4f, 0xa0, 0xfe, 0x19, 0x17, 0xe8, 0x7f, 0xd0, 0xb6, 0x51, 0x87, 0x58, 0x2e, 0x49, 0x4a, 0x65, 0xcb, 0x9c, 0xd2, 0x6d, 0xc3, 0x14, 0xfb, 0xcc, 0xba, 0xd5, 0x33, 0x84, 0x02, 0x9d, + 0xd4, 0x03, 0x2d, 0x12, 0xdc, 0x8b, 0xa5, 0x0a, 0x5d, 0x42, 0x44, 0x73, 0x15, 0x7a, 0x0c, 0xbb, 0x28, 0xd7, 0x59, 0xbe, 0x60, 0x8f, 0xa1, 0x86, 0x89, 0xee, 0x98, 0x47, 0x11, 0xf6, 0x90, 0xbf, + 0xa2, 0x3d, 0x93, 0xe4, 0x1a, 0x35, 0x9b, 0x6c, 0x63, 0xb4, 0x32, 0xcd, 0x2b, 0xfc, 0xea, 0x05, 0xce, 0xe9, 0xe7, 0xb8, 0x16, 0xb1, 0x1f, 0x70, 0xb7, 0x88, 0x5e, 0x79, 0xaf, 0x00, 0xe6, 0x81, + 0xb2, 0x4d, 0xe3, 0x34, 0x6a, 0x85, 0xab, 0x7c, 0x13, 0x64, 0x22, 0xbd, 0x1b, 0xec, 0x9a, 0xb5, 0xde, 0xf9, 0x37, 0x08, 0x66, 0x01, 0x2f, 0x80, 0x67, 0x38, 0x4e, 0x69, 0x9f, 0xf0, 0x96, 0x31, + 0xc4, 0xf3, 0xdd, 0xc2, 0x8c, 0x3b, 0x95, 0xfa, 0xad, 0x92, 0x54, 0x83, 0x25, 0x8a, 0x5c, 0x0b, 0x18, 0xc7, 0x09, 0x6e, 0x10, 0x3f, 0x91, 0x76, 0xd9, 0x3e, 0xa8, 0x57, 0x21, 0x06, 0xe0, 0x0f, + }, + .round_base = + { + 0xf5, 0xed, 0x73, 0xf9, 0x12, 0x16, 0x8f, 0x8a, 0x0c, 0xa1, 0xb2, 0xf4, 0xaf, 0xac, 0x18, 0x7e, 0xbe, 0xc6, 0x62, 0x4d, 0x68, 0x1a, 0xe7, 0x3d, 0x94, 0x58, 0x3c, 0xcf, 0xba, 0x90, 0x0b, 0x9f, + 0x95, 0x9d, 0x52, 0xc5, 0x5c, 0x99, 0x1d, 0x6b, 0x85, 0xeb, 0xd5, 0xc9, 0x55, 0x57, 0x61, 0xda, 0xb6, 0xd2, 0xce, 0x5b, 0x5e, 0x48, 0x78, 0x13, 0x7d, 0x74, 0xcb, 0x65, 0xfb, 0xf6, 0xf8, 0xdf, + 0x71, 0x4c, 0x23, 0x91, 0xfe, 0x00, 0xdd, 0x33, 0x09, 0x20, 0xc8, 0x7c, 0x81, 0x66, 0xfc, 0x6a, 0x5d, 0xd9, 0x45, 0x27, 0x83, 0x0a, 0x0f, 0x7a, 0xdc, 0xe5, 0x04, 0x43, 0x9e, 0xa5, 0xb4, 0x9c, + 0x25, 0xbd, 0x08, 0xc2, 0x3a, 0x28, 0x69, 0xc1, 0xde, 0xe6, 0x4e, 0x6e, 0x6c, 0x7b, 0x21, 0xb3, 0x59, 0x8e, 0xa3, 0x5f, 0xc0, 0x10, 0xe0, 0xab, 0xc3, 0x92, 0x88, 0x07, 0xd6, 0xb9, 0x93, 0x51, + 0xbc, 0xe9, 0xbb, 0xf2, 0x76, 0x89, 0x80, 0x9a, 0xf3, 0xc7, 0x32, 0x2c, 0x01, 0xe8, 0xad, 0x2e, 0x75, 0xfd, 0x4f, 0xa0, 0x50, 0xa6, 0x3b, 0x5a, 0x87, 0x2d, 0x9b, 0xbf, 0x70, 0xec, 0xa4, 0x6d, + 0x79, 0x36, 0xc4, 0x96, 0x49, 0x2a, 0x47, 0xe4, 0xef, 0xf0, 0x2b, 0xb0, 0xa8, 0xae, 0x39, 0x0d, 0x2f, 0x1f, 0x14, 0xd3, 0xf7, 0x64, 0x1b, 0x84, 0x03, 0xd8, 0xe1, 0x41, 0xf1, 0xb5, 0xcc, 0xb1, + 0x1e, 0x40, 0xa2, 0xdb, 0x97, 0x29, 0x82, 0xea, 0x42, 0x11, 0x77, 0xb8, 0x3f, 0xd4, 0x22, 0x4b, 0x98, 0x8b, 0x7f, 0x26, 0xe2, 0x72, 0x31, 0xca, 0x67, 0xfa, 0x46, 0x8d, 0x02, 0x56, 0x86, 0x24, + 0x06, 0x4a, 0x38, 0xa9, 0x1c, 0x60, 0xe3, 0x3e, 0x44, 0x37, 0xee, 0x63, 0x53, 0x6f, 0x8c, 0x30, 0x05, 0x34, 0xd0, 0xcd, 0xd1, 0x35, 0xa7, 0xff, 0xb7, 0xd7, 0x19, 0x15, 0x54, 0x0e, 0x17, 0xaa, + }, + .round_affine = + { + { + 0x85, + 0xa4, + 0x92, + 0x5d, + 0x9c, + 0xb3, + 0xc5, + 0x56, + 0x08, + }, + { + 0xb0, + 0xa4, + 0x92, + 0x5d, + 0x9c, + 0xb3, + 0xc5, + 0x56, + 0x08, + }, + { + 0xd6, + 0xa5, + 0x90, + 0x59, + 0x94, + 0xa3, + 0xe5, + 0x16, + 0x88, + }, + { + 0x75, + 0xa5, + 0x90, + 0x59, + 0x94, + 0xa3, + 0xe5, + 0x16, + 0x88, + }, + { + 0x5e, + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + }, + { + 0x18, + 0xa4, + 0x92, + 0x5d, + 0x9c, + 0xb3, + 0xc5, + 0x56, + 0x08, + }, + { + 0x21, + 0xa4, + 0x92, + 0x5d, + 0x9c, + 0xb3, + 0xc5, + 0x56, + 0x08, + }, + { + 0x9d, + 0xa4, + 0x92, + 0x5d, + 0x9c, + 0xb3, + 0xc5, + 0x56, + 0x08, + }, + { + 0x54, + 0xa5, + 0x90, + 0x59, + 0x94, + 0xa3, + 0xe5, + 0x16, + 0x88, + }, + { + 0xd0, + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + }, + { + 0xd4, + 0xa4, + 0x92, + 0x5d, + 0x9c, + 0xb3, + 0xc5, + 0x56, + 0x08, + }, + { + 0xcb, + 0xa4, + 0x92, + 0x5d, + 0x9c, + 0xb3, + 0xc5, + 0x56, + 0x08, + }, + { + 0xac, + 0xa4, + 0x92, + 0x5d, + 0x9c, + 0xb3, + 0xc5, + 0x56, + 0x08, + }, + { + 0x5b, + 0xa5, + 0x90, + 0x59, + 0x94, + 0xa3, + 0xe5, + 0x16, + 0x88, + }, + { + 0x00, + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + }, + }, + .schedule = { + 0x42, 0x27, 0x0e, 0xb8, 0xa6, 0x9e, 0x0d, 0xd8, 0x77, 0x54, 0xa8, 0x16, 0xae, 0xb3, 0x57, 0x13, 0xb8, 0x14, 0x2c, 0xc5, 0x1e, 0x8a, 0x21, 0x1d, 0x69, 0xde, 0x89, 0x0b, 0xc7, 0x6d, + 0xde, 0x18, 0x4d, 0x48, 0x5e, 0xf3, 0x53, 0xc2, 0x7f, 0xee, 0x3a, 0x1c, 0xf6, 0xe5, 0xfd, 0x71, 0x28, 0xfd, 0xc0, 0xfb, 0x46, 0xeb, 0x93, 0x39, 0x39, 0x05, 0xa9, 0x25, 0xcf, 0xe0, + 0x54, 0x54, 0xe7, 0x1d, 0xd4, 0x18, 0xef, 0xd2, 0x47, 0x21, 0xd6, 0xd7, 0xee, 0x04, 0x19, 0x37, 0xba, 0x50, 0xfe, 0x2a, 0x39, 0x28, 0x47, 0x74, 0x7e, 0x09, 0x91, 0xa3, 0x90, 0x0d, + 0x88, 0x94, 0x2a, 0x5d, 0x76, 0xbe, 0xfe, 0x5b, 0x50, 0xdc, 0x80, 0x52, 0xc1, 0x7f, 0x10, 0x5f, 0x49, 0xeb, 0x3a, 0x02, 0x3f, 0x55, 0x2c, 0xa5, 0xd1, 0x99, 0xac, 0xf7, 0x10, 0xe6, + 0xbc, 0xa8, 0x59, 0x0d, 0x86, 0xaa, 0x66, 0x58, 0x9a, 0xc7, 0x5d, 0x1d, 0x36, 0x30, 0x4d, 0xfb, 0x8a, 0x98, 0x14, 0xf6, 0x0c, 0x32, 0x72, 0xae, 0x6b, 0xe7, 0x20, 0x59, 0x5d, 0xd7, + 0x6d, 0xa2, 0xd7, 0x4f, 0x79, 0x54, 0xdb, 0x7d, 0x0b, 0xfa, 0x42, 0xf6, 0x70, 0x5c, 0x1f, 0x21, 0x1d, 0xfe, 0xc8, 0x6e, 0x64, 0xaa, 0x13, 0x13, 0x6f, 0x50, + }}; diff --git a/tools/prebuild/vendor/hashab/src/cipher_tables.h b/tools/prebuild/vendor/hashab/src/cipher_tables.h new file mode 100644 index 00000000..7d895a79 --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/cipher_tables.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +enum { HASHAB_MAP_SIZE = 256, HASHAB_ROUND_AFFINE_COUNT = 15, HASHAB_ROUND_AFFINE_PARAMETERS = 9, HASHAB_SCHEDULE_SIZE = 176 }; + +typedef struct cipher_profile { + uint8_t input_map[HASHAB_MAP_SIZE]; + uint8_t final_map[HASHAB_MAP_SIZE]; + uint8_t output_map[HASHAB_MAP_SIZE]; + uint8_t round_base[HASHAB_MAP_SIZE]; + uint8_t round_affine[HASHAB_ROUND_AFFINE_COUNT][HASHAB_ROUND_AFFINE_PARAMETERS]; + uint8_t schedule[HASHAB_SCHEDULE_SIZE]; +} cipher_profile; + +extern const cipher_profile stage1_profile; +extern const cipher_profile stage2_profile; + +extern const uint8_t stage1_initial_chain[16]; diff --git a/tools/prebuild/vendor/hashab/src/edonr.c b/tools/prebuild/vendor/hashab/src/edonr.c new file mode 100644 index 00000000..6a0681d9 --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/edonr.c @@ -0,0 +1,133 @@ +/* edonr.c - an implementation of the EDON-R' 256 hash function + * + * Copyright (c) 2011, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * This implementation is based on the article: + * D. Gligoroski, R. S. Odegard, M. Mihova, S. J. Knapskog, ..., + * Cryptographic Hash Function EDON-R - Submission to NIST, 2008 + * with modifications from the article: + * D. Gligoroski, R. S. Odegard, M. Mihova, S. J. Knapskog, ..., + * Cryptographic Hash Function EDON-R - May 20, 2009 + * + * EDON-R has been designed to be much more efficient than SHA-2 + * cryptographic hash functions, offering the same or better security. + */ + +#include "edonr.h" + +#define ROTL32(value, shift) (((value) << (shift)) | ((value) >> (32 - (shift)))) + +#define Q256(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, y6, y7, z0, z1, z2, z3, z4, z5, z6, z7) \ + { \ + t8 = x0 + x4; \ + t9 = x1 + x7; \ + t12 = t8 + t9; \ + t10 = x2 + x3; \ + t11 = x5 + x6; \ + t13 = t10 + t11; \ + t0 = UINT32_C(0xaaaaaaaa) + t12 + x2; \ + t1 = ROTL32(t12 + x3, 4); \ + t2 = ROTL32(t12 + x6, 8); \ + t3 = ROTL32(t13 + x7, 13); \ + t4 = ROTL32(x1 + t13, 17); \ + t5 = ROTL32(t8 + t10 + x5, 22); \ + t6 = ROTL32(x0 + t9 + t11, 24); \ + t7 = ROTL32(t13 + x4, 29); \ + t16 = t0 ^ t4; \ + t17 = t1 ^ t7; \ + t18 = t2 ^ t3; \ + t19 = t5 ^ t6; \ + t8 = t3 ^ t19; \ + t9 = t2 ^ t19; \ + t10 = t18 ^ t5; \ + t11 = t16 ^ t1; \ + t12 = t16 ^ t7; \ + t13 = t17 ^ t6; \ + t14 = t18 ^ t4; \ + t15 = t0 ^ t17; \ + t16 = y0 + y1; \ + t17 = y2 + y5; \ + t20 = t16 + t17; \ + t18 = y3 + y4; \ + t22 = t16 + t18; \ + t19 = y6 + y7; \ + t21 = t18 + t19; \ + t23 = t17 + t19; \ + t0 = UINT32_C(0x55555555) + t20 + y7; \ + t1 = ROTL32(t22 + y6, 5); \ + t2 = ROTL32(t20 + y3, 9); \ + t3 = ROTL32(y2 + t21, 11); \ + t4 = ROTL32(t22 + y5, 15); \ + t5 = ROTL32(t23 + y4, 20); \ + t6 = ROTL32(y1 + t23, 25); \ + t7 = ROTL32(y0 + t21, 27); \ + t16 = t0 ^ t1; \ + t17 = t2 ^ t5; \ + t18 = t3 ^ t4; \ + t19 = t6 ^ t7; \ + z5 = t8 + (t18 ^ t6); \ + z6 = t9 + (t17 ^ t7); \ + z7 = t10 + (t4 ^ t19); \ + z0 = t11 + (t16 ^ t5); \ + z1 = t12 + (t2 ^ t19); \ + z2 = t13 + (t16 ^ t3); \ + z3 = t14 + (t0 ^ t18); \ + z4 = t15 + (t1 ^ t17); \ + } + +void EdonR256Init(EdonR256State* state) { + static const uint32_t initial_hash[16] = {UINT32_C(0x40414243), UINT32_C(0x44454647), UINT32_C(0x48494a4b), UINT32_C(0x4c4d4e4f), UINT32_C(0x50515253), UINT32_C(0x54555657), + UINT32_C(0x58595a5b), UINT32_C(0x5c5d5e5f), UINT32_C(0x60616263), UINT32_C(0x64656667), UINT32_C(0x68696a6b), UINT32_C(0x6c6d6e6f), + UINT32_C(0x70717273), UINT32_C(0x74757677), UINT32_C(0x78797a7b), UINT32_C(0x7c7d7e7f)}; + + __builtin_memcpy(state->hash, initial_hash, sizeof(initial_hash)); +} + +void EdonR256Compress(EdonR256State* state, const uint32_t block[static EDONR256_BLOCK_WORDS]) { + uint32_t* hash = state->hash; + uint32_t t0, t1, t2, t3, t4, t5, t6, t7; + uint32_t t8, t9, t10, t11, t12, t13, t14, t15; + uint32_t t16, t17, t18, t19, t20, t21, t22, t23; + uint32_t p16, p17, p18, p19, p20, p21, p22, p23; + uint32_t p24, p25, p26, p27, p28, p29, p30, p31; + + Q256(block[15], block[14], block[13], block[12], block[11], block[10], block[9], block[8], block[0], block[1], block[2], block[3], block[4], block[5], block[6], block[7], p16, p17, p18, p19, p20, p21, p22, p23); + Q256(p16, p17, p18, p19, p20, p21, p22, p23, block[8], block[9], block[10], block[11], block[12], block[13], block[14], block[15], p24, p25, p26, p27, p28, p29, p30, p31); + + Q256(hash[8], hash[9], hash[10], hash[11], hash[12], hash[13], hash[14], hash[15], p16, p17, p18, p19, p20, p21, p22, p23, p16, p17, p18, p19, p20, p21, p22, p23); + Q256(p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31, p24, p25, p26, p27, p28, p29, p30, p31); + + Q256(p16, p17, p18, p19, p20, p21, p22, p23, hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7], p16, p17, p18, p19, p20, p21, p22, p23); + Q256(p24, p25, p26, p27, p28, p29, p30, p31, p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31); + + Q256(block[7], block[6], block[5], block[4], block[3], block[2], block[1], block[0], p16, p17, p18, p19, p20, p21, p22, p23, p16, p17, p18, p19, p20, p21, p22, p23); + Q256(p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31, p24, p25, p26, p27, p28, p29, p30, p31); + + hash[0] ^= block[8] ^ p16; + hash[1] ^= block[9] ^ p17; + hash[2] ^= block[10] ^ p18; + hash[3] ^= block[11] ^ p19; + hash[4] ^= block[12] ^ p20; + hash[5] ^= block[13] ^ p21; + hash[6] ^= block[14] ^ p22; + hash[7] ^= block[15] ^ p23; + hash[8] ^= block[0] ^ p24; + hash[9] ^= block[1] ^ p25; + hash[10] ^= block[2] ^ p26; + hash[11] ^= block[3] ^ p27; + hash[12] ^= block[4] ^ p28; + hash[13] ^= block[5] ^ p29; + hash[14] ^= block[6] ^ p30; + hash[15] ^= block[7] ^ p31; +} diff --git a/tools/prebuild/vendor/hashab/src/edonr.h b/tools/prebuild/vendor/hashab/src/edonr.h new file mode 100644 index 00000000..94932f1e --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/edonr.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +enum { EDONR256_BLOCK_WORDS = 16 }; + +typedef struct { + uint32_t hash[16]; +} EdonR256State; + +void EdonR256Init(EdonR256State* state); +void EdonR256Compress(EdonR256State* state, const uint32_t block[static EDONR256_BLOCK_WORDS]); diff --git a/tools/prebuild/vendor/hashab/src/expand.c b/tools/prebuild/vendor/hashab/src/expand.c new file mode 100644 index 00000000..7ea6fca2 --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/expand.c @@ -0,0 +1,56 @@ +#include "expand.h" + +#include "byteorder.h" +#include "hash.h" + +#include +#include + +static void transform_hash_chunk(const uint8_t hash[32], uint8_t output[32]) { + for (size_t i = 0; i < 8; i++) { + uint32_t word = load32_le(hash + i * 4); + uint32_t product = word * UINT32_C(0xc8180aff); + uint32_t transformed = (uint32_t)(uint8_t)(word * UINT32_C(0x6b)) | (uint32_t)(uint8_t)((product >> 8) * UINT32_C(0x95)) << 8 | (uint32_t)(uint8_t)((product >> 16) * UINT32_C(0x95)) << 16 | + (uint32_t)(uint8_t)((product >> 24) * UINT32_C(0x95)) << 24; + + store32_le(output + i * 4, transformed); + } +} + +int expand(uint8_t output[190], const uint8_t ciphertext_at_20[24], uint32_t selector, const uint8_t ciphertext_at_0[24]) { + uint8_t work[64] = {0}; + uint8_t digest[32]; + uint8_t chunk[32]; + + if (output == NULL || ciphertext_at_20 == NULL || ciphertext_at_0 == NULL) { + return -1; + } + + if (selector * UINT32_C(0xb26ed57f) != 20) { + return -1; + } + + for (size_t i = 0; i < 24; i++) { + work[i] = (uint8_t)(ciphertext_at_20[i] * UINT32_C(0x6f)); + } + for (size_t i = 0; i < 20; i++) { + work[24 + i] = (uint8_t)(ciphertext_at_0[i] * UINT32_C(0x6f)); + } + + store32_le(work + 48, 12); + + for (size_t iteration = 0; iteration < 6; iteration++) { + size_t offset = iteration * sizeof(chunk); + size_t remaining = 190 - offset; + size_t copy_length = remaining < sizeof(chunk) ? remaining : sizeof(chunk); + + store32_le(work + 44, (uint32_t)iteration); + if (hash(work, digest) != 0) { + return -1; + } + transform_hash_chunk(digest, chunk); + __builtin_memcpy(output + offset, chunk, copy_length); + } + + return 0; +} diff --git a/tools/prebuild/vendor/hashab/src/expand.h b/tools/prebuild/vendor/hashab/src/expand.h new file mode 100644 index 00000000..816cb240 --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/expand.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +enum { HASHAB_EXPAND_SEED_SIZE = 24, HASHAB_EXPAND_OUTPUT_SIZE = 190 }; + +int expand(uint8_t output[HASHAB_EXPAND_OUTPUT_SIZE], const uint8_t ciphertext_at_20[HASHAB_EXPAND_SEED_SIZE], uint32_t selector, const uint8_t ciphertext_at_0[HASHAB_EXPAND_SEED_SIZE]); diff --git a/tools/prebuild/vendor/hashab/src/hash.c b/tools/prebuild/vendor/hashab/src/hash.c new file mode 100644 index 00000000..9e6a3a4d --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/hash.c @@ -0,0 +1,166 @@ +#include "hash.h" + +#include "byteorder.h" +#include "edonr.h" + +#include +#include + +static uint32_t encoded_rol(uint32_t value, uint32_t count) { + uint32_t decoded = value * UINT32_C(0x8fa491df); + uint32_t shift = count & 31; + + return (decoded << shift) | (decoded >> ((32 - shift) & 31)); +} + +static const uint8_t hash_byte_table_0[256] = { + 0x41, 0xe9, 0x90, 0x48, 0x7a, 0x4a, 0x0b, 0xae, 0x92, 0x79, 0xbf, 0x2d, 0x46, 0x49, 0x28, 0x51, 0x06, 0x75, 0x1d, 0xce, 0xa1, 0x18, 0x3e, 0x5c, 0x03, 0x5a, 0x91, 0x24, 0x56, 0xe1, 0x77, 0x47, + 0x42, 0xa3, 0x78, 0x5f, 0x8d, 0x20, 0xdd, 0x2b, 0x40, 0x89, 0xd7, 0x15, 0x61, 0xde, 0x27, 0x21, 0xdc, 0x86, 0xfc, 0x6c, 0x97, 0xea, 0xb1, 0xbb, 0xbd, 0x71, 0xe0, 0x44, 0x87, 0x45, 0xf1, 0xb6, + 0x6d, 0x6a, 0xf2, 0x38, 0x82, 0x76, 0xcf, 0x8b, 0x6f, 0xc0, 0x55, 0x5e, 0x57, 0xad, 0xd2, 0x99, 0x7f, 0x2a, 0x68, 0x25, 0xd4, 0xc6, 0x94, 0xdf, 0x4f, 0x04, 0xc4, 0x39, 0xfb, 0x80, 0x62, 0x84, + 0x0e, 0xbe, 0xa9, 0xec, 0xe4, 0xa6, 0x1c, 0x37, 0xe5, 0x59, 0x35, 0x00, 0xd1, 0xe7, 0xfa, 0x1f, 0x0c, 0x73, 0xd8, 0xc8, 0xf9, 0xef, 0x13, 0xe6, 0x50, 0xcc, 0xc7, 0xd5, 0x7e, 0x19, 0x9f, 0xa2, + 0x63, 0xe2, 0xb5, 0x74, 0x66, 0x01, 0x09, 0xb7, 0x96, 0x33, 0x98, 0x1e, 0xc3, 0x4e, 0x08, 0xac, 0xda, 0xa5, 0x2e, 0xf4, 0xc2, 0x67, 0x81, 0x4d, 0xf5, 0xb9, 0xed, 0xa4, 0x3b, 0x11, 0x0a, 0xf7, + 0x05, 0x02, 0x12, 0x93, 0x16, 0x72, 0xb4, 0xe8, 0x9a, 0xba, 0xdb, 0x53, 0xaf, 0x1a, 0xb3, 0x22, 0x88, 0x31, 0xf0, 0x7c, 0xd0, 0xf3, 0x6e, 0x58, 0xfe, 0x8e, 0x64, 0xb0, 0xd9, 0x85, 0x2c, 0x9b, + 0xee, 0x3a, 0x8c, 0xf8, 0x43, 0x2f, 0x65, 0x14, 0x5d, 0x6b, 0xc5, 0x34, 0x26, 0x0f, 0xab, 0x1b, 0xa7, 0xc9, 0xe3, 0x54, 0x07, 0xbc, 0xff, 0xcb, 0x83, 0x30, 0xb8, 0x9c, 0xb2, 0x7d, 0xa8, 0x52, + 0x70, 0xeb, 0x9e, 0xcd, 0x5b, 0xc1, 0x10, 0x8a, 0x23, 0xd6, 0xa0, 0x8f, 0x36, 0x4b, 0x0d, 0x7b, 0x3f, 0x17, 0xd3, 0xaa, 0xfd, 0x9d, 0xca, 0x4c, 0x69, 0x29, 0xf6, 0x3d, 0x32, 0x3c, 0x60, 0x95, +}; + +static const uint8_t hash_byte_table_1[256] = { + 0x02, 0xbf, 0x20, 0x4f, 0xea, 0x48, 0x23, 0xc4, 0x6f, 0x84, 0x7d, 0xee, 0x1f, 0x41, 0x29, 0x97, 0xa1, 0xb4, 0x75, 0x2f, 0x73, 0xec, 0x2b, 0x70, 0x71, 0xe5, 0xb9, 0x24, 0x36, 0x3b, 0x49, 0x95, + 0xad, 0xd5, 0x6b, 0xc0, 0xb5, 0xb0, 0x80, 0x86, 0xd7, 0x79, 0xfe, 0xf4, 0x65, 0xdf, 0xaa, 0x33, 0x46, 0x96, 0x10, 0xed, 0x1b, 0xe0, 0x67, 0x90, 0x91, 0x57, 0xa4, 0xa2, 0x9d, 0x44, 0x27, 0x04, + 0x92, 0x52, 0xfd, 0x76, 0x7c, 0x7f, 0xae, 0xbb, 0xb1, 0xbe, 0x4c, 0x93, 0x07, 0x6a, 0x58, 0xfa, 0x4a, 0x7b, 0xcb, 0xe6, 0xac, 0xd1, 0xd2, 0x66, 0x08, 0x3c, 0x56, 0x62, 0x1c, 0xd9, 0x0f, 0xf0, + 0x01, 0xf9, 0x5c, 0x9a, 0xca, 0x19, 0x32, 0x82, 0x99, 0x13, 0xf3, 0xb8, 0x3d, 0xa9, 0xcf, 0xbc, 0x3a, 0x18, 0x00, 0xbd, 0x63, 0xa8, 0xc1, 0xdb, 0x64, 0x68, 0xd6, 0x14, 0x5e, 0x9e, 0x5d, 0xb7, + 0x72, 0xc7, 0xba, 0x53, 0x15, 0xf8, 0x8b, 0x5f, 0x39, 0xab, 0x5a, 0x8c, 0x88, 0x1a, 0x11, 0xa0, 0x0d, 0x26, 0x59, 0x51, 0xdc, 0xa7, 0x21, 0x0b, 0x3e, 0x2c, 0x6c, 0x42, 0x55, 0x3f, 0x47, 0x0a, + 0xcd, 0xd4, 0xe8, 0xa5, 0xff, 0x81, 0x0e, 0xd8, 0xb6, 0xd3, 0xef, 0xdd, 0x37, 0x50, 0x6d, 0x1e, 0xc6, 0xeb, 0xe3, 0x60, 0xf6, 0x28, 0xf7, 0x87, 0x7e, 0xb2, 0xfb, 0x0c, 0x8f, 0xcc, 0x2a, 0x78, + 0x8d, 0x9b, 0xe2, 0x1d, 0x85, 0x09, 0x06, 0x6e, 0x8e, 0x89, 0x77, 0x83, 0xa6, 0x98, 0x12, 0xa3, 0x4d, 0x40, 0x31, 0xe4, 0xc2, 0xf5, 0xda, 0x69, 0x03, 0x05, 0xf2, 0xe7, 0x17, 0x30, 0xe1, 0x25, + 0xd0, 0xc5, 0x34, 0x38, 0xde, 0xc8, 0x94, 0x8a, 0x16, 0x4e, 0xb3, 0x43, 0x74, 0xe9, 0x35, 0xfc, 0x22, 0x61, 0xc3, 0x45, 0x7a, 0x5b, 0xf1, 0xc9, 0x2d, 0x9c, 0x9f, 0xce, 0x2e, 0xaf, 0x4b, 0x54, +}; + +static const uint8_t hash_byte_table_2[256] = { + 0xd1, 0xc9, 0xe3, 0x5a, 0xa0, 0x31, 0x3d, 0x46, 0x7e, 0x26, 0x9c, 0x0d, 0x51, 0xd5, 0x86, 0x50, 0xfd, 0x72, 0x68, 0x8c, 0x36, 0x71, 0x74, 0x8b, 0xd8, 0xb5, 0x12, 0x13, 0xbb, 0xd6, 0xec, 0xa8, + 0x81, 0x14, 0x89, 0x70, 0x91, 0x4e, 0x2c, 0x96, 0xeb, 0xa1, 0xc0, 0x16, 0xe0, 0xc6, 0x22, 0x98, 0x83, 0x6a, 0xda, 0x27, 0xa6, 0xc3, 0x95, 0x49, 0x73, 0xfb, 0xcc, 0x5c, 0x82, 0xfe, 0x62, 0xc4, + 0xfa, 0xad, 0x41, 0xd7, 0xd2, 0x92, 0xdb, 0xae, 0x17, 0x8e, 0xf6, 0xd4, 0x47, 0xc5, 0x29, 0x2f, 0xe4, 0x54, 0x80, 0x5f, 0x23, 0x2b, 0x79, 0xdd, 0x61, 0xf8, 0x05, 0xd9, 0x0f, 0x3b, 0x34, 0xdc, + 0x9e, 0x6c, 0x76, 0xac, 0x6d, 0xa2, 0x0e, 0x9f, 0x7c, 0x93, 0x08, 0xbf, 0x6b, 0x2e, 0xe9, 0x7f, 0x55, 0x65, 0x0c, 0xf1, 0x3f, 0xea, 0x6e, 0x4c, 0x64, 0x11, 0xd3, 0x52, 0x01, 0x94, 0x56, 0x1d, + 0x3c, 0xbd, 0xb7, 0xf3, 0x43, 0x0a, 0xcd, 0xde, 0x7a, 0xc7, 0xb6, 0x06, 0x67, 0x1c, 0xbc, 0x18, 0xb1, 0x44, 0x1f, 0xff, 0x90, 0x10, 0x8a, 0xe5, 0xc1, 0xab, 0xcb, 0xb8, 0x8f, 0xba, 0x85, 0x66, + 0x19, 0xcf, 0x4a, 0x9d, 0x4d, 0x03, 0x1e, 0xf4, 0x24, 0xce, 0x30, 0xfc, 0x78, 0x07, 0xca, 0x15, 0x7b, 0x99, 0xf9, 0x38, 0xf7, 0xf2, 0x9a, 0xf5, 0x00, 0xe8, 0x37, 0x42, 0x8d, 0x84, 0x53, 0xef, + 0xd0, 0xbe, 0x60, 0x1a, 0xb0, 0xaf, 0x69, 0x0b, 0xed, 0x4b, 0x87, 0x5e, 0xa7, 0x32, 0x7d, 0x77, 0xaa, 0xf0, 0xb4, 0x59, 0x25, 0xb3, 0x1b, 0x57, 0xa9, 0x5d, 0x88, 0xe2, 0x40, 0xee, 0x48, 0x9b, + 0xb2, 0xa3, 0x2a, 0x20, 0x21, 0x02, 0xc2, 0x63, 0x45, 0x6f, 0x58, 0xa5, 0x09, 0xe7, 0x2d, 0x4f, 0x5b, 0x33, 0x3e, 0xe6, 0xb9, 0x35, 0xdf, 0x75, 0x04, 0x28, 0xa4, 0x39, 0x3a, 0xe1, 0x97, 0xc8, +}; + +static const uint8_t hash_byte_table_3[256] = { + 0x8f, 0x37, 0x1a, 0x93, 0xaf, 0xdc, 0x24, 0x12, 0xf6, 0x89, 0x81, 0xf4, 0xe7, 0x27, 0xe0, 0xc1, 0x67, 0x15, 0x98, 0xc4, 0xd9, 0x9f, 0x7c, 0x16, 0x49, 0xec, 0x28, 0x26, 0xd2, 0x65, 0x7d, 0x11, + 0x74, 0x8d, 0xae, 0x61, 0x0a, 0xc0, 0x19, 0x58, 0x6f, 0x82, 0xb9, 0xdf, 0x20, 0x2d, 0x54, 0x4d, 0xf1, 0xd0, 0x59, 0x7a, 0x95, 0xb0, 0xca, 0x66, 0xdb, 0xea, 0x63, 0x84, 0x2c, 0xa4, 0xf3, 0x06, + 0xd3, 0x01, 0x9c, 0x60, 0x6b, 0xc6, 0x6a, 0xe2, 0xf8, 0x2f, 0x32, 0x8c, 0xc3, 0x78, 0xc8, 0xd6, 0x69, 0xb6, 0x92, 0xda, 0x99, 0xd5, 0xff, 0x2a, 0x3d, 0x0e, 0xb1, 0x41, 0x44, 0xe6, 0xfe, 0x50, + 0xfa, 0xbb, 0xd4, 0xb3, 0x45, 0xcf, 0x6c, 0xcb, 0x0c, 0x09, 0xed, 0x3c, 0xab, 0xd8, 0x2b, 0xee, 0x52, 0xad, 0x7f, 0x94, 0x38, 0x53, 0x29, 0xfb, 0xd1, 0x80, 0x5a, 0x03, 0x51, 0x14, 0xa2, 0x77, + 0xc2, 0x0d, 0x04, 0x71, 0x3e, 0xfc, 0xf5, 0x90, 0x39, 0x62, 0xe9, 0x3f, 0x17, 0xf0, 0xb8, 0x1c, 0xe4, 0xb5, 0xbf, 0x76, 0xfd, 0x83, 0xc7, 0x36, 0xa7, 0xb4, 0xac, 0xe1, 0x00, 0x91, 0x1d, 0xb2, + 0x7b, 0x8e, 0x4b, 0x7e, 0xf9, 0x85, 0xbd, 0xa9, 0xbc, 0x9e, 0x75, 0xcc, 0x2e, 0x79, 0xce, 0x13, 0xbe, 0xaa, 0xe3, 0x0f, 0xa8, 0xa0, 0xba, 0x3b, 0x43, 0x5e, 0x21, 0x97, 0x02, 0x31, 0xa3, 0xcd, + 0x4e, 0xe5, 0x30, 0x70, 0x5c, 0xdd, 0xf2, 0x9d, 0x1f, 0x07, 0x8b, 0x57, 0x42, 0x4f, 0xe8, 0x22, 0xeb, 0x05, 0x5f, 0xde, 0x10, 0x68, 0x6d, 0x1b, 0x08, 0xa1, 0xa6, 0x72, 0x34, 0x9a, 0x96, 0xd7, + 0x56, 0xef, 0xb7, 0x4a, 0x64, 0x87, 0x48, 0x8a, 0x4c, 0x3a, 0x18, 0x88, 0x86, 0x46, 0xc5, 0x9b, 0x5d, 0xf7, 0x6e, 0xc9, 0x35, 0x25, 0x5b, 0x0b, 0x33, 0x73, 0x1e, 0x55, 0x23, 0x47, 0x40, 0xa5, +}; + +static uint32_t hash_preprocess_word(uint32_t input) { + const uint32_t input_multiplier = UINT32_C(0xa272be3f); + const uint32_t combine_multiplier = UINT32_C(0x7f8631bf); + const uint32_t byte_multiplier = UINT32_C(0xc83bebc7); + const uint32_t word_bias = UINT32_C(0x44e57c7e); + uint32_t byte0 = hash_byte_table_0[input & 0xff]; + uint32_t byte1 = hash_byte_table_1[(input >> 8) & 0xff]; + uint32_t byte2 = hash_byte_table_2[(input >> 16) & 0xff]; + uint32_t byte3 = hash_byte_table_3[input >> 24]; + uint32_t high_mask; + uint32_t partial; + uint32_t scaled; + + scaled = (((byte0 * UINT32_C(0xc9) * UINT32_C(0x79)) & 0xff) << 8) * input_multiplier; + + byte1 = byte1 * UINT32_C(0xc9) & 0xff; + high_mask = ~(byte1 * UINT32_C(0x79)) | 0xff; + partial = byte1 * byte_multiplier + word_bias + ~high_mask * input_multiplier + high_mask * word_bias + scaled; + scaled = partial * combine_multiplier * UINT32_C(0x100) * input_multiplier; + + high_mask = ~(byte2 * UINT32_C(0x79)) | 0xff; + partial = byte2 * byte_multiplier + word_bias + ~high_mask * input_multiplier + high_mask * word_bias + partial + scaled; + + high_mask = ~(byte3 * UINT32_C(0x79)) | 0xff; + partial = high_mask * word_bias + byte3 * byte_multiplier + word_bias + ~high_mask * input_multiplier + partial * combine_multiplier * UINT32_C(0x100) * input_multiplier + partial; + + partial = partial * combine_multiplier * UINT32_C(0x100) * input_multiplier + partial; + return partial * combine_multiplier; +} + +static uint8_t hash_byte_index(uint32_t value) { + return (uint8_t)(value * UINT32_C(0xbf)); +} + +static uint32_t hash_state_to_word(uint32_t canonical_state_word) { + const uint32_t state_encode_multiplier = UINT32_C(0xc2ed397f); + const uint32_t input_multiplier = UINT32_C(0xa272be3f); + const uint32_t index_decode = UINT32_C(0x7f8631bf); + const uint32_t index_multiplier = UINT32_C(0x5d8d41c1); + const uint32_t state_multiplier = UINT32_C(0xbcb95b41); + const uint32_t byte_multiplier = UINT32_C(0xc83bebc7); + const uint32_t word_bias = UINT32_C(0x44e57c7e); + const uint32_t rotate_input = UINT32_C(0x1e051c21); + const uint32_t encoded_or_multiplier = UINT32_C(0x00f39c82); + uint32_t mapped; + uint32_t high_mask; + uint32_t rotated; + uint32_t combined; + uint32_t state_word = canonical_state_word * state_encode_multiplier; + + mapped = hash_byte_table_0[hash_byte_index(state_word * state_multiplier)]; + mapped = mapped * UINT32_C(0xc9) & 0xff; + high_mask = ~(mapped * UINT32_C(0x79)) | 0xff; + combined = state_word * state_multiplier + high_mask * word_bias + mapped * byte_multiplier + word_bias + ~high_mask * input_multiplier; + rotated = encoded_rol(combined * rotate_input, 11); + rotated *= input_multiplier; + + mapped = hash_byte_table_1[hash_byte_index(rotated)]; + mapped = mapped * UINT32_C(0xc9) & 0xff; + high_mask = ~(mapped * UINT32_C(0x79)) | 0xff; + combined = ~high_mask * input_multiplier + mapped * byte_multiplier + word_bias + high_mask * word_bias; + rotated = encoded_rol((((combined * encoded_or_multiplier - 1) | (rotated * encoded_or_multiplier - 1)) * input_multiplier + rotated + combined - index_multiplier) * rotate_input, 10); + + mapped = hash_byte_table_2[hash_byte_index(rotated * input_multiplier)]; + high_mask = ~(mapped * UINT32_C(0x79)) | 0xff; + combined = mapped * byte_multiplier + word_bias + ~high_mask * input_multiplier + high_mask * word_bias; + rotated = encoded_rol((rotated * input_multiplier - combined) * rotate_input, 7); + return rotated * input_multiplier * index_decode; +} + +static void hash_final(const uint32_t state[16], uint8_t output[32]) { + for (size_t i = 0; i < 8; i++) { + uint32_t value = state[8 + i] * UINT32_C(0xf46ef4ff); + store32_le(output + i * 4, value); + } +} + +int hash(uint8_t work[64], uint8_t output[32]) { + EdonR256State edon_r_state; + uint32_t* state; + uint32_t block[16]; + + if (work == NULL || output == NULL) { + return -1; + } + + for (size_t i = 0; i < 16; i++) { + block[i] = hash_preprocess_word(load32_le(work + i * 4)); + } + EdonR256Init(&edon_r_state); + state = edon_r_state.hash; + + for (size_t round = 0; round < 4; round++) { + EdonR256Compress(&edon_r_state, block); + if (round != 3) { + for (size_t i = 0; i < 16; i++) { + block[i] = hash_state_to_word(state[i]); + } + } + } + + for (size_t i = 0; i < 16; i++) { + store32_le(work + i * 4, block[i]); + } + hash_final(state, output); + return 0; +} diff --git a/tools/prebuild/vendor/hashab/src/hash.h b/tools/prebuild/vendor/hashab/src/hash.h new file mode 100644 index 00000000..5b1fffbb --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/hash.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +enum { HASHAB_HASH_WORK_SIZE = 64, HASHAB_HASH_OUTPUT_SIZE = 32 }; + +int hash(uint8_t work[HASHAB_HASH_WORK_SIZE], uint8_t output[HASHAB_HASH_OUTPUT_SIZE]); diff --git a/tools/prebuild/vendor/hashab/src/reduce.c b/tools/prebuild/vendor/hashab/src/reduce.c new file mode 100644 index 00000000..2fad7462 --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/reduce.c @@ -0,0 +1,197 @@ +#include "reduce.h" + +#include +#include + +enum { REDUCE_INPUT_SIZE = 190, REDUCE_OUTPUT_SIZE = 16 }; + +static uint32_t rotate_left(uint32_t value, unsigned count) { + return (value << count) | (value >> (32 - count)); +} + +static uint32_t rotate_left_or_ones(uint32_t value, unsigned count) { + return count ? rotate_left(value, count) : UINT32_MAX; +} + +static uint32_t rotate_right_or_ones(uint32_t value, unsigned count) { + return count ? (value >> count) | (value << (32 - count)) : UINT32_MAX; +} + +static uint32_t choose(uint32_t mask, uint32_t when_set, uint32_t when_clear) { + return (mask & when_set) | (~mask & when_clear); +} + +static uint32_t maj(uint32_t a, uint32_t b, uint32_t c) { + return (a & b) | (a & c) | (b & c); +} + +static uint8_t fold_word(uint32_t word) { + uint8_t folded = (uint8_t)word; + folded += (uint8_t)(word >> 8); + folded ^= (uint8_t)(word >> 16); + folded += (uint8_t)(word >> 24); + return folded; +} + +static const uint32_t key_seed[64] = {UINT32_C(0x5099fdb1), UINT32_C(0x9bbf4349), UINT32_C(0x5cd7686c), UINT32_C(0x92aebb6d), UINT32_C(0x2df36af2), UINT32_C(0x552f4079), UINT32_C(0x219dfeae), UINT32_C(0x343eb35a), + UINT32_C(0x5dac0284), UINT32_C(0x91bae1f7), UINT32_C(0x64ec9d83), UINT32_C(0xa2ff04aa), UINT32_C(0x46df36ef), UINT32_C(0xd0d7ebdc), UINT32_C(0xf334c68d), UINT32_C(0x99a81ecf), + UINT32_C(0x8a85f4e4), UINT32_C(0xb96ef2ae), UINT32_C(0x2e132f59), UINT32_C(0x91d1d6ee), UINT32_C(0x73628eaa), UINT32_C(0x7bfcd2c8), UINT32_C(0xcbc8db84), UINT32_C(0x2c358ebb), + UINT32_C(0xcdce075f), UINT32_C(0x096dddf4), UINT32_C(0xf86bae65), UINT32_C(0xb408b83c), UINT32_C(0x3aa5ae3b), UINT32_C(0xef04fe54), UINT32_C(0xcbae433f), UINT32_C(0xbafb26c5), + UINT32_C(0x546e42be), UINT32_C(0x694466f8), UINT32_C(0xc0141741), UINT32_C(0x51fb98d5), UINT32_C(0x6a53c93d), UINT32_C(0x784b88bc), UINT32_C(0xe689e637), UINT32_C(0x720cffaf), + UINT32_C(0x3fb860f4), UINT32_C(0x1d78ec71), UINT32_C(0xa8a5ae0b), UINT32_C(0x6eeaa5b8), UINT32_C(0x4fd82294), UINT32_C(0xa61ea409), UINT32_C(0x3919851c), UINT32_C(0x577402c5), + UINT32_C(0xe8b09ca2), UINT32_C(0x11676e33), UINT32_C(0xfa7fa4ec), UINT32_C(0x707bcd8d), UINT32_C(0x52c20d40), UINT32_C(0x734b0bfb), UINT32_C(0x744241ff), UINT32_C(0x089fb6e0), + UINT32_C(0xe1cea850), UINT32_C(0xa91e1a9a), UINT32_C(0x9e9db0f3), UINT32_C(0x6fedd35b), UINT32_C(0xee066a04), UINT32_C(0x178a973a), UINT32_C(0x00aa4e6f), UINT32_C(0xf2994295)}; + +static const uint32_t state_seed[64] = { + UINT32_C(0xf825e4f8), UINT32_C(0x202997a3), UINT32_C(0x65055cfe), UINT32_C(0xd588806b), UINT32_C(0x9a396a74), UINT32_C(0x210d95d7), UINT32_C(0x760fa7a5), UINT32_C(0x5b85496a), + UINT32_C(0x65ad1cb7), UINT32_C(0xd93ce632), UINT32_C(0xb44bfe86), UINT32_C(0x97e5e021), UINT32_C(0x774af9ea), UINT32_C(0xfed3b7c5), UINT32_C(0x0d36b3e5), UINT32_C(0x7d7242ac), + UINT32_C(0x648df3be), UINT32_C(0xfda267a7), UINT32_C(0x0abf3e91), UINT32_C(0xea9d68c3), UINT32_C(0xb9de0b6e), UINT32_C(0xd4b020aa), UINT32_C(0x2af3296f), UINT32_C(0xb5bd052c), + UINT32_C(0xb0b3bf8e), UINT32_C(0x4b3d7bfe), UINT32_C(0xfd689550), UINT32_C(0xc3c8d629), UINT32_C(0xf1683297), UINT32_C(0x70e73403), UINT32_C(0x7b681b65), UINT32_C(0x4c566e07), + UINT32_C(0x9d39df6e), UINT32_C(0x7ced44ca), UINT32_C(0xd85121e1), UINT32_C(0x8f89dab8), UINT32_C(0x49d5dcd4), UINT32_C(0x630a3cdd), UINT32_C(0xa43cd850), UINT32_C(0xe05011fa), + UINT32_C(0xc79a3eb2), UINT32_C(0xe5912da3), UINT32_C(0x4683718c), UINT32_C(0xd0a154f1), UINT32_C(0x9c6aec15), UINT32_C(0x0e0d0468), UINT32_C(0x27de8990), UINT32_C(0x412adb3d), + UINT32_C(0xf8775990), UINT32_C(0xe5e21243), UINT32_C(0xc753f819), UINT32_C(0x0663c6ef), UINT32_C(0x93c9d773), UINT32_C(0xab28ffff), UINT32_C(0x5204a510), UINT32_C(0x11a9d2c5), + UINT32_C(0x6ec18106), UINT32_C(0x20ea572e), UINT32_C(0x6fe51b3a), UINT32_C(0x6e812c3d), UINT32_C(0x014b907b), UINT32_C(0x5c86830b), UINT32_C(0x9934a3d8), UINT32_C(0x3c57d8a8)}; + +static const uint32_t table[64] = {UINT32_C(0x3dd3c06d), UINT32_C(0x02eda8fb), UINT32_C(0xb82d5a58), UINT32_C(0xe4d2adec), UINT32_C(0x5c66b0d9), UINT32_C(0x5a4f2459), UINT32_C(0x08da381e), UINT32_C(0x35ba3a4c), + UINT32_C(0xa86bb294), UINT32_C(0xfb7fc66a), UINT32_C(0xf53a25f2), UINT32_C(0x5dfe38f1), UINT32_C(0x6cb6b4af), UINT32_C(0xebeb4c63), UINT32_C(0x66209bd7), UINT32_C(0x29271b4e), + UINT32_C(0x7a91205b), UINT32_C(0x7993f4f5), UINT32_C(0x8fb73ed6), UINT32_C(0x974262cd), UINT32_C(0xf47cab3b), UINT32_C(0x757b65cc), UINT32_C(0x92e55b93), UINT32_C(0xfa401668), + UINT32_C(0xe8f42bcc), UINT32_C(0xa1796665), UINT32_C(0xc544210e), UINT32_C(0x4ff7d275), UINT32_C(0x1090b646), UINT32_C(0xad8b53cb), UINT32_C(0x7c871580), UINT32_C(0x67fb52d6), + UINT32_C(0x5ae5b9a1), UINT32_C(0x69becb2c), UINT32_C(0xd2b974af), UINT32_C(0xb709f90d), UINT32_C(0x95671e5b), UINT32_C(0x2479e36e), UINT32_C(0x305facbd), UINT32_C(0xd7524214), + UINT32_C(0xd7f29e79), UINT32_C(0x30cf42db), UINT32_C(0x86bad405), UINT32_C(0x370d4cb9), UINT32_C(0xaf348764), UINT32_C(0xc5ea190b), UINT32_C(0x06b2f0c7), UINT32_C(0x62ecaf33), + UINT32_C(0x59f1cee3), UINT32_C(0xf7f35186), UINT32_C(0xf89d98cf), UINT32_C(0x1c9e3527), UINT32_C(0x31e62b5c), UINT32_C(0x89851e84), UINT32_C(0x9d67e93a), UINT32_C(0xec33dc26), + UINT32_C(0x913bca92), UINT32_C(0x92f8982e), UINT32_C(0x5f160f82), UINT32_C(0x2691b03f), UINT32_C(0x433589fd), UINT32_C(0x18d97e1a), UINT32_C(0xba586b58), UINT32_C(0x8907cb75)}; + +int reduce(const uint8_t input[REDUCE_INPUT_SIZE], uint8_t output[REDUCE_OUTPUT_SIZE]) { + uint32_t key[64]; + uint32_t state[64]; + uint32_t work[128]; + uint32_t s6, s7, s8, s9, s10, s18, s19, s21, s22, s24, s25, s26; + + if (input == NULL || output == NULL) { + return -1; + } + + __builtin_memcpy(key, key_seed, sizeof(key)); + __builtin_memcpy(state, state_seed, sizeof(state)); + + for (size_t i = 0; i < 128; i++) { + uint32_t word = 0; + for (size_t byte = 0; byte < 4; byte++) { + word |= (uint32_t)(uint8_t)(input[(4 * i + byte) % REDUCE_INPUT_SIZE] * UINT32_C(0xbd)) << (24 - 8 * byte); + } + work[i] = word; + } + + for (size_t pass = 0; pass < 4; pass++) { + for (size_t j = 0; j < 128; j++) { + work[j] = (work[j] ^ rotate_left(work[(j - 13) & 127], 3)) + rotate_left(work[(j + 71) & 127], 5) - rotate_left(work[(j + 101) & 127], 23); + } + } + + s7 = (work[22] / 5) * UINT32_C(0xc3456f96); + s8 = maj(state[60], table[63], UINT32_C(0x908d4d25)) / 3 + UINT32_C(0xef9695dd); + key[42] += ((table[work[33] & 63] & UINT32_C(0x3cba9069)) + UINT32_C(0x43440114)) / 5; + s9 = choose(state[26], state[work[87] & 63], ~UINT32_C(0x3cba9069)); + uint32_t rotation = rotate_left_or_ones(UINT32_C(0x6f72b2da), -state[work[55] & 63] & 31); + uint32_t k = key[s7 & 63] / 5; + uint32_t selected = choose(state[42], ~UINT32_C(0x3047874d), ~UINT32_C(0xb0dea603)); + s10 = rotation ^ UINT32_C(0x908d4d25); + key[63] -= maj(selected, k, s7); + state[25] += key[s7 & 63]; + state[60] = choose(key[21], s8, ~UINT32_C(0x1fb7d8d3)) * work[122]; + work[5] += choose(s7, ~UINT32_C(0xd36d6fb6), table[work[120] & 63]); + + key[35] += UINT32_C(0xb1a7e76f); + s18 = key[s10 & 63] ^ UINT32_C(0x908d4d25); + state[44] ^= 3 * choose(UINT32_C(0xc3456f96), table[29], table[work[18] & 63]); + work[107] += state[work[44] & 63] + 1; + uint32_t inverted_rotation = ~rotate_right_or_ones(UINT32_C(0xb0dea603), work[71] & 31); + key[54] -= maj(state[29], inverted_rotation, ~UINT32_C(0x6f72b2da)); + work[62] ^= UINT32_C(0xdbd3d289); + key[19] += state[s7 & 63] / 5 - maj(state[45], table[work[45] & 63], ~UINT32_C(0x4cf50fa3)); + key[49] = table[state[s18 & 63] * state[s18 & 63] & 63]; + key[21] ^= table[work[27] & 63]; + s19 = UINT32_C(0x908d4d24) - rotate_left_or_ones(UINT32_C(0x98b99bd8), state[60] * state[41] & 31); + state[10] = work[73]; + key[20] ^= rotate_left_or_ones(~state[work[98] & 63], key[60] & 31); + key[49] += choose(state[56], key[41], table[s19 & 63]) >> 1; + uint32_t table_word = table[work[10] & 63]; + state[24] ^= choose(key[39], state[37], table_word * table_word * table_word); + s6 = table[work[43] & 63] & ~UINT32_C(0x45059986); + s6 = (s6 | UINT32_C(0x41050082)) ^ UINT32_C(0x3cba9069); + uint32_t rotated = rotate_right_or_ones(~s18, table[work[31] & 63] & 31); + state[13] ^= (key[work[50] & 63] & ~UINT32_C(0x147f3886)) | ~(rotated | UINT32_C(0xeb80c779)); + + key[28] -= choose(key[35], maj(key[58], table[s9 & 63], UINT32_C(0x908d4d25)), ~key[29]); + state[23] += ~maj(s8, state[work[106] & 63], UINT32_C(0x2725fe3b)); + state[24] ^= work[118] * work[118]; + key[35] ^= maj(state[54], 2 * work[108], state[41] * state[41] * state[41]); + work[34] ^= choose(s18, state[work[39] & 63] ^ state[37], table[29] + UINT32_C(0x02e5a7e3)); + state[42] ^= choose(state[40], state[work[35] & 63], work[78] * work[78]); + work[49] += 3 * key[work[39] & 63] + UINT32_C(0x62116be6); + key[48] = s7; + state[62] = key[s19 & 63] - state[work[93] & 63] - UINT32_C(0xef9695dd); + key[55] += key[60] / 3 - state[work[112] & 63] / 3; + s21 = maj(s6, table[60] + UINT32_C(0x4d485456), UINT32_C(0x41d44d69)) + UINT32_C(0x3cba9069); + s25 = s9 ^ key[37]; + state[55] += UINT32_C(0x1d0f1b76); + work[77] -= table[(state[60] & 127) >> 1]; + state[29] = key[work[68] & 63] / 15; + state[46] ^= table[table[table[37] & 63] & 63]; + s22 = UINT32_C(0x4f2159fc) - maj(state[59], table[29], table[60] / 5); + s24 = choose(UINT32_C(0xbd06dd88), maj(UINT32_C(0x59089be9), table[work[16] & 63], state[60]), table[60]) + UINT32_C(0x4f2159fc); + work[10] = UINT32_C(0xde2849dd); + work[8] += maj(choose(state[52], UINT32_C(0x908d4d25), UINT32_C(0x5987b420)), work[48] ^ s22, UINT32_C(0x51127d0f)); + state[20] += table[work[1] & 63]; + work[32] += 1 + rotate_right_or_ones(UINT32_C(0x0c7ad237), s22 * s22 * s22 & 31); + state[4] -= choose(key[35], UINT32_C(0x4f2159fc), UINT32_C(0xc0540978)) | ~rotate_right_or_ones(UINT32_C(0x46ef99f4), key[41] & 31); + state[60] = ~rotate_left_or_ones(~(work[62] / 3), (work[22] + key[work[93] & 63]) & 31); + key[30] = UINT32_C(0x3cba9069); + key[10] = ~(table[work[122] & 63] + UINT32_C(0x972da190)); + state[46] += maj(state[8], table[work[35] & 63] | table[41], UINT32_C(0xf3d39067)); + s26 = UINT32_C(0x3cba9069) - ((table[work[67] & 63] | table[work[120] & 63]) - table[work[98] & 63] - 1); + uint32_t word = state[work[38] & 63] >> 1; + state[17] ^= word * word * word; + state[44] += table[62]; + key[26] -= table[s6 & 63]; + uint32_t first = choose(UINT32_C(0x313ec15d), key[s6 & 63], state[60]); + uint32_t second = (table[work[79] & 63] & UINT32_C(0x7fbbd9fd)) | UINT32_C(0x0c201068); + key[35] -= maj(key[34], first, second); + key[16] ^= maj(state[43], work[27] >> 1, state[41] | state[37]); + work[33] = UINT32_C(0x1c3e9665); + key[58] -= work[118]; + state[21] += key[60]; + state[10] -= state[60]; + + const uint8_t initial[REDUCE_OUTPUT_SIZE] = { + (uint8_t)(UINT32_C(0xba) + s25), + UINT8_C(0x97), + UINT8_C(0x97), + (uint8_t)(UINT32_C(0x9b) + s24), + (uint8_t)(UINT32_C(0xba) + s8), + UINT8_C(0x97), + (uint8_t)(UINT32_C(0xd3) + s22), + UINT8_C(0x97), + (uint8_t)(UINT32_C(0xbe) + s18 + s6), + (uint8_t)(UINT32_C(0x27) + s7), + UINT8_C(0x4c), + (uint8_t)(UINT32_C(0x27) + s19), + (uint8_t)(UINT32_C(0xbe) + s10 + s21), + (uint8_t)(UINT32_C(0xe3) + s26), + UINT8_C(0x4c), + (uint8_t)(UINT32_C(0x27) + work[92]), + }; + + for (size_t i = 0; i < REDUCE_OUTPUT_SIZE; i++) { + uint8_t reduced = initial[i]; + for (size_t j = i; j < 64; j += REDUCE_OUTPUT_SIZE) { + reduced ^= (uint8_t)key[j] ^ (uint8_t)state[j]; + } + for (size_t j = i; j < 128; j += REDUCE_OUTPUT_SIZE) { + reduced ^= fold_word(work[j]); + } + output[i] = (uint8_t)(UINT32_C(0xa5) * reduced); + } + + return 0; +} diff --git a/tools/prebuild/vendor/hashab/src/reduce.h b/tools/prebuild/vendor/hashab/src/reduce.h new file mode 100644 index 00000000..9409f26c --- /dev/null +++ b/tools/prebuild/vendor/hashab/src/reduce.h @@ -0,0 +1,5 @@ +#pragma once + +#include + +int reduce(const uint8_t input[190], uint8_t output[16]); diff --git a/tools/prebuild/vendor/hashab/test-data.json b/tools/prebuild/vendor/hashab/test-data.json new file mode 100644 index 00000000..81d29b3c --- /dev/null +++ b/tools/prebuild/vendor/hashab/test-data.json @@ -0,0 +1,502 @@ +[ + { + "sha1": "915dd7203086aeea302740e97febb7aefa9ca9f1", + "uuid": "f832c65917da6785", + "target": "030056d7474d4843465449bd444266eb516bf4552726b94169d1424c4f433a8208fc5345e34507c3fe4a8f4e8a5052664515aad30e4b195c57" + }, + { + "sha1": "6a1e089e993ea643ac8a03732d65d70fb49ab2be", + "uuid": "0f78bc1413025880", + "target": "03005643474d48e7465449104442732251aebf554cbd08415bf5a14c4f43d6ad37e853c84145b182034a724e5b505226158c73bf284bdeb757" + }, + { + "sha1": "91e301635017bee75e2c603ddf6513dc8d3230f5", + "uuid": "a9a79ad5c49c8e17", + "target": "030056e0474d4865465449da4442081551880e5512e05341db5db54c4f431cd9e18a538806458afbb74ac94eac505290473f53250f4be0be57" + }, + { + "sha1": "b0701b04c0b46dcbc8f488b0e075d54ecfa73277", + "uuid": "397a06be88415bd9", + "target": "03005620474d48af465449914442f6225177fa55330d0a410d2d254c4f437b1d1a3253be5845be83274af94e6c505225768bdbfffd4b0f7057" + }, + { + "sha1": "1a98e53fd5dd47a9c84af12d091ecf76c1f270f9", + "uuid": "05249e5de1935cc1", + "target": "03005627474d4883465449bd44421ed1513fac551dd858418e59794c4f4331ce035453cdca45be50244a704efe505286c2209121c64b802d57" + }, + { + "sha1": "5a05d308d206a7a73ab4c329dfb97d402a7702f1", + "uuid": "c2882429a6085fde", + "target": "03005609474d484a4654492b4442208e51324a55f4288d4145891c4c4f43655817fe53001a45aa407a4a1e4ea250527b80b4ef72b24b281757" + }, + { + "sha1": "39d55281f69af2f1f9fbf1569df4e90eb91485d0", + "uuid": "84c08f2a2fed6b75", + "target": "030056a0474d48ca4654497044425cb951e39a557326494122c79b4c4f43f25ed85453c52c458035314a9c4ef05052ed2df42079834b8b9357" + }, + { + "sha1": "2df046403f400949b5d07a423537151dac94cbea", + "uuid": "d2cb1c7a39b74a16", + "target": "0300561b474d489046544949444293ec51737a55042766410665274c4f433d62208253ea9f45ce182d4adc4e3e50528c6dec7d317e4ba1ad57" + }, + { + "sha1": "bcbfa11c37bfb2d68a8178c7a9aa5df3b85ced49", + "uuid": "4efa5451e57ca17f", + "target": "03005665474d480e465449fb4442af8651338655967e0441e0cb1e4c4f43ac90070f533175454202624afb4e855052e7793288071a4b76ff57" + }, + { + "sha1": "f837c5dd970640c7f56b77483908b22280589071", + "uuid": "7a6366f870c3dca6", + "target": "03005688474d483e4654490b444252b25191f055abfd7241cb5d3f4c4f431757d01b53b5044535fd5b4aa24e0b5052667f20a319ae4b24fb57" + }, + { + "sha1": "7a13aee3a981521a20aab75ac424a3a7a2f0a0b2", + "uuid": "0b39a6aa6661ba35", + "target": "030056dc474d487c46544997444282485173ae5565e66441cacf8c4c4f4391eaf025534b5345a66d374a594e3a505270a79827b3734b3e7757" + }, + { + "sha1": "33902379cac88f3710b632ab4cb205f1f03e78be", + "uuid": "58574fd90081de4d", + "target": "03005687474d489b4654498e444250005183d45528d636414f1d5c4c4f4389499cbc53c403456790c84a714ecf5052968dfb3bde634bc73757" + }, + { + "sha1": "583866c486c2d1f5b60c31818939565e87e0a5f8", + "uuid": "194e3e7db3a76e52", + "target": "030056cd474d48a5465449524442e1195125d855ed8d1c41470e394c4f43e9c7935c53784d45a65e624a504e795052696c864da1644b188357" + }, + { + "sha1": "3de81dff1a6a747b24c89e08c4bf81d2bd8e36a6", + "uuid": "30a45d064ab4e632", + "target": "0300562e474d48ad465449ea44428f9a519ae755d8237a413034b94c4f43bc5f395153710f45c9bdee4a604e925052c1a9b9ddbb534b877d57" + }, + { + "sha1": "0d9bac217417cadcb8275ee1fbfea89181adc933", + "uuid": "dc7b2ef3c668ba31", + "target": "0300567a474d4899465449eb4442df79517fc4550dfa98414df3c74c4f434a734c3c53a1fd45ff5ce74a934ec15052f8bf4cf214264b20dd57" + }, + { + "sha1": "5a65f1853d970e6b0396525c4af9f4158c9cecb7", + "uuid": "4af415d490a27e8d", + "target": "030056ef474d48cf465449ed4442fcf5518f2355849a1041c7092f4c4f439e5d402d5303bc455fad814a554e205052999c7583ad244bbd1857" + }, + { + "sha1": "6cad3fd60a089770fb10d84c2afed7147ca2bf9b", + "uuid": "07df6c303a5e863e", + "target": "0300565e474d48c0465449714442f394510eda55b9363241134e694c4f4343e6d0625307b545bb0b324a334e6d50525eace49aa1594b6db257" + }, + { + "sha1": "8a863b1da077eb3730ebdf94cd6cc7b3126b4bb7", + "uuid": "131c569def576bb8", + "target": "03005613474d488b4654497d4442a9ee51997055517b8941a4dd284c4f438cea6f30537373454e03bc4ae24e1d50522924a676c5cb4b060057" + }, + { + "sha1": "9eae2e3c27bc7e9f9af62f3259e4dbdaee9df884", + "uuid": "610dabf506f35fdd", + "target": "03005685474d489c465449c94442eeae518917550acc4f41204fe94c4f43c321338d53b78d45d334834a1e4e345052a03c7af077f44ba3bd57" + }, + { + "sha1": "d470b1fa4fae90163b9fc349d1edb80cf73aa3cf", + "uuid": "2fc27ce9aa52687f", + "target": "030056f1474d48a14654495c444265e251aae4554ac0ff4123a4ac4c4f437146a7d253444c45a5191c4a0b4edd50523589f41f75444b415c57" + }, + { + "sha1": "bc92e8760c26b3e5b46810476756ae5e4ad8252e", + "uuid": "c65b1d6417fab8fb", + "target": "0300560a474d4884465449e9444227a851338455640e4541016e2f4c4f4357542b0853995145f8c0364ae44e17505237bb4105171e4b41d857" + }, + { + "sha1": "948bf3706ffafddc2d112a24a54758380842b7f0", + "uuid": "ba3a07ea4457d763", + "target": "030056d4474d48a94654492a4442c5b4518d67552500a4411d339f4c4f43d1b2455153fb5d455c9f914ab84eec50523cbb0e1e05c44b392157" + }, + { + "sha1": "13a0b6747c91da5143e8027f0bbbd515a5b90f4c", + "uuid": "dff8767d83c127fb", + "target": "03005637474d48df4654495c44422cc751a6a555aafe2a413e2d2b4c4f43a0199eb853aaba456324d14a2f4e06505255d4bce609cd4bc82757" + }, + { + "sha1": "00de7f47b6a84d6050bb75d82211f47c30d92c94", + "uuid": "f0caaf5af72808fd", + "target": "0300569c474d48344654496c44427bb1513452553e57e9415bad144c4f436916b41153b8a84573be624a954e185052b7e754d02cf34b13ee57" + }, + { + "sha1": "f897c303a1e58ec03a381bfa9b92791be2784259", + "uuid": "9a7c8178338760b1", + "target": "03005625474d48fa4654498844425e9751131e55eeb23f41dc18704c4f438d8d18de53ed63451a2f0b4a494eef505237b62ae36eed4b3e1557" + }, + { + "sha1": "b66d5a9139f51487d532eb4bf342ed91f223827f", + "uuid": "0f751e294682eed1", + "target": "03005618474d48ec4654496e44420c1151535c55e33905415586f54c4f43300fc53353cfb9451c34844ac44ecf505224f70c644ed84bfb8e57" + }, + { + "sha1": "24888e44bf9e51979e4b68a2b4aa737700d3be2a", + "uuid": "5b971293fccf0049", + "target": "03005694474d48174654495944428b8951304155294f944165c5634c4f43e20cd4a353e6874592bff64a334ef65052a842d71087024b43b657" + }, + { + "sha1": "cd66ef830c5e3414b95592c2afb4392e5eb9858e", + "uuid": "0438acaa93d48f76", + "target": "0300561f474d487b465449c744424c57517ff1556c8a164188f9434c4f43855f304753a1b4457634074a5a4ed6505211a6245ce9c74b453a57" + }, + { + "sha1": "e036f865f0fd6fe7323c2d692329ab44a3d1a83a", + "uuid": "3764b6e1906b5870", + "target": "03005615474d4850465449644442ac28516d2555200845412674e04c4f43f48dad9e53f9c145ea6f144aed4ea7505272c2170164664bd72657" + }, + { + "sha1": "c4905d0be6e7790205c9c703bb2b5c18703d07fd", + "uuid": "c69fcce420cdf9f6", + "target": "03005629474d48b5465449f7444226855189ed554a746d4110ce6f4c4f43e69ff57c5317a3453377ed4a884e2b5052aeed5ca2604c4b909457" + }, + { + "sha1": "58feb56cc22c83009893285333c1e9782a72053c", + "uuid": "046388a75dba660d", + "target": "030056be474d48694654492a444274bc5180a455db667b4141bc794c4f43a1b910a653fa7845750cf94a524eed50526760d084630c4b512057" + }, + { + "sha1": "be2a42bb2f378eccd0d27347f66705fb8dff7f34", + "uuid": "081e23685457e38e", + "target": "0300564c474d48a4465449544442f27151a27555b92c774164a78a4c4f435695e6fd53c575456634f64af44e03505253d00e65de9b4bd13357" + }, + { + "sha1": "eccb7e3aa7669841bf5f73dca25889a8fcee132c", + "uuid": "24e96e7245e0e725", + "target": "0300569c474d48c7465449d24442876b516d1d550001514105e3474c4f43049d45705372ab45fe68b04ab84e0f50525911773093004b600a57" + }, + { + "sha1": "2fd95dded0c99a46d5f9425b93a704f6c604a5a7", + "uuid": "6453e4ace39544d2", + "target": "030056d5474d480046544968444220bf51c59855e39d50418b5e754c4f434aba83a553f61c457d188f4acf4ea4505231ed228c5e5a4bc3c957" + }, + { + "sha1": "c88412d743e9f2044e3f6df6fa55c13f93c7a7b5", + "uuid": "76b164dce3eda0d8", + "target": "030056cf474d487a4654493f4442de6751cc3155c90a0a41ed64804c4f43494629a653eeba45fc91bc4ac74e145052862a4aeae4ed4b986f57" + }, + { + "sha1": "cc9c19c3518a984f5f5b5562eb13d2c8e900fe98", + "uuid": "f46cea5d411da413", + "target": "030056fc474d48c5465449e34442487d5141cd5524e2c741fc8a134c4f4305ffbbfe53d1be45369b524ad64e9c505226cd7e73cad74b7f4a57" + }, + { + "sha1": "78eb2288cd528d039a405c4e4478053da9799848", + "uuid": "d9dc9883d4a9b7a7", + "target": "030056ac474d48a14654494844429f1551156e55fdbc7241ec52f04c4f43267d49c2532eaf4576e6924a824ef45052ac91b6f041ac4b0f3a57" + }, + { + "sha1": "f9646fccf42ab64b3b56190415e0b020a3a4b514", + "uuid": "a1d0165e5ad68dd8", + "target": "0300568a474d482046544982444245a9510d0055d8683941c09cdf4c4f43b7c3f4ce5347b845bece454a024e555052db5c9cb163c24bbba857" + }, + { + "sha1": "84e226f1a1efd91af3b2c96c4e05c7811bf9718f", + "uuid": "048ee93db835d127", + "target": "030056bc474d48ef4654492544422a0951aa1d55fc6451419d13884c4f4303f1d18e53602145e1072c4ace4ee150522706bc5a2d0c4bef8f57" + }, + { + "sha1": "1b9aaceba9afac29118fca84437c3fee2789821c", + "uuid": "f3dcc86fc28b4102", + "target": "030056eb474d488b465449da44427aba518d2755f3936f41d7327a4c4f43d03bde12535aca45c75b384add4e7b5052fe180a078a214bb32057" + }, + { + "sha1": "55422cc6d7fbdbe25da1271146bb56963dc46d9b", + "uuid": "d524bad6d23a87e1", + "target": "030056ff474d48b04654493044425334514999556deca841f8a7ef4c4f43803cf136530717459784934aa04eea50524b9bc13987e04b9bde57" + }, + { + "sha1": "2eaa654fa5abcfc7efe6636b666f2bd63501d6af", + "uuid": "8e5ce0a91916eb43", + "target": "03005638474d483a4654497d4442e45151284655521f2e413dd5a94c4f4344510c0e53a353457fc9274a8d4e4250523798e9ec8c9e4bb48e57" + }, + { + "sha1": "42319b8e2124866204527b511d322b33fe738cec", + "uuid": "f36f0956920514b5", + "target": "03005648474d48d2465449c8444207a4512c1255f44c7e412a8ed84c4f43220c433f5349d54580404f4ac04e28505276afc9f918b24b66e957" + }, + { + "sha1": "fb82256ff721be7aeed451818cf242a714af75b8", + "uuid": "99cc7e9bc673d6a0", + "target": "0300561f474d483a465449764442dc01519fbd55e0d6e041a155fa4c4f431b9f91a553052b459166244a8e4ed0505270e3d45c896c4be22857" + }, + { + "sha1": "ded094d2bc0e9479b95cacfb8abdf9a4103dcbeb", + "uuid": "656c123fcd22fce9", + "target": "03005692474d4884465449d14442e75c51ba3955e4ed2741d345084c4f4337d794fc53cb6e453e11f04a724e7950529215e225b3034ba40557" + }, + { + "sha1": "18d884ce349cd5a70df612b12e86d2c63c97cd61", + "uuid": "6668e1da7d174be7", + "target": "03005617474d48a64654491444422c3251be695598400341ddf6274c4f438da5e0ba5336dc45c9f8064a954e215052e511d014ad084b2c6c57" + }, + { + "sha1": "b326cb2bcb0e2d4900769db2cf3a8a62fc9ee97c", + "uuid": "16d71e2f462b980a", + "target": "03005685474d48314654492c4442a4f5516e8b55f9c9004110ffa84c4f4307be0d8b5388ca456bff034a054e1750520047ad8295114b8f5757" + }, + { + "sha1": "283781f1c3bc74bbb94789d92bf4dd381f2b7018", + "uuid": "73f7bafa27c89771", + "target": "030056bd474d48f5465449fc44420f0351f5c955a0b99941f71ff14c4f4307045c4753892b45cae1fd4ac44e3d50520b2887048b8c4bc84257" + }, + { + "sha1": "2d40e1a6b005128b6653ea08b27d05474e312d70", + "uuid": "de9fa46d1c9e80b7", + "target": "03005688474d4864465449ea4442d66351024955454cf34174630f4c4f431cd546fb53fcd2456b0d9b4a2b4e585052f5dbbb8fff994b50f057" + }, + { + "sha1": "54d242fcd1041b533bbf8ffa6dc973b44ac253fa", + "uuid": "72a63608b078848f", + "target": "03005667474d482b465449374442c47a510e8d55ec5873412eb1974c4f436ef651cd5388784597ca1c4a784e925052afdcffdd395c4b50ad57" + }, + { + "sha1": "457b7fc697ee76b0a64844fd0ae871f524afca75", + "uuid": "42f10dc6c056bc67", + "target": "030056a4474d48e14654494a4442aac051622955974688414385484c4f437ca63bd45315c6451f114a4a674e36505203cc1c98f1964b8b8957" + }, + { + "sha1": "7e972b178435645715af12d8052dfa8830da348f", + "uuid": "23bc3143207ce3b2", + "target": "0300566e474d485c465449b34442aa365177a25554a37541a507e14c4f433340c94553f154453df8854a7a4eca5052f1a1920567274b184a57" + }, + { + "sha1": "1e8df7f401765c7a9eb67f97346128ae8eb50143", + "uuid": "fa65b7cfbd6fb5a1", + "target": "0300568b474d48f94654491f44422a1451fe8f55dd1981419883754c4f439865d85b532e36458ed5ee4a5d4e555052dbeef35628284b1a2d57" + }, + { + "sha1": "0679c894ca400e913934ab0e4b0bc642771cef86", + "uuid": "010dd651884ee6a4", + "target": "030056bc474d4877465449f1444203ea512f0755643b6c41cfc6714c4f43cd52b6ba5378bc45645bdd4a4c4e29505252edf20006ac4b0ec557" + }, + { + "sha1": "0ab7bc32ea08e4e1311119802719b5f99c042377", + "uuid": "bbb5277d527275c0", + "target": "0300569f474d4825465449c1444282cc51f85d55e4b98f41b9450a4c4f43a944476b5324f5453293874a1c4e905052999e1c4dd13d4b6eb657" + }, + { + "sha1": "c6c44656352bf2f2d5aec96fca5c896e066285c7", + "uuid": "3d4d4e59fafd6972", + "target": "03005667474d48244654490d44422929517fd555e44694411bd85d4c4f43bead63fc533b9245d8ee674a574e0250521f2ef00448a74b1dfe57" + }, + { + "sha1": "2fcd4afd967ecca1749e292fc1b4290d76c754ba", + "uuid": "da0e759d28120ee3", + "target": "0300561a474d48804654497e4442bde651a8fb55989a4c419f9c384c4f43bef83a2f533c5c450bf2554a1e4e0750528b2db6eea4314b505a57" + }, + { + "sha1": "14d039c7aa66041f43dbdadad8254e55242c2c5e", + "uuid": "65cc848f0532ce2b", + "target": "03005663474d4827465449974442e52151d9e155acfdb4416a093e4c4f438b10d23a535667454796ed4af84e6c5052cb004b7765154bad6657" + }, + { + "sha1": "93d76cda5f0c4fc48391d2c5711543443bcfedc4", + "uuid": "092794f42fb48730", + "target": "0300563d474d4884465449214442ee98516b7d553d09be41070fc04c4f43ba67e0ad5330754542c47b4a584e5f5052e3faa45880b84ba6a357" + }, + { + "sha1": "9644f2e49cffffe157b2abd16ccb619fc3ba6d8e", + "uuid": "68a3e557d72d4292", + "target": "03005605474d4892465449324442dff8513fb15525e570416b470b4c4f43a130d3545317ea45f60cdd4a0c4ecb5052b1082100adcb4b84c157" + }, + { + "sha1": "9c62f06e84d27856e432e9f60d045f175cbb72cc", + "uuid": "41aac2d49c47be3a", + "target": "030056e3474d4895465449104442a21051721055b44a06414fd85d4c4f43867e5f9d5357bd455fb4234a9d4e4850520d2b09892c014b88b957" + }, + { + "sha1": "949e3130f452917db53b72b67e0347529dbc1496", + "uuid": "db8ecabee1747be6", + "target": "030056cd474d48da465449dd4442eb7a51240455632d36415393514c4f43cca7886c53a8b745bfe9c44abf4e5750526efca71f0a364b815b57" + }, + { + "sha1": "63d1bfaddd408a3c019e7cf79501d65986441df2", + "uuid": "4d0f9122f09e0cae", + "target": "03005673474d48814654498744422d6d510d35550f112c41fc2d194c4f43fa4947b053df75458fe5d04a414e4a50525a648c3ee0c34ba71a57" + }, + { + "sha1": "9ccc5c9990db4fa5c119396cc8193a147ade3665", + "uuid": "7b0c5d66bbd10ab0", + "target": "030056bb474d48c146544923444290b651903655ed0c5c4167823e4c4f431308e09053dba845b4f6ce4a524e8f505224df4247be2e4bc9a457" + }, + { + "sha1": "f34413feca88633f0b424959a866cbb0cfcee9f1", + "uuid": "a37e660c71bc59b3", + "target": "03005660474d4899465449fa4442ffb251276955dfe19441fd7aa04c4f4346402d7653ad3345bf64d54acc4e245052cf459a6e25114b59ea57" + }, + { + "sha1": "98592ea0600681b087a35da5ff6d5f5b238dd407", + "uuid": "21f8e732aa0434e3", + "target": "03005640474d48b54654497644420e01512c0255737a9e41031d484c4f43b0eb3c8353431a454c5b9e4a5f4e905052c1362c25982b4bd17d57" + }, + { + "sha1": "b019318959955b202d040cb89ba0609bdce0c3b2", + "uuid": "8042917a0bc0574b", + "target": "030056e1474d48b546544957444214645135f25521e7b041280ada4c4f43973e0f64531fb745684f1e4aa14e6a5052e8334eae15db4b544457" + }, + { + "sha1": "68bbd3901dbb43f46287e3f6d4661ecee05d215c", + "uuid": "a931729e3247d8f6", + "target": "0300564c474d483246544965444216915115da55f658d6411ad2c64c4f43a9af14d5538a634581ce094a3e4e8e5052ed3f707756294b8abc57" + }, + { + "sha1": "63135270ec9626abbc175a98e3e8b9d0f7838901", + "uuid": "664aa52d24758183", + "target": "030056a9474d48c0465449514442073051582c5545848b414ef4ac4c4f43d9379e4d53c406454d50df4aa34eb9505250151b26d5b84b1d4357" + }, + { + "sha1": "822a213b547c13537399c9cb9e4c8a2d8f240b31", + "uuid": "7cd49cfd020e718d", + "target": "03005601474d48564654497d44427461519287559660e1416ba10c4c4f43fdfcfdcc53730b45d7a9a74a044ec350520cf1d1b491ca4be97857" + }, + { + "sha1": "1b272fe1e357402c3e1d0696bb28a1d8fe76ef71", + "uuid": "5eca5deb007b2c38", + "target": "030056aa474d48f2465449ed444244bc5126d9552bca9a4107966e4c4f43a8d93621537b724523cde84adc4ed250526e04f5eb8c9b4b31ef57" + }, + { + "sha1": "381574a8470788be72af5ccbf3248ecf956e3da2", + "uuid": "d231df4caa3cec69", + "target": "030056fa474d4864465449cb4442319951f3f155badfae416d5e2f4c4f431c9657b9536a8e45dd85714a914e2a50526937a7fc8beb4bf3e457" + }, + { + "sha1": "a7e50e89d9b4733eecf8b5052e08c8ee3bd1b5e2", + "uuid": "a7045a0b15532771", + "target": "03005614474d48f0465449a24442d020516e3b55fb9daf41d1f5e34c4f43fcfc76dc533b99455525bb4a654ecb5052e23a0a87662c4b4dda57" + }, + { + "sha1": "ccd1d18ddcf756dc343c132fc763f0e5c503140d", + "uuid": "7fa83c07e8aaad27", + "target": "03005600474d48254654498644420e565180d95558fcd84137cb664c4f43f52c219253c6c145a70b004a8d4ed95052d500c10befe54b82cb57" + }, + { + "sha1": "19b38abfcfa4772186718a4470de9a7a50bca204", + "uuid": "403edb1a8590c148", + "target": "03005695474d48674654499344426bbb513315556f9e44418e3ae94c4f43413a781453936d45aff47b4ae44e1a5052c615e76dd8ed4bb1ce57" + }, + { + "sha1": "03a17ea57759256d52241b14a1a0989418cdcefb", + "uuid": "bddff14477b8fece", + "target": "030056a2474d48d34654496544426db251f9de55b790ab4125b7de4c4f433350889853a21d457c98234ad94e2750521b6c2f3111754ba44857" + }, + { + "sha1": "4d2b0681a3fe1d4f5495812cf9f9b9625dd5a46d", + "uuid": "d219ef9105f0b6d0", + "target": "030056c1474d48cc4654494a4442555b5117cc558040df41715dd44c4f434b6c6e1d53b78145035b104aec4e755052fdc5a2e8c4914bc6a857" + }, + { + "sha1": "3e272ad01968b7f13845aca0a1d069b70b1d6feb", + "uuid": "5a7057a9d9c60dab", + "target": "03005624474d4875465449b144428f7951fd1e55057ed241b798974c4f433b1c4abb53894c459b8b8b4a854e1d5052212304e269854b2a4f57" + }, + { + "sha1": "7ca08d54c34d55b5d776c3689b485abf060a1736", + "uuid": "615c78a88c0f8157", + "target": "03005644474d4838465449ee4442891f5195fb5595890b41fd75974c4f433c5ee03c53627c4510a1864ac34ef3505209aa00960e744b26be57" + }, + { + "sha1": "e77ffcd50e9ef189b9a323706c7597f4a5b6fc45", + "uuid": "af6e44a1eeb7ee6b", + "target": "030056ca474d489a4654495d44421aef51846655c9d0eb41efbc0b4c4f43b24fa0a153d43045bf46414a014e2550520b53ae5af3e64bc64857" + }, + { + "sha1": "2529688e002d6a85dfdf80f60fa8de36a53f150c", + "uuid": "79206d8b96ee0f3f", + "target": "0300560a474d48ab465449844442f18951b02655d4c470411a58034c4f431038e6b25350b24522746e4a344e4f5052153987ca9c994bb2ae57" + }, + { + "sha1": "0795449c3bbd91a19f0ffe71eb931269c7dd62c9", + "uuid": "a9ae502031e0a25f", + "target": "030056d1474d485b465449f04442cc1051f089557d139241ed28324c4f434ae3733453ed184590c2bc4ae94e515052826d98b864cd4bc0ce57" + }, + { + "sha1": "8c798c33edf097465b7b043724cc454e8c2c7c17", + "uuid": "e2f4554559bea605", + "target": "03005691474d48ae465449784442fce65191b455ead714414362594c4f43d10d85a953be6145657b814aa54e5e505269d4b7b4d7ea4ba91157" + }, + { + "sha1": "27f72b03bfc9b572a553c89e2b9cc37a8cdfbe07", + "uuid": "7a1672408d7af8e7", + "target": "03005661474d4806465449dc4442e24d514d4e559b6dfd418cb3914c4f43bdfd0b3553d4784528736b4aff4e745052f01d42b8d9a04bb76b57" + }, + { + "sha1": "9086fec0c53ea1444ae5368af17ee4cfc37d7f1a", + "uuid": "ce07aa09972f16dd", + "target": "03005670474d4871465449454442d48e51c8f0559742784199909e4c4f43a6a0cf1e5381994543c1914a464e8a5052a11fb21933f24b9bb757" + }, + { + "sha1": "c351c31469af433ed693c972463432c782485633", + "uuid": "1415a69fcef1c185", + "target": "03005681474d48c64654493b4442216f51a2865598a7ca41ba700b4c4f430db99c6f53a31b4579d98e4ac54eaa5052057b11f8501a4bd59357" + }, + { + "sha1": "e95f34039af60004fa811419159a4cd742293e75", + "uuid": "c82499cf03c54cd3", + "target": "0300560d474d482e4654491c44429668511e8755792c1e41a06e604c4f432c16f6065383a045d9f7194a244ee05052982c5dd946414b8a5657" + }, + { + "sha1": "7693bf352787e8332564086f6184393b2856df39", + "uuid": "d3b41292df7f052a", + "target": "030056af474d48504654493d44423a8f51e8fc554418e141c7bc344c4f430e7d334053f9974588a4384a4c4e6c505230801c3bd4cc4b2f4457" + }, + { + "sha1": "0e10e2427b89d5a77293c4eeae6d1970b00e20b1", + "uuid": "db8dbe5fec8932e6", + "target": "03005663474d48ce4654492b4442190e51b43655e79ed741cb53584c4f43036102ea53ee9d45de8d774a404e4c505223881c973dc44ba9e357" + }, + { + "sha1": "6937548b10ed1b83cc82842796f8aa5a59f32a30", + "uuid": "9d1ffa0cbb77466c", + "target": "03005622474d48b6465449de444295c05141db55dc402641e3e41c4c4f43613f8c42538425455c9c804a8f4e5e5052d78c14d7afae4b79d757" + }, + { + "sha1": "39f18c90b783a904432373a5abd6f9eeabd7b7c0", + "uuid": "59dff372671d9e78", + "target": "030056ef474d4842465449804442050051a38a5502d5f641cc2c9e4c4f43c4a92ab5537db845bd15af4ad94e4d5052b6cee1fdbdca4bc22657" + }, + { + "sha1": "d2f6b470dd2d20860e8731efbbe35d25b5c6225d", + "uuid": "1aa493a1a0934e66", + "target": "030056a2474d48014654494c4442a06951d5e4556bb5e541e0693f4c4f431c467ab453325c45cb46344a544efa50527b1be15f95024bfe6757" + }, + { + "sha1": "47a43da716c867a1604ab4b9009995528ac632d8", + "uuid": "889d638b2b8f2670", + "target": "030056da474d4863465449034442cb24512f67553f43f8414668e44c4f43b00dc168538a55452d54ef4ab44ee8505231122f9e3bb84b76cd57" + }, + { + "sha1": "76a25c3eebb44cd923af8782a9effe85cb4d8729", + "uuid": "d8c4bc694318fed0", + "target": "03005653474d48e4465449774442621a51fc82551c156e41fdabd24c4f438b7c7c50531959452471904a8e4ef1505224f8de2690c54b81f357" + }, + { + "sha1": "75e6fe96853225d5af0e247e996d2922518e4d5a", + "uuid": "888bdeaf85fe5819", + "target": "030056c4474d48d2465449094442f1df51f3eb554b4ad8416a9c144c4f434308a21a5344d045c2840b4a844edd505206eb3be938484b0c5057" + }, + { + "sha1": "f39dd1c3463037b85a84acfcb712a957015cbd82", + "uuid": "259bf557538b6083", + "target": "030056ed474d489e4654490544420a0151983a55a1f38441c487814c4f435ddc1b74535fbb45c6274d4a3b4ee05052236f32be57f94b41e157" + }, + { + "sha1": "fd93ee5c23930c8e3ac7b2899ff084ed1625570a", + "uuid": "2c905c9463a65c2c", + "target": "03005641474d48ad465449fa4442ec9a515c2055114a4641f16e244c4f43b2a2ff99532124452e49a64ada4e1c5052e285ab0d6f294b84d057" + }, + { + "sha1": "a439f6ab243b3124e8f9526b4c1fc24a10ba0511", + "uuid": "4677e63a6cb92220", + "target": "030056d8474d48e9465449d24442b028516de3555edcf1414fc50d4c4f430496227c5395244580820c4aa94ef85052a2118be157284be03757" + }, + { + "sha1": "54d1ee718f6f2c8da8179acad1da8fa8a61d8ee5", + "uuid": "dd78dcdf7cedd18c", + "target": "03005680474d4898465449d04442e52f519f5f55762a6441e93d9e4c4f4303e0edbe5378e545bcf2ff4a014e5b50528cea626f938b4b7e9d57" + }, + { + "sha1": "987589f7a7e2511d86651d9777608ed6cb615e44", + "uuid": "bd1f3238d729c5f0", + "target": "03005619474d488d4654494c4442c0d05107055509271541d7d3ee4c4f43d2fcabe25352874540d81a4aaf4e74505209703cb79dad4b0bc757" + } +] From 3031e58fe5babfe8453a8a4489156178a72047c2 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 20:00:36 +0100 Subject: [PATCH 05/21] feat(libgpod): sign iPod nano 6G/7G databases with the vendored HashAB signer libgpod 0.8.3 loads calcHashAB from an external hashab module in LIBGPOD_BLOB_DIR and fails nano 6G/7G writes without it. A small patch replaces that loader with a direct call into the vendored sources, which prepare-libgpod-nano.sh copies into the libgpod tree before autoreconf. They are built as an internal convenience library with -fwrapv, and their global symbols are renamed and hidden, so that they neither clash with other code linked with libgpod nor become part of its interface. The macOS developer build and the static prebuild dependencies both apply the patch. The signer only accepts an eight-byte FireWire GUID, so the patch checks for 16 hex digits first and fails the write otherwise. The static dependency cache now records a fingerprint of the patch and the vendored sources, and the CI cache keys hash all of tools/prebuild, so either change rebuilds libgpod. macOS static links name the system sqlite3 and libxml2 that libgpod already uses, and Linux static links group the archives, so standalone executables link too. tools/nano7 holds the checks: a read-only probe that verifies the signatures of a copied Locations.itdb/.cbk pair and iTunesCDB, including the hashing scheme libgpod writes, tests against the upstream vectors and a reference checksum book, and a libgpod roundtrip for nano 6G/7G that the prebuild workflow runs. ADR-030 records the decision. Both generations stay read-only. --- .changeset/nano-native-hashab.md | 6 + .github/workflows/build-platform.yml | 8 +- .github/workflows/prebuild.yml | 16 ++- docs/adr/adr-030-native-nano-hashab.md | 70 +++++++++++ docs/adr/index.md | 1 + tools/libgpod-macos/build.sh | 2 + tools/nano7/.gitignore | 1 + tools/nano7/README.md | 63 ++++++++++ tools/nano7/build.py | 25 ++++ tools/nano7/fixtures/LICENSE | 21 ++++ tools/nano7/fixtures/Locations.itdb | Bin 0 -> 20480 bytes tools/nano7/fixtures/Locations.itdb.cbk | Bin 0 -> 477 bytes tools/nano7/fixtures/UPSTREAM.md | 7 ++ tools/nano7/libgpod-roundtrip.c | 65 ++++++++++ tools/nano7/probe.py | 156 ++++++++++++++++++++++++ tools/nano7/test-libgpod.sh | 26 ++++ tools/nano7/test_libgpod.py | 107 ++++++++++++++++ tools/nano7/test_probe.py | 148 ++++++++++++++++++++++ tools/prebuild/build-linux-glibc.sh | 2 +- tools/prebuild/build-linux-musl.sh | 2 +- tools/prebuild/build-static-deps.sh | 8 +- tools/prebuild/get-ldflags.sh | 7 +- tools/prebuild/libgpod-nano.patch | 126 +++++++++++++++++++ tools/prebuild/prepare-libgpod-nano.sh | 27 ++++ 24 files changed, 882 insertions(+), 12 deletions(-) create mode 100644 .changeset/nano-native-hashab.md create mode 100644 docs/adr/adr-030-native-nano-hashab.md create mode 100644 tools/nano7/.gitignore create mode 100644 tools/nano7/README.md create mode 100644 tools/nano7/build.py create mode 100644 tools/nano7/fixtures/LICENSE create mode 100644 tools/nano7/fixtures/Locations.itdb create mode 100644 tools/nano7/fixtures/Locations.itdb.cbk create mode 100644 tools/nano7/fixtures/UPSTREAM.md create mode 100644 tools/nano7/libgpod-roundtrip.c create mode 100644 tools/nano7/probe.py create mode 100755 tools/nano7/test-libgpod.sh create mode 100644 tools/nano7/test_libgpod.py create mode 100644 tools/nano7/test_probe.py create mode 100644 tools/prebuild/libgpod-nano.patch create mode 100755 tools/prebuild/prepare-libgpod-nano.sh diff --git a/.changeset/nano-native-hashab.md b/.changeset/nano-native-hashab.md new file mode 100644 index 00000000..16ee099e --- /dev/null +++ b/.changeset/nano-native-hashab.md @@ -0,0 +1,6 @@ +--- +'@podkit/libgpod-node': patch +'podkit': patch +--- + +podkit's libgpod build now signs iPod nano 6th and 7th generation databases itself, instead of needing an external `hashab` module. Both generations remain read-only: the nano 7th generation shows no music from a library written this way yet, although its signatures are valid. diff --git a/.github/workflows/build-platform.yml b/.github/workflows/build-platform.yml index 1f939f63..479ba2af 100644 --- a/.github/workflows/build-platform.yml +++ b/.github/workflows/build-platform.yml @@ -89,7 +89,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.STATIC_DEPS_DIR }} - key: static-deps-${{ matrix.platform }}-${{ matrix.arch }}-${{ hashFiles('tools/prebuild/build-static-deps.sh') }} + key: static-deps-${{ matrix.platform }}-${{ matrix.arch }}-${{ hashFiles('tools/prebuild/**') }} # Install build toolchain (only when building prebuilds from scratch) @@ -250,7 +250,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.STATIC_DEPS_DIR }} - key: static-deps-linux-x64-${{ hashFiles('tools/prebuild/build-static-deps.sh') }} + key: static-deps-linux-x64-${{ hashFiles('tools/prebuild/**') }} - name: Install Alpine build deps (full) if: steps.prebuild-cache.outputs.cache-hit != 'true' && steps.static-cache.outputs.cache-hit != 'true' @@ -448,7 +448,7 @@ jobs: uses: actions/cache@v4 with: path: static-deps - key: static-deps-linux-arm64-${{ hashFiles('tools/prebuild/build-static-deps.sh') }} + key: static-deps-linux-arm64-${{ hashFiles('tools/prebuild/**') }} - name: Build prebuild in Alpine if: steps.prebuild-cache.outputs.cache-hit != 'true' @@ -685,7 +685,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.STATIC_DEPS_DIR }} - key: static-deps-linux-${{ matrix.arch }}-gnu-${{ hashFiles('tools/prebuild/build-static-deps.sh', 'tools/prebuild/build-linux-glibc.sh') }} + key: static-deps-linux-${{ matrix.arch }}-gnu-${{ hashFiles('tools/prebuild/**') }} # Full build toolchain. ubuntu:20.04 ships meson/ninja/cmake too old for # recent glib/gdk-pixbuf/libxml2, so those come from pip, not apt. diff --git a/.github/workflows/prebuild.yml b/.github/workflows/prebuild.yml index 9daf3281..0a364b67 100644 --- a/.github/workflows/prebuild.yml +++ b/.github/workflows/prebuild.yml @@ -43,7 +43,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.STATIC_DEPS_DIR }} - key: static-deps-${{ matrix.platform }}-${{ matrix.arch }}-${{ hashFiles('tools/prebuild/build-static-deps.sh', 'tools/prebuild/build-linux-glibc.sh') }} + key: static-deps-${{ matrix.platform }}-${{ matrix.arch }}-${{ hashFiles('tools/prebuild/**') }} # ---- Cache MISS: install full build toolchain and build from source ---- @@ -129,6 +129,13 @@ jobs: # Linux glibc verification happens inside tools/prebuild/build-linux-glibc.sh # (the shared script). No additional step needed here. + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Test libgpod nano HashAB signing + run: bash tools/nano7/test-libgpod.sh + - name: Run tests working-directory: packages/libgpod-node run: bun run test:unit @@ -164,7 +171,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.STATIC_DEPS_DIR }} - key: static-deps-linux-x64-musl-${{ hashFiles('tools/prebuild/build-static-deps.sh') }} + key: static-deps-linux-x64-musl-${{ hashFiles('tools/prebuild/**') }} - name: Install Alpine build deps (full) if: steps.cache.outputs.cache-hit != 'true' @@ -217,6 +224,9 @@ jobs: echo "ERROR: Found unexpected dynamic dependencies"; exit 1 fi + - name: Test libgpod nano HashAB signing + run: bash tools/nano7/test-libgpod.sh + - name: Run tests working-directory: packages/libgpod-node run: bun run test:unit @@ -259,6 +269,8 @@ jobs: curl -fsSL https://bun.sh/install | bash bun install + bash tools/nano7/test-libgpod.sh + cd packages/libgpod-node npx prebuildify --napi --strip mv prebuilds/linux-arm64 prebuilds/linux-arm64-musl diff --git a/docs/adr/adr-030-native-nano-hashab.md b/docs/adr/adr-030-native-nano-hashab.md new file mode 100644 index 00000000..b45ef9a4 --- /dev/null +++ b/docs/adr/adr-030-native-nano-hashab.md @@ -0,0 +1,70 @@ +--- +title: 'ADR-030: Link an Open HashAB Signer Into libgpod' +description: Build libgpod with the Unlicense dstaley/hashab C sources instead of loading an external hashab blob, so the iPod nano 6th/7th generation databases can be signed. Signing alone does not make those generations syncable; they stay read-only. +sidebar: + order: 31 +--- + +# ADR-030: Link an Open HashAB Signer Into libgpod + +## Status + +**Proposed** (2026-10-27) + +## Context + +The iPod nano 6th and 7th generation (SysInfoExtended `DBVersion` 5) sign +`iTunesCDB` and `iTunes Library.itlp/Locations.itdb.cbk` with HashAB. +libgpod 0.8.3 already writes both files, including the SQLite databases and +the artwork, but `itdb_hashAB.c` computes the signature only through a +`hashab` module loaded from `LIBGPOD_BLOB_DIR`, and fails the write without +it. podkit ships no such module. + +[dstaley/hashab](https://github.com/dstaley/hashab) is an Unlicense C +implementation of the same `calcHashAB(target, sha1, uuid, rnd)` function, +with 100 reference vectors. It takes the eight-byte FireWire GUID of a nano, +not an iOS device identifier. + +## Decision + +- Vendor the signer unmodified in `tools/prebuild/vendor/hashab` at a pinned + commit, with its licence and provenance. +- `tools/prebuild/prepare-libgpod-nano.sh` applies + `tools/prebuild/libgpod-nano.patch` and copies the sources into + `src/nano_hashab/` before `autoreconf`, in both the macOS developer build + and the static prebuild dependencies. The patch replaces the module loader + with a direct call and builds the sources as an internal convenience + library, with `-fwrapv` as upstream does, and with their global symbols + renamed and hidden so that they neither clash with nor leak from libgpod. +- Before signing, the patch requires a 16-digit hexadecimal `FireWireGUID` + (an optional `0x` prefix is accepted) and otherwise fails the write with an + error. libgpod's CDB normalisation, SQLite generation and artwork writing + are unchanged. +- The static dependency cache records a fingerprint of the patch and the + vendored sources, and the CI cache keys hash all of `tools/prebuild/`, so a + change to either rebuilds libgpod. +- No runtime dependency, subprocess or N-API surface is added. Python is used + only by the developer checks in `tools/nano7/`. + +## Verification + +- The 100 upstream vectors and a third-party reference + `Locations.itdb`/`.cbk` pair reproduce byte for byte. +- `tools/nano7/test-libgpod.sh`, run in the prebuild workflow, lets the + patched libgpod create, reparse, resave and delete a synthetic track for a + nano 7th generation (from the repository's USB SysInfoExtended capture) and + a nano 6th generation (synthetic capabilities), and checks SQLite integrity + and both signatures. Malformed and iOS-sized GUIDs fail; a 5th generation + iPod still gets an unsigned `iTunesDB`. +- On a nano 7th generation (firmware 1.0.4), Apple-written `iTunesCDB` and + `Locations.itdb.cbk` verify with libgpod's normalisation (hashing scheme 3) + and the device GUID. + +## Consequences + +Signing is necessary but not sufficient. On a nano 7th generation (firmware +1.0.4), a library written by this libgpod has valid signatures, yet the +firmware shows no music. `nano_6g` and `nano_7g` therefore keep +`support.access = 'read-only'`; the nano 6th generation's writes are covered +only by synthetic tests. Enabling sync for either generation needs a complete +library write that is verified on the device. diff --git a/docs/adr/index.md b/docs/adr/index.md index 8db41176..a7f6cbcd 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -36,6 +36,7 @@ This section contains Architecture Decision Records (ADRs) documenting significa | [ADR-026](./adr-026-dual-libc-linux-distribution.md) | Dual-libc Linux Distribution (glibc + musl) | Accepted | 2026-08-04 | | [ADR-027](./adr-027-lima-vm-substrate-consolidation.md) | Lima VM Substrate Consolidation (`@podkit/lima`) | Accepted | 2026-08-24 | | [ADR-028](./adr-028-substrate-agnostic-device-harness.md) | Substrate-Agnostic Device Harness | Accepted | 2026-09-07 | +| [ADR-030](./adr-030-native-nano-hashab.md) | Link an Open HashAB Signer Into libgpod | Proposed | 2026-10-27 | ## What is an ADR? diff --git a/tools/libgpod-macos/build.sh b/tools/libgpod-macos/build.sh index 72890261..2e86b690 100755 --- a/tools/libgpod-macos/build.sh +++ b/tools/libgpod-macos/build.sh @@ -136,6 +136,8 @@ extract_and_patch() { log_info " Adding --without-libusb opt-out (podkit reads SysInfoExtended via @podkit/ipod-firmware)..." "$SCRIPT_DIR/../prebuild/disable-libgpod-libusb.sh" configure.ac + bash "$SCRIPT_DIR/../prebuild/prepare-libgpod-nano.sh" "$BUILD_DIR/$LIBGPOD_DIR" + log_info "Patching complete" } diff --git a/tools/nano7/.gitignore b/tools/nano7/.gitignore new file mode 100644 index 00000000..84c048a7 --- /dev/null +++ b/tools/nano7/.gitignore @@ -0,0 +1 @@ +/build/ diff --git a/tools/nano7/README.md b/tools/nano7/README.md new file mode 100644 index 00000000..f52b3316 --- /dev/null +++ b/tools/nano7/README.md @@ -0,0 +1,63 @@ +# Nano HashAB checks + +The iPod nano 6th and 7th generation sign `iTunesCDB` and +`iTunes Library.itlp/Locations.itdb.cbk` with HashAB. podkit's libgpod builds +link the vendored signer from `tools/prebuild/vendor/hashab` through +`tools/prebuild/libgpod-nano.patch` (see +[ADR-030](../../docs/adr/adr-030-native-nano-hashab.md)). Both generations stay +**read-only** in podkit: the nano 7th generation firmware shows no music for a +library libgpod writes, although its signatures verify. + +These tools need Python 3.11+ and a C compiler, and no pip or npm packages. + +## Signature checks + +```sh +python3 tools/nano7/build.py +python3 tools/nano7/test_probe.py -v +python3 tools/nano7/probe.py tools/nano7/fixtures/Locations.itdb --guid 000a270024bd17a1 +``` + +`build.py` compiles the vendored signer into `tools/nano7/build/` for the +Python checks only. The tests cover the 100 upstream reference vectors, a +byte-exact reproduction of the reference checksum book in `fixtures/`, nonce +handling, `iTunesCDB` normalisation and rejection of corrupted data or a wrong +GUID. + +## libgpod roundtrip + +Build the patched libgpod as usual, then run the roundtrip: + +```sh +bash tools/libgpod-macos/build.sh all # macOS; installs to $PREFIX (default ~/.local) +bash tools/nano7/test-libgpod.sh +``` + +`test-libgpod.sh` compiles `libgpod-roundtrip.c` with the same flag helpers as +`@podkit/libgpod-node` (`STATIC_DEPS_DIR` selects a prebuild dependency tree) +and lets libgpod create, reparse, resave and delete a synthetic track on +temporary devices: a nano 7th generation from +`docs/sysinfo-captures/nano-7g-16gb-usb.xml` and a nano 6th generation with +synthetic capabilities. It checks the integrity of the five SQLite databases, +both signatures, rejection of malformed or iOS-sized GUIDs, and an unsigned +`iTunesDB` for a 5th generation iPod. + +## Checking a copied device library + +A nano 7th generation synced by macOS carries no `SysInfoExtended` file, so +read it over USB (`inquireFirmwareDetailed` in `@podkit/ipod-firmware`) for its +`FireWireGUID`. Copy `iPod_Control/iTunes/iTunes Library.itlp/` and +`iPod_Control/iTunes/iTunesCDB` from the device and run the probe on the copy: + +```sh +python3 tools/nano7/probe.py /path/to/copy/Locations.itdb --guid <16 hex digits> \ + --cdb /path/to/copy/iTunesCDB +``` + +The probe reads SQLite only from an in-memory copy, prints the schema, +integrity and signature results as JSON and exits with 0 or 1. It never +writes. + +On a nano 7th generation (firmware 1.0.4), Apple's `iTunesCDB` and +`Locations.itdb.cbk` verify with libgpod's CDB normalisation (hashing scheme 3) +and the device GUID. diff --git a/tools/nano7/build.py b/tools/nano7/build.py new file mode 100644 index 00000000..28c7e156 --- /dev/null +++ b/tools/nano7/build.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Build the vendored HashAB signer as a shared library for the checks; installs nothing.""" +import os +from pathlib import Path +import platform +import subprocess + +ROOT = Path(__file__).resolve().parent + + +def build(output=None): + suffix = ".dylib" if platform.system() == "Darwin" else ".so" + output = Path(output) if output else ROOT / "build" / ("libhashab" + suffix) + output.parent.mkdir(parents=True, exist_ok=True) + sources = sorted((ROOT.parent / "prebuild/vendor/hashab/src").glob("*.c")) + link = "-dynamiclib" if platform.system() == "Darwin" else "-shared" + subprocess.run([ + os.environ.get("CC", "cc"), "-std=c11", "-fwrapv", "-O2", "-fPIC", + link, *map(str, sources), "-o", str(output), + ], check=True) + return output.resolve() + + +if __name__ == "__main__": + print(build()) diff --git a/tools/nano7/fixtures/LICENSE b/tools/nano7/fixtures/LICENSE new file mode 100644 index 00000000..942047bb --- /dev/null +++ b/tools/nano7/fixtures/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Alex Buldiei + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/nano7/fixtures/Locations.itdb b/tools/nano7/fixtures/Locations.itdb new file mode 100644 index 0000000000000000000000000000000000000000..905abe5d53879fe04c628df6b120cc8e56040348 GIT binary patch literal 20480 zcmeI)F>ljA7zW@w*G(HlYL+T2LB~*~k*u~!tCp$@A-Je&T1s$5O9#uTT`)Lth%YEG zfhtwv7cep~Abtl3CM496fe8jSBo;<4Zm+RX%f?W6FR^{+yYqMV*ct37xp|}NahL4` z?S{?SG&x5oC6^f^geYPe6pKs)BBRQJ*wR@3q^v?Ftn(S|Cs8xE$w*FnpLr!RNDzPk z1Rwwb2tWV=5P$##{waZFHI*G3qg&jr`)>Ha7q?kf$ZcMX=Lah_qih%Q(eUA@<*zz2r1`8<7^kB(NiL)Y?yZJT>RGf_|z$4gYPanE5(tEO?ysIk@SCR<&v zR(000`CVoj8>Y_IYD+8S+9q2zHhYZ=h21v2>=*9seY&+V`f+ioYAlxKC&iP1 zJFUPemF9}2*@;GLdSz{zO{KnM$n)1P7ihBcyR8==-hO|!A9bFYm7V7<%@^lq`kg;- zk|*`uGh$YX_LXQq#0m)l5P$##AOHafKmY;|fB*y_0D;phkWo{^R6bKs&Wci6J|yUU z|4+4VM7)t8009U<00Izz00bZa0SG_<0uVUu0;6<*6uq^;u_{56w*!B2r4xGF2lxN` vMB6{@6U6u-009U<00Izz00bZa0SG_<0{@eMPSso`D}U(!hyMa7GV%HazXdE? literal 0 HcmV?d00001 diff --git a/tools/nano7/fixtures/Locations.itdb.cbk b/tools/nano7/fixtures/Locations.itdb.cbk new file mode 100644 index 0000000000000000000000000000000000000000..6823abe1f828c9ee528579c7099fbd0218a2207a GIT binary patch literal 477 zcmZQ(2%GQj>v7U8#8cnJ>DThW{6(Qt=S+5N*N*Y=cdif&{1QB4mg`DkIWH!^(*Z%5 zO%Ls}YQ4=f!`aTsA4wGG%$>t|y?DYV9{!1@#qT3k4xL_PQM~us`T2h(mrEZr&6s1I zaO(8u>w0n(CJMI8IGC%Ye^@)bz$O-$?7YstOyydD`T5Vm`}gi|`sT1RvS;q;wVvni zA8otor?62m@kYAMZ)|$8iTOW0RQUJFiXW->v^T7n6QQEYtUhzXX4%zqvZq)E)$B-S z5vmsxnf~-GHoe%yu6u2{b7Nbo>nSHuv0pD#O?p;dx;Y_0<@S_D(T9!=J9i}%O`Z^F zSgnaoFE%lc{_QFC-G+%jD$>t=yqtKyE;YR=U-v?7XwaMZ^A~q}*2d^Q;Syb7k4-N& aF(3KDT~4hNxnn#U7c&Qa@%H;L`7{8Cj@x+v literal 0 HcmV?d00001 diff --git a/tools/nano7/fixtures/UPSTREAM.md b/tools/nano7/fixtures/UPSTREAM.md new file mode 100644 index 00000000..ff6dd8c0 --- /dev/null +++ b/tools/nano7/fixtures/UPSTREAM.md @@ -0,0 +1,7 @@ +# Reference database provenance + +Source: https://github.com/buldiei/ipodsync/tree/28174c7f2f8ded61d19782fd2b2ee4922414db71/tests/data +License: MIT (see LICENSE) + +`Locations.itdb` and `Locations.itdb.cbk` are copied unmodified. The checksum +book was signed with FireWire GUID `000a270024bd17a1`. diff --git a/tools/nano7/libgpod-roundtrip.c b/tools/nano7/libgpod-roundtrip.c new file mode 100644 index 00000000..d0d4808f --- /dev/null +++ b/tools/nano7/libgpod-roundtrip.c @@ -0,0 +1,65 @@ +/* Create, resave or delete one synthetic track with libgpod. + * Usage: libgpod-roundtrip create|resave|delete */ +#include +#include + +int main(int argc, char **argv) +{ + Itdb_iTunesDB *db; + Itdb_Playlist *master; + Itdb_Track *track; + GError *error = NULL; + gboolean saved; + + if (argc != 3) + return 2; + if (strcmp(argv[2], "create") == 0) { + db = itdb_new(); + itdb_set_mountpoint(db, argv[1]); + master = itdb_playlist_new("Nano test", FALSE); + itdb_playlist_set_mpl(master); + itdb_playlist_add(db, master, -1); + track = itdb_track_new(); + track->title = g_strdup("Synthetic track"); + track->artist = g_strdup("Test artist"); + track->album = g_strdup("Test album"); + track->filetype = g_strdup("MPEG audio file"); + track->ipod_path = g_strdup(":iPod_Control:Music:F00:TEST.mp3"); + track->mediatype = ITDB_MEDIATYPE_AUDIO; + track->tracklen = 2000; + track->samplerate = 44100; + track->bitrate = 128; + track->size = 32000; + track->transferred = TRUE; + itdb_track_add(db, track, -1); + itdb_playlist_add_track(master, track, -1); + master = itdb_playlist_new("Test playlist", FALSE); + itdb_playlist_add(db, master, -1); + itdb_playlist_add_track(master, track, -1); + } else { + db = itdb_parse(argv[1], &error); + if (!db) { + g_printerr("parse: %s\n", error ? error->message : "unknown error"); + return 4; + } + if (g_list_length(db->tracks) != 1) + return 5; + track = db->tracks->data; + if (strcmp(track->title, "Synthetic track") != 0) + return 6; + if (strcmp(argv[2], "delete") == 0) { + GList *item; + for (item = db->playlists; item; item = item->next) + itdb_playlist_remove_track(item->data, track); + itdb_track_remove(track); + } else if (strcmp(argv[2], "resave") != 0) { + return 2; + } + } + saved = itdb_write(db, &error); + if (!saved) + g_printerr("write: %s\n", error ? error->message : "unknown error"); + g_clear_error(&error); + itdb_free(db); + return saved ? 0 : 1; +} diff --git a/tools/nano7/probe.py b/tools/nano7/probe.py new file mode 100644 index 00000000..779372af --- /dev/null +++ b/tools/nano7/probe.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Verify the HashAB signatures of a copied Locations.itdb/.cbk pair and iTunesCDB; read-only, prints JSON.""" +import argparse +from contextlib import closing +import ctypes +import hashlib +import hmac +import json +from pathlib import Path +import platform +import sqlite3 +import struct +import sys + +ROOT = Path(__file__).resolve().parent +FIXED_RANDOM = b"ABCDEFGHIJKLMNOPQRSTUVW" +# HashAB exposes its 23 nonce bytes through this output permutation. +PERMUTATION = ( + 21, 28, 6, 12, 7, 26, 5, 19, 8, 25, 3, 1, 45, 30, 16, 49, + 29, 20, 40, 39, 53, 0, 47, 27, 38, 11, 14, 2, 35, 23, 36, 34, + 18, 31, 32, 4, 41, 37, 33, 9, 24, 13, 50, 15, 17, 46, 51, 43, + 48, 42, 54, 10, 44, 52, 22, +) + + +def guid_bytes(value): + value = value.removeprefix("0x").removeprefix("0X") + if len(value) != 16: + raise ValueError("Nano GUID must contain exactly 16 hexadecimal characters") + try: + return bytes.fromhex(value) + except ValueError as exc: + raise ValueError("Invalid hexadecimal Nano GUID") from exc + + +class HashAB: + def __init__(self, path): + self.lib = ctypes.CDLL(str(Path(path).resolve())) + self.calculate = self.lib.calcHashAB + self.calculate.argtypes = [ctypes.POINTER(ctypes.c_ubyte)] * 4 + self.calculate.restype = None + + def sign(self, sha1, guid, nonce=FIXED_RANDOM): + if (len(sha1), len(guid), len(nonce)) != (20, 8, 23): + raise ValueError("Expected SHA1=20, Nano GUID=8, nonce=23 bytes") + array = ctypes.c_ubyte + target = (array * 57)() + self.calculate(target, (array * 20).from_buffer_copy(sha1), + (array * 8).from_buffer_copy(guid), + (array * 23).from_buffer_copy(nonce)) + result = bytes(target) + if result[:2] != b"\x03\x00": + raise ValueError("HashAB calculation failed") + return result + + +def signature_nonce(signature): + if len(signature) != 57 or signature[:2] != b"\x03\x00": + raise ValueError("Unsupported HashAB signature header") + nonce = bytearray(23) + for index, source in enumerate(PERMUTATION): + if source < 23: + nonce[source] = signature[index + 2] + return bytes(nonce) + + +def block_hashes(data): + if not data or len(data) % 1024: + raise ValueError("Expected non-empty SQLite data aligned to 1024 bytes") + return b"".join(hashlib.sha1(data[i:i + 1024]).digest() + for i in range(0, len(data), 1024)) + + +def make_cbk(data, guid, hasher, nonce=FIXED_RANDOM): + blocks = block_hashes(data) + master = hashlib.sha1(blocks).digest() + return hasher.sign(master, guid, nonce) + master + blocks + + +def verify_cbk(data, cbk, guid, hasher): + blocks = block_hashes(data) + if len(cbk) != 77 + len(blocks): + raise ValueError("CBK length does not match Locations.itdb") + nonce = signature_nonce(cbk[:57]) + master = hashlib.sha1(blocks).digest() + checks = { + "blockHashes": hmac.compare_digest(cbk[77:], blocks), + "masterHash": hmac.compare_digest(cbk[57:77], master), + "hashAB": hmac.compare_digest(cbk[:57], hasher.sign(master, guid, nonce)), + } + return {"success": all(checks.values()), "checks": checks, "blocks": len(blocks) // 20} + + +def cdb_sha1(cdb): + """SHA1 of an iTunesCDB as libgpod 0.8.3 computes it before signing.""" + if len(cdb) < 0xF4 or cdb[:4] != b"mhbd": + raise ValueError("Expected an iTunesCDB with an mhbd header") + data = bytearray(cdb) + struct.pack_into("FireWireGUID\s*)[^<]+", + lambda m: m[1] + guid, capture) + (self.control / "Device/SysInfoExtended").write_text(capture) + (self.control / "Device/SysInfo").write_text("ModelNumStr: MD480\n") + + def run_helper(self, operation, success=True): + result = subprocess.run([str(self.helper), str(self.mount), operation], + capture_output=True, text=True) + if success: + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + else: + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + return result + + def check_library(self, count): + bundle = self.control / "iTunes/iTunes Library.itlp" + for name in ("Library", "Locations", "Dynamic", "Extras", "Genius"): + with sqlite3.connect(f"file:{bundle / (name + '.itdb')}?mode=ro", uri=True) as db: + self.assertEqual(db.execute("PRAGMA integrity_check").fetchall(), [("ok",)]) + guid = guid_bytes("0123456789ABCDEF") + result = verify_cbk((bundle / "Locations.itdb").read_bytes(), + (bundle / "Locations.itdb.cbk").read_bytes(), guid, self.hasher) + self.assertTrue(result["success"], result) + cdb = verify_cdb((self.control / "iTunes/iTunesCDB").read_bytes(), guid, self.hasher) + self.assertEqual((cdb["cdbScheme"], cdb["cdbHashAB"]), (3, True), cdb) + with sqlite3.connect(f"file:{bundle / 'Library.itdb'}?mode=ro", uri=True) as db: + self.assertEqual(db.execute("SELECT title FROM item").fetchall(), + [("Synthetic track",)] * count) + + def test_nano_6_create_resave_delete(self): + # Synthetic capabilities: no Nano 6 SysInfoExtended capture is available. + properties = {"FireWireGUID": "0123456789ABCDEF", "SQLiteDB": True, "DBVersion": 5} + (self.control / "Device/SysInfoExtended").write_bytes(plistlib.dumps(properties)) + (self.control / "Device/SysInfo").write_text("ModelNumStr: MC526\n") + for operation, count in (("create", 1), ("resave", 1), ("delete", 0)): + self.run_helper(operation) + self.check_library(count) + + def test_nano_7_create_resave_delete(self): + self.setup_nano() + for operation, count in (("create", 1), ("resave", 1), ("delete", 0)): + self.run_helper(operation) + self.check_library(count) + + def test_invalid_or_ios_guid_is_rejected(self): + for guid in ("", "0123456789ABCDEG", "0123456789ABCDE", "01" * 20): + with self.subTest(guid=guid): + self.setup_nano(guid) + result = self.run_helper("create", success=False) + self.assertIn("GUID", result.stderr) + + def test_prefixed_guid(self): + self.setup_nano("0x0123456789ABCDEF") + self.run_helper("create") + self.check_library(1) + + def test_video_5g_remains_unsigned(self): + (self.control / "Device/SysInfo").write_text("ModelNumStr: MA002\n") + self.run_helper("create") + data = (self.control / "iTunes/iTunesDB").read_bytes() + self.assertEqual(data[:4], b"mhbd") + self.assertEqual(struct.unpack_from("/dev/null | grep -q musl; then diff --git a/tools/prebuild/libgpod-nano.patch b/tools/prebuild/libgpod-nano.patch new file mode 100644 index 00000000..cc8b46ba --- /dev/null +++ b/tools/prebuild/libgpod-nano.patch @@ -0,0 +1,126 @@ +--- a/src/itdb_hashAB.c ++++ b/src/itdb_hashAB.c +@@ -37,35 +37,9 @@ + #include "db-itunes-parser.h" + #include "itdb_private.h" + +-typedef void (*calcHashAB_t)(unsigned char target[57], const unsigned char sha1[20], const unsigned char uuid[20], const unsigned char rnd_bytes[23]); +-static calcHashAB_t calc_hashAB = NULL; +- +-static gboolean load_libhashab() +-{ +- gchar *path; +- GModule *handle; +- +- if (!g_module_supported()) { +- return FALSE; +- } +- path = g_module_build_path(LIBGPOD_BLOB_DIR, "hashab"); +- handle = g_module_open(path, G_MODULE_BIND_LAZY); +- g_free(path); +- if (!handle) { +- return FALSE; +- } +- +- if (!g_module_symbol(handle, "calcHashAB", (void**)&calc_hashAB)) { +- g_module_close(handle); +- g_warning("symbol 'calcHashAB' not found"); +- return FALSE; +- } +- g_module_make_resident(handle); +- +- printf("***** hashAB support successfully loaded *****\n"); +- +- return TRUE; +-} ++/* The vendored signer's symbols are renamed in src/Makefile.am. */ ++#define calcHashAB itdb_hashab_calcHashAB ++#include "nano_hashab/calcHashAB.h" + + static void itdb_hashAB_compute_itunesdb_sha1 (unsigned char *itdb_data, + gsize itdb_len, +@@ -111,16 +85,26 @@ gboolean itdb_hashAB_compute_hash_for_sha1 (const Itdb_Device *device, + unsigned char uuid[20]; + unsigned char rndpart[23] = "ABCDEFGHIJKLMNOPQRSTUVW"; + +- if (calc_hashAB == NULL) { +- if (!load_libhashab()) { +- g_set_error (error, 0, -1, "Unsupported checksum type"); +- return FALSE; +- } ++ const char *guid = itdb_device_get_firewire_id(device); ++ guint i; ++ ++ /* The open signer accepts eight-byte Nano GUIDs, not iOS UUIDs. */ ++ if (guid != NULL && (g_str_has_prefix(guid, "0x") || g_str_has_prefix(guid, "0X"))) ++ guid += 2; ++ if (guid == NULL || strlen(guid) != 16) { ++ g_set_error(error, 0, -1, "HashAB requires an eight-byte Nano GUID"); ++ return FALSE; ++ } ++ for (i = 0; i < 16; i++) { ++ if (!g_ascii_isxdigit(guid[i])) { ++ g_set_error(error, 0, -1, "Invalid hexadecimal Nano GUID"); ++ return FALSE; ++ } + } + + if (!itdb_device_get_hex_uuid(device, uuid)) return FALSE; + +- calc_hashAB(signature, sha1, uuid, rndpart); ++ calcHashAB(signature, sha1, uuid, rndpart); + + return TRUE; + } +--- a/src/Makefile.am ++++ b/src/Makefile.am +@@ -31,6 +31,33 @@ libgpod_la_CFLAGS = + libgpod_la_LDFLAGS = -version-info $(LIBGPOD_SO_VERSION) -no-undefined + libgpod_la_CPPFLAGS = -DLIBGPOD_BLOB_DIR=\"$(LIBGPOD_BLOB_DIR)\" + ++# The vendored HashAB signer for the iPod nano 6G/7G relies on wrapping signed ++# arithmetic. Its global symbols are renamed and hidden, so that they neither ++# clash with other code linked with libgpod nor become part of its interface. ++noinst_LTLIBRARIES = libnanohashab.la ++libnanohashab_la_SOURCES = \ ++ nano_hashab/calcHashAB.c \ ++ nano_hashab/cipher.c \ ++ nano_hashab/cipher_tables.c \ ++ nano_hashab/edonr.c \ ++ nano_hashab/expand.c \ ++ nano_hashab/hash.c \ ++ nano_hashab/reduce.c ++libnanohashab_la_CFLAGS = -fwrapv -fvisibility=hidden ++libnanohashab_la_CPPFLAGS = \ ++ -DcalcHashAB=itdb_hashab_calcHashAB \ ++ -DEdonR256Compress=itdb_hashab_EdonR256Compress \ ++ -DEdonR256Init=itdb_hashab_EdonR256Init \ ++ -Dexpand=itdb_hashab_expand \ ++ -Dhash=itdb_hashab_hash \ ++ -Dreduce=itdb_hashab_reduce \ ++ -Dstage1_cipher=itdb_hashab_stage1_cipher \ ++ -Dstage1_initial_chain=itdb_hashab_stage1_initial_chain \ ++ -Dstage1_profile=itdb_hashab_stage1_profile \ ++ -Dstage2_cipher=itdb_hashab_stage2_cipher \ ++ -Dstage2_profile=itdb_hashab_stage2_profile ++libgpod_la_LIBADD = libnanohashab.la ++ + libgpodincludebase = $(includedir)/gpod-1.0 + libgpodincludedir = $(libgpodincludebase)/gpod + libgpodinclude_HEADERS = itdb.h +@@ -51,6 +78,15 @@ noinst_HEADERS = \ + itdb_zlib.h \ + pixmaps.h \ + rijndael.h ++noinst_HEADERS += \ ++ nano_hashab/byteorder.h \ ++ nano_hashab/calcHashAB.h \ ++ nano_hashab/cipher.h \ ++ nano_hashab/cipher_tables.h \ ++ nano_hashab/edonr.h \ ++ nano_hashab/expand.h \ ++ nano_hashab/hash.h \ ++ nano_hashab/reduce.h + + INCLUDES=$(LIBGPOD_CFLAGS) + LIBS=$(LIBGPOD_LIBS) $(Z_LIBS) -lm diff --git a/tools/prebuild/prepare-libgpod-nano.sh b/tools/prebuild/prepare-libgpod-nano.sh new file mode 100755 index 00000000..5eda88e3 --- /dev/null +++ b/tools/prebuild/prepare-libgpod-nano.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Apply the shared Nano signing patch before autoreconf. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +fingerprint() { + cat "$SCRIPT_DIR/libgpod-nano.patch" "$SCRIPT_DIR"/vendor/hashab/src/* | cksum +} + +case "${1:-}" in + --check) + test -f "$2/lib/libgpod.a" && test -f "$2/lib/libgpod-nano.stamp" && + test "$(cat "$2/lib/libgpod-nano.stamp")" = "$(fingerprint)" + ;; + --stamp) + fingerprint > "$2/lib/libgpod-nano.stamp" + ;; + *) + SOURCE="${1:?usage: prepare-libgpod-nano.sh | --check/--stamp }" + # Reject an unexpected or already-patched source before copying anything. + patch --dry-run --batch --forward -d "$SOURCE" -p1 < "$SCRIPT_DIR/libgpod-nano.patch" + patch --batch --forward -d "$SOURCE" -p1 < "$SCRIPT_DIR/libgpod-nano.patch" + mkdir -p "$SOURCE/src/nano_hashab" + cp "$SCRIPT_DIR"/vendor/hashab/src/* "$SOURCE/src/nano_hashab/" + cp "$SCRIPT_DIR/vendor/hashab/LICENSE" "$SOURCE/src/nano_hashab/" + ;; +esac From 534cc7443bcdfa1cf7faf7e1ca94ecf3fcbf43f7 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 20:02:34 +0100 Subject: [PATCH 06/21] fix(devices): describe nano 6G/7G as signed but not yet writable The support notes, unsupported-device reasons and docs still said libgpod cannot produce the nano 6G database format or needs an external hashab blob. With the signer linked in, the remaining blocker is different: on a nano 7G the firmware shows no music for a library libgpod writes, even though its signatures verify. The nano 6G write path is covered only by synthetic tests. Both generations stay read-only. The notes and reasons now say that syncing is not supported yet and, for the nano 7G, what the firmware shows; the generated support matrix, the supported-devices page and the device notes follow, the generation notes in docs/formats/generations.md and docs/test-devices.md point to ADR-030, and the classify/identify tests track the new reason. The generation comments drop the test devices' serial numbers. --- .changeset/nano-native-hashab.md | 3 ++- devices/ipod.md | 9 ++++++-- .../device/identity-support-matrix.md | 4 ++-- docs/formats/generations.md | 19 +++++++--------- docs/test-devices.md | 2 ++ packages/devices-ipod/src/classify.test.ts | 2 +- packages/devices-ipod/src/identity.test.ts | 4 ++-- packages/devices-ipod/src/support.test.ts | 3 --- .../devices-ipod/src/tables/generations.ts | 22 ++++++++----------- .../devices-ipod/src/tables/unsupported.ts | 20 +++++++---------- packages/devices-ipod/src/types.ts | 7 +++--- .../docs/devices/supported-devices.mdx | 4 ++-- 12 files changed, 46 insertions(+), 53 deletions(-) diff --git a/.changeset/nano-native-hashab.md b/.changeset/nano-native-hashab.md index 16ee099e..d3841b8a 100644 --- a/.changeset/nano-native-hashab.md +++ b/.changeset/nano-native-hashab.md @@ -1,6 +1,7 @@ --- '@podkit/libgpod-node': patch +'@podkit/devices-ipod': patch 'podkit': patch --- -podkit's libgpod build now signs iPod nano 6th and 7th generation databases itself, instead of needing an external `hashab` module. Both generations remain read-only: the nano 7th generation shows no music from a library written this way yet, although its signatures are valid. +podkit's libgpod build now signs iPod nano 6th and 7th generation databases itself, instead of needing an external `hashab` module. Signing alone does not make them writable: the nano 7th generation shows no music from a library written this way, although its signatures are valid, so both stay read-only with this change. Their support notes and unsupported-device messages now say so, instead of naming the missing module. diff --git a/devices/ipod.md b/devices/ipod.md index 2a352fec..2b2f112d 100644 --- a/devices/ipod.md +++ b/devices/ipod.md @@ -293,8 +293,8 @@ iPod_Control/ | Generation | Reason | |------------|--------| | Shuffle 3-4G | Buttonless; writing their iTunesSD playback database is unverified on hardware | -| Nano 6G | Writes an iTunesDB format libgpod cannot produce; read is untested | -| Nano 7G | Reads and archives cleanly (hardware-confirmed); writing needs the hashAB signature libgpod cannot produce without an external blob podkit doesn't ship | +| Nano 6G | Reads and archives (hardware-confirmed); syncing is not supported yet | +| Nano 7G | Reads and archives cleanly (hardware-confirmed); the firmware shows no music for a library libgpod writes | | Touch (all) | iOS protocol + cryptographic iTunesDB signing | | iPhone (all) | iOS protocol + cryptographic iTunesDB signing | | iPad | iOS protocol + cryptographic iTunesDB signing | @@ -348,3 +348,8 @@ Fully implemented in podkit. Key implementation files: - **Video handler:** `packages/podkit-core/src/sync/handlers/video-handler.ts` - **Generation metadata:** `packages/podkit-core/src/ipod/generation.ts` - **Model identification:** `packages/podkit-core/src/device/ipod-models.ts` + +## Nano 6G/7G signing + +The nano 6th and 7th generation sign `iTunesCDB` and `Locations.itdb.cbk` with hashAB, which podkit's libgpod builds compute ([ADR-030](../docs/adr/adr-030-native-nano-hashab.md); checks in `tools/nano7/`). +On a nano 7G the firmware shows no music for a library libgpod writes, so both generations stay read-only. diff --git a/docs/architecture/device/identity-support-matrix.md b/docs/architecture/device/identity-support-matrix.md index 0fdc7ced..2c6ec6e8 100644 --- a/docs/architecture/device/identity-support-matrix.md +++ b/docs/architecture/device/identity-support-matrix.md @@ -70,8 +70,8 @@ is the fallback on USB *transport* error | iPod nano 1G–2G | **no (2G does not respond; 1G untested)** | **yes** | SCSI (host only) | | iPod nano 3G | **yes** (first USB-capable) | untested | USB | | iPod nano 4G–5G | **yes** | yes | USB | -| iPod nano 6G | yes (inquiry) | yes | Inquiry works but the generation is **read-only** (iTunesDB format podkit cannot write) | -| iPod nano 7G | yes (inquiry) | yes | Inquiry works; the generation is **read-only**, hardware-confirmed (hashAB signing unavailable — libgpod needs an external blob podkit doesn't ship — but reads and archives cleanly) | +| iPod nano 6G | yes (inquiry) | yes | Inquiry works but the generation is **read-only**, hardware-confirmed (reads and archives; syncing is not supported yet) | +| iPod nano 7G | yes (inquiry) | yes | Inquiry works; the generation is **read-only**, hardware-confirmed (reads and archives cleanly; the firmware shows no music for a library libgpod writes) | | iPod shuffle 1G–2G | no | **yes** | SCSI (host only) | | iPod shuffle 3G–4G | yes | yes | Inquiry works but sync is **refused** (bdhs iTunesSD write unverified; read-only tier) | | iPod touch / iPhone / iPad | no | no | No disk mode at all — out of scope | diff --git a/docs/formats/generations.md b/docs/formats/generations.md index 3929a7f7..4487b8ea 100644 --- a/docs/formats/generations.md +++ b/docs/formats/generations.md @@ -69,8 +69,8 @@ after changing the table, then regenerate this block from | iPod nano (3rd Generation) | `nano_3g` | syncable | inferred | — | | iPod nano (4th Generation) | `nano_4g` | syncable | inferred | — | | iPod nano (5th Generation) | `nano_5g` | syncable | inferred | — | -| iPod nano (6th Generation) | `nano_6g` | read-only | hardware | Reads and archives (259 tracks confirmed on hardware); writing needs an iTunesDB format libgpod cannot produce. | -| iPod nano (7th Generation) | `nano_7g` | read-only | hardware | Reads iTunesDB (1,414 tracks confirmed) and archives cleanly; writing needs the hashAB signature libgpod cannot produce without an external blob podkit does not ship. | +| iPod nano (6th Generation) | `nano_6g` | read-only | hardware | Reads and archives (259 tracks confirmed on hardware); syncing is not supported yet. | +| iPod nano (7th Generation) | `nano_7g` | read-only | hardware | Reads iTunesDB (1,414 tracks confirmed) and archives cleanly; the firmware shows no music for a library libgpod writes, so syncing is not supported yet. | | iPod shuffle (1st Generation) | `shuffle_1g` | syncable | inferred | — | | iPod shuffle (2nd Generation) | `shuffle_2g` | syncable | inferred | — | | iPod shuffle (3rd Generation) | `shuffle_3g` | read-only | hardware | Reads iTunesDB; writing the bdhs iTunesSD playback DB is unverified on hardware. | @@ -111,17 +111,14 @@ after changing the table, then regenerate this block from shown to leave a playable device, so podkit does not attempt it. Both are `hardware`-confirmed. See [itunessd-bdhs.md](itunessd-bdhs.md). -- **nano 6g — `read-only`.** Its write is a format libgpod cannot produce, but - its read is merely *untested* — and a read is non-destructive, so the tier - permits the attempt rather than forbidding a safe operation on a guess. +- **nano 6g — `read-only`, hardware-confirmed.** A real nano 6G read and + archived 259 tracks. It signs its databases with hashAB like the nano 7g; + its writes are covered only by synthetic tests. - **nano 7g — `read-only`, hardware-confirmed.** libgpod opens its classic iTunesCDB `iTunesDB` fine — a real nano 7G read 1,414 tracks and archived - cleanly. Writing needs a hashAB signature, which libgpod only computes via - an external `hashab` blob loaded through `LIBGPOD_BLOB_DIR` - (`itdb_hashAB.c`); podkit ships no such blob, so the write path fails - closed. This corrects an earlier claim that nano 7g had no libgpod table - entry at all — it reads fine; only the write is blocked, and for a - different reason. + cleanly. podkit's libgpod build signs its databases with hashAB + ([ADR-030](../adr/adr-030-native-nano-hashab.md)), but the firmware shows no + music for a library libgpod writes, so writes stay refused. - **iPod touch (all), not-in-libgpod — `none`.** No mountable database, or Apple's proprietary sync protocol that podkit cannot speak over disk mode. diff --git a/docs/test-devices.md b/docs/test-devices.md index 34e2e3ac..05915943 100644 --- a/docs/test-devices.md +++ b/docs/test-devices.md @@ -237,6 +237,8 @@ archiving work fine. | Filesystem | Confirms readability | Device also carries `iTunes Library.itlp`, the SQLite database the firmware plays from — separate from the libgpod-readable `iTunesCDB`. | | Sync (write) | **Genuinely refused** | Not a false refusal this time: `checksumType: 'hashAB'`, and libgpod does not implement hashAB signing. `itdb_hashAB.c:43-68` (`load_libhashab`) `g_module_open`s an external `hashab` blob from `LIBGPOD_BLOB_DIR` and fails closed when the symbol is absent. podkit ships no such blob (not referenced by `tools/prebuild/*.sh` or `tools/libgpod-macos/build.sh`, none installed). An unsigned database is rejected by the firmware. | +The hashAB part of this refusal is superseded by [ADR-030](adr/adr-030-native-nano-hashab.md). + **Why this unit matters:** the prior `access: 'none'` entry for nano 7G was itself wrong — it read "not in libgpod's `ipod_info_table`" as "no mountable database," but the two are unrelated. libgpod's classic iTunesCDB reader works diff --git a/packages/devices-ipod/src/classify.test.ts b/packages/devices-ipod/src/classify.test.ts index c608223e..442072f0 100644 --- a/packages/devices-ipod/src/classify.test.ts +++ b/packages/devices-ipod/src/classify.test.ts @@ -88,7 +88,7 @@ describe('classifyAsIpod — unsupported iPod-family devices', () => { const result = classifyAsIpod({ vendorId: '05ac', productId: '120d' }); expect(result).not.toBeNull(); expect(result!.supported).toBe(false); - expect(result!.unsupportedReason?.headline).toContain('iTunesDB format'); + expect(result!.unsupportedReason?.headline).toContain('writing its library is not supported'); }); it('classifies iPod nano 7G as unsupported (0x05ac:0x1267)', () => { diff --git a/packages/devices-ipod/src/identity.test.ts b/packages/devices-ipod/src/identity.test.ts index 67672cd2..fe7f200f 100644 --- a/packages/devices-ipod/src/identity.test.ts +++ b/packages/devices-ipod/src/identity.test.ts @@ -228,11 +228,11 @@ describe('identify', () => { expect(model!.unsupportedReason!.headline).toContain('unverified on hardware'); }); - test('nano 6G (0x120d) returns unsupportedReason (iTunesDB format)', () => { + test('nano 6G (0x120d) returns unsupportedReason (writes not supported)', () => { const model = identify({ from: 'usb', productId: '0x120d' }); expect(model).toBeDefined(); expect(model!.generationId).toBe('nano_6g'); - expect(model!.unsupportedReason!.headline).toContain('iTunesDB format'); + expect(model!.unsupportedReason!.headline).toContain('writing its library is not supported'); }); test('supported devices do NOT have unsupportedReason', () => { diff --git a/packages/devices-ipod/src/support.test.ts b/packages/devices-ipod/src/support.test.ts index e89ba920..f45d1d26 100644 --- a/packages/devices-ipod/src/support.test.ts +++ b/packages/devices-ipod/src/support.test.ts @@ -42,9 +42,6 @@ describe('resolveGenerationSupport', () => { }); it('reports nano 7g as read-only, hardware-verified', () => { - // Promoted from `none`/`inferred`: read on real hardware (1,414 tracks, - // `device archive` succeeded). The write refusal is hashAB signing, not - // an absent libgpod table entry as previously claimed. const support = resolveGenerationSupport('nano_7g'); expect(support.access).toBe('read-only'); expect(support.verified).toBe('hardware'); diff --git a/packages/devices-ipod/src/tables/generations.ts b/packages/devices-ipod/src/tables/generations.ts index 37d0c4b9..cdc13e81 100644 --- a/packages/devices-ipod/src/tables/generations.ts +++ b/packages/devices-ipod/src/tables/generations.ts @@ -15,7 +15,7 @@ * libgpod equivalent. Generations that map to libgpod's `unknown` (nano 7G, * touch 5G–7G) are sourced exclusively from this table for capability data. * Their `support.access` differs by generation: nano 7G is `read-only` - * (hardware-confirmed readable; hashAB signing unavailable), touch 5G–7G + * (hardware-confirmed readable; library writes not supported yet), touch 5G–7G * carry `support.access: 'none'` (no disk mode at all). * * @module @@ -190,13 +190,13 @@ export const GENERATIONS: Record = { family: 'iPod nano', ordinal: 6, checksumType: 'hashAB', - // Multi-touch nano with no video playback hardware. Write is a format - // libgpod cannot produce. Read is confirmed on hardware: a 16GB nano 6G - // (serial DCYGLUGVDDW4) listed and archived 259 tracks cleanly. + // Multi-touch nano with no video playback hardware. Read is confirmed on + // hardware (259 tracks listed and archived). It signs with hashAB like the + // nano 7G (ADR-030); its writes are covered only by synthetic tests. support: { access: 'read-only', verified: 'hardware', - note: 'Reads and archives (259 tracks confirmed on hardware); writing needs an iTunesDB format libgpod cannot produce.', + note: 'Reads and archives (259 tracks confirmed on hardware); syncing is not supported yet.', }, supportsAlac: false, supportsVideo: false, @@ -207,17 +207,13 @@ export const GENERATIONS: Record = { family: 'iPod nano', ordinal: 7, checksumType: 'hashAB', - // Hardware-confirmed readable: podkit read 1,414 tracks off a real - // nano 7G (serial DCYN83SFF0GQ) via libgpod's classic iTunesCDB parser, - // and `device archive` succeeded. The write refusal is hashAB signing — - // libgpod only computes it via an external `hashab` blob loaded through - // `LIBGPOD_BLOB_DIR` (itdb_hashAB.c:43-68) and fails closed without it; - // podkit ships no such blob. Not a libgpod-table gap, as previously - // claimed here. + // Read on real hardware through libgpod's iTunesCDB parser (1,414 tracks, + // `device archive` succeeded). Its databases are signed (ADR-030), but the + // firmware shows no music for a library libgpod writes, so writes stay refused. support: { access: 'read-only', verified: 'hardware', - note: 'Reads iTunesDB (1,414 tracks confirmed) and archives cleanly; writing needs the hashAB signature libgpod cannot produce without an external blob podkit does not ship.', + note: 'Reads iTunesDB (1,414 tracks confirmed) and archives cleanly; the firmware shows no music for a library libgpod writes, so syncing is not supported yet.', }, supportsAlac: true, supportsVideo: true, diff --git a/packages/devices-ipod/src/tables/unsupported.ts b/packages/devices-ipod/src/tables/unsupported.ts index 7fcdd89c..575f307d 100644 --- a/packages/devices-ipod/src/tables/unsupported.ts +++ b/packages/devices-ipod/src/tables/unsupported.ts @@ -15,10 +15,10 @@ import { SUPPORTED_DEVICES_DOCS_URL } from '../build-unsupported-reason.js'; * Unsupported categories: * - Shuffle 3G/4G: readable, but writing their `bdhs` playback database has * never been verified on hardware, so podkit does not attempt it. - * - Nano 6G: libgpod has table entries but cannot write the iTunesDB format. + * - Nano 6G: readable; syncing is not supported yet. * - Nano 7G: readable — libgpod opens its iTunesCDB fine (hardware-confirmed, - * 1,414 tracks). Write is refused because libgpod's hashAB signing needs an - * external blob (`LIBGPOD_BLOB_DIR`) podkit does not ship. + * 1,414 tracks). Its databases are signed (ADR-030), but the firmware shows no + * music for a library libgpod writes. * - iPod touch (all generations): Apple's proprietary sync protocol; no disk mode. * - iPhone / iPad: Apple's proprietary sync protocol; no disk mode. * - Apple TV, Apple Watch, HomePod: non-iPod Apple USB devices; out of scope. @@ -46,16 +46,12 @@ const SHUFFLE_REASON = 'iPod shuffle 3rd/4th gen can be read but not written: writing its iTunesSD playback database is unverified on hardware.'; const NANO_6G_REASON = - 'iPod nano 6th gen uses an iTunesDB format podkit cannot write; read access is untested.'; - -// nano_7g reads fine — libgpod opens its classic iTunesCDB database without -// trouble (hardware-confirmed: 1,414 tracks, `device archive` succeeded). -// Writing needs a hashAB signature; libgpod only computes hashAB via an -// external `hashab` blob loaded through `LIBGPOD_BLOB_DIR` -// (itdb_hashAB.c:43-68) and fails closed without it. podkit ships no such -// blob, so the write path is refused while reads remain fine. + 'iPod nano 6th gen can be read and archived, but not synced yet: writing its library is not supported.'; + +// Valid signatures are not enough: the firmware shows no music for a library +// libgpod writes (ADR-030). const NANO_7G_REASON = - 'iPod nano 7th gen can be read and archived, but not synced: writing needs the hashAB signature libgpod cannot produce without an external blob podkit does not ship.'; + 'iPod nano 7th gen can be read and archived, but not synced yet: its firmware shows no music for a library podkit writes.'; const itouch = (gen: string) => `iPod touch (${gen}) uses Apple's proprietary sync protocol; podkit only supports iPod disk mode.`; diff --git a/packages/devices-ipod/src/types.ts b/packages/devices-ipod/src/types.ts index b0a7e4b1..84f401d7 100644 --- a/packages/devices-ipod/src/types.ts +++ b/packages/devices-ipod/src/types.ts @@ -104,11 +104,10 @@ export interface IpodGeneration { * Notable non-`syncable` generations: * - shuffle_3g/4g: `read-only` — readable iTunesDB, but writing their * `bdhs` iTunesSD has never been confirmed on hardware. - * - nano_6g: `read-only` — write is a format libgpod cannot produce; read - * is untested (non-destructive, so permitted). + * - nano_6g: `read-only` — reads and archives (hardware-confirmed); + * syncing is not supported yet. * - nano_7g: `read-only` — reads its iTunesDB fine (hardware-confirmed); - * write is refused because libgpod's hashAB signing needs an external - * blob podkit doesn't ship. + * the firmware shows no music for a library libgpod writes. * - touch/iPhone/iPad, not-in-libgpod: `none` — no mountable database or a * protocol podkit cannot speak. */ diff --git a/packages/docs-site/src/content/docs/devices/supported-devices.mdx b/packages/docs-site/src/content/docs/devices/supported-devices.mdx index d8855f0f..3a3c7dd7 100644 --- a/packages/docs-site/src/content/docs/devices/supported-devices.mdx +++ b/packages/docs-site/src/content/docs/devices/supported-devices.mdx @@ -112,7 +112,7 @@ podkit uses [libgpod](https://github.com/fadingred/libgpod) for iPod database ma iPod support comes in three tiers, because reading a device and writing to it are different operations: - **Full** — podkit reads *and* syncs the device. This is every classic, mini, nano (1st–5th gen), photo, video, and shuffle (1st–2nd gen) iPod. -- **Read-only** — podkit reads the library and can [archive](/user-guide/devices/archive) it, but **cannot sync**. The buttonless shuffles (3rd/4th gen) keep a libgpod-readable `iTunesDB`, but writing the `iTunesSD` playback database their firmware plays from has never been shown to leave such a device playable. The nano 6th gen writes a database format libgpod cannot write. The nano 7th gen reads and archives cleanly, but writing needs a hashAB signature libgpod can only produce via an external blob podkit doesn't ship. Reading and archiving these is entirely safe; only writes are refused. +- **Read-only** — podkit reads the library and can [archive](/user-guide/devices/archive) it, but **cannot sync**. The buttonless shuffles (3rd/4th gen) keep a libgpod-readable `iTunesDB`, but writing the `iTunesSD` playback database their firmware plays from has never been shown to leave such a device playable. The nano 6th and 7th gen read and archive cleanly; podkit's libgpod build can sign their databases (hashAB), but complete library writes are not supported yet. Reading and archiving these is entirely safe; only writes are refused. - **Unsupported** — podkit cannot use the device at all. iOS devices (iPod Touch, iPhone, iPad) don't mount as mass storage. ### How We Test iPods @@ -158,7 +158,7 @@ Reach for `podkit device archive` to preserve one of these devices: podkit device archive -d "/Volumes/IPOD" ``` -Why read but not write? The buttonless shuffles (3rd/4th gen) keep a libgpod-readable `iTunesDB` alongside the `iTunesSD` playback database the firmware actually plays from. Reading the `iTunesDB` is fine; writing the `iTunesSD` is not blocked by anything cryptographic, but no such write has been confirmed to leave a playable device, so podkit does not attempt it. The nano 6th gen's write path is likewise unsupported by libgpod, while its read is merely untested — so podkit permits the safe read and refuses the write. The nano 7th gen reads its `iTunesDB` fine (confirmed on real hardware, 1,414 tracks), but writing requires a hashAB signature that libgpod can only compute via an external blob podkit doesn't ship — so, same as the shuffle and nano 6th gen, the read is permitted and the write is refused. +Why read but not write? The buttonless shuffles (3rd/4th gen) keep a libgpod-readable `iTunesDB` alongside the `iTunesSD` playback database the firmware actually plays from. Reading the `iTunesDB` is fine; writing the `iTunesSD` is not blocked by anything cryptographic, but no such write has been confirmed to leave a playable device, so podkit does not attempt it. The nano 6th and 7th gen read fine (confirmed on real hardware: 259 and 1,414 tracks). Their databases carry a hashAB signature, which podkit's libgpod build now computes, but signing alone is not enough: the nano 7th gen firmware shows no music for a library written by libgpod, and the nano 6th gen write path is only covered by synthetic tests. So, same as the shuffle, the read is permitted and the write is refused. ### Unsupported iPods From caf1d7c83daa6a7656c00ab9e22687d3d532d9cf Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 20:12:24 +0100 Subject: [PATCH 07/21] feat(libgpod): give iPod nano 6G/7G SQLite libraries iTunes' schema level A nano 7G (firmware 1.0.4) shows no music for a library libgpod writes, although its signatures verify. Replacing the databases on the device showed that the firmware needs the hidden category containers together with iTunes' schema level; the containers alone are not enough. For devices that sign with HashAB and are not iOS devices, the patch now - adds the columns iTunes' Library.itdb has beyond libgpod's schema, appended after the tables are created; the inserts that relied on column positions name their columns, - writes version_info 1.117 as iTunes does and sets the Genius.itdb user_version to 2, - runs the device's post-process command set 25, which fits that schema, adds is_home_video and sets user_version 26, instead of the newest set. Two commands of the set add a column that is already there: item_stats.rental_expired, which libgpod's schema has, is skipped once the column is confirmed, and the repeated AddGenreMapCompilationCount only after its first run succeeded. On these devices a missing command set, a missing, malformed or failing command, an insert that fails while the databases are generated or a failed commit now fails itdb_write() before any library file is replaced; failed post-process commands are rolled back. Other devices keep logging such failures. Attached database paths are quoted. The tools/nano7 roundtrip checks the schema level for nano 6G/7G, that each kind of command failure, a rejected insert and a commit that fails as on a full disk leave the library unchanged, that the first of the repeated genre commands runs, and that a nano 5G keeps libgpod's layout. Both generations stay read-only. --- tools/nano7/README.md | 8 + tools/nano7/libgpod-roundtrip.c | 10 +- tools/nano7/test_libgpod.py | 143 +++++++- tools/prebuild/libgpod-nano.patch | 584 ++++++++++++++++++++++++++++++ 4 files changed, 735 insertions(+), 10 deletions(-) diff --git a/tools/nano7/README.md b/tools/nano7/README.md index f52b3316..44d3930f 100644 --- a/tools/nano7/README.md +++ b/tools/nano7/README.md @@ -42,6 +42,14 @@ synthetic capabilities. It checks the integrity of the five SQLite databases, both signatures, rejection of malformed or iOS-sized GUIDs, and an unsigned `iTunesDB` for a 5th generation iPod. +For the nano 6th and 7th generation the checks also cover iTunes' schema +level: its additional columns, `user_version` 26 (`Library.itdb`) and 2 +(`Genius.itdb`) and the `version_info` row. The device's SQLite post-process +command set 25 completes that schema. A missing set, a missing or failing +command, or an insert or commit that fails while the databases are generated +fails the write and leaves the library unchanged. A nano 5th generation keeps +libgpod's previous layout. + ## Checking a copied device library A nano 7th generation synced by macOS carries no `SysInfoExtended` file, so diff --git a/tools/nano7/libgpod-roundtrip.c b/tools/nano7/libgpod-roundtrip.c index d0d4808f..11a36f24 100644 --- a/tools/nano7/libgpod-roundtrip.c +++ b/tools/nano7/libgpod-roundtrip.c @@ -1,5 +1,6 @@ -/* Create, resave or delete one synthetic track with libgpod. - * Usage: libgpod-roundtrip create|resave|delete */ +/* Create, resave or delete one synthetic track with libgpod, give it a 1 MiB + * comment (grow) or the second playlist the master playlist's id (clash). + * Usage: libgpod-roundtrip create|resave|delete|grow|clash */ #include #include @@ -52,6 +53,11 @@ int main(int argc, char **argv) for (item = db->playlists; item; item = item->next) itdb_playlist_remove_track(item->data, track); itdb_track_remove(track); + } else if (strcmp(argv[2], "grow") == 0) { + g_free(track->comment); + track->comment = g_strnfill(1 << 20, 'x'); + } else if (strcmp(argv[2], "clash") == 0) { + ((Itdb_Playlist *)g_list_last(db->playlists)->data)->id = itdb_playlist_mpl(db)->id; } else if (strcmp(argv[2], "resave") != 0) { return 2; } diff --git a/tools/nano7/test_libgpod.py b/tools/nano7/test_libgpod.py index 5ff8efe2..c122c452 100644 --- a/tools/nano7/test_libgpod.py +++ b/tools/nano7/test_libgpod.py @@ -1,8 +1,11 @@ """Exercise a compiled libgpod-roundtrip helper on temporary synthetic devices.""" +import contextlib import os import plistlib from pathlib import Path import re +import resource +import signal import sqlite3 import struct import subprocess @@ -12,6 +15,39 @@ from build import build, ROOT from probe import HashAB, guid_bytes, verify_cbk, verify_cdb +CAPTURE = ROOT.parents[1] / "docs/sysinfo-captures/nano-7g-16gb-usb.xml" +POST_PROCESS = "com.apple.mobile.iTunes.SQLMusicLibraryPostProcessCommands" +APPLE_COLUMNS = { + "item": {"is_home_video", "is_tone", "is_book", "is_itunes_u", "is_digital_booklet", + "chosen_by_auto_fill", "collection_description", "copyright"}, + "avformat_info": {"channels"}, + "video_info": {"is_hd"}, + "store_info": {"xid", "flavor"}, + "container": {"smart_evaluation_order"}, +} + + +def rows(path, sql, params=()): + with contextlib.closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as db: + return db.execute(sql, params).fetchall() + + +def columns(path, table): + return {row[1] for row in rows(path, f"PRAGMA table_info({table})")} + + +def post_process_commands(): + # The capture is not strictly valid XML elsewhere; take only this dictionary. + text = CAPTURE.read_text() + start = text.index("", text.index(f"{POST_PROCESS}")) + depth = 0 + for match in re.finditer(r"|", text[start:]): + depth += 1 if match[0] == "" else -1 + if depth == 0: + body = text[start:start + match.end()] + break + return plistlib.loads(f'{body}'.encode()) + class LibgpodRoundtripTests(unittest.TestCase): @classmethod @@ -21,6 +57,7 @@ def setUpClass(cls): raise FileNotFoundError(cls.helper) cls.build_dir = tempfile.TemporaryDirectory() cls.hasher = HashAB(build(Path(cls.build_dir.name) / "hashab.so")) + cls.commands = post_process_commands() @classmethod def tearDownClass(cls): @@ -31,21 +68,32 @@ def setUp(self): self.addCleanup(self.temp.cleanup) self.mount = Path(self.temp.name) self.control = self.mount / "iPod_Control" + self.bundle = self.control / "iTunes/iTunes Library.itlp" for name in ("Device", "Artwork", "iTunes", "Music/F00"): (self.control / name).mkdir(parents=True) (self.control / "Music/F00/TEST.mp3").write_bytes(bytes(32000)) + def db(self, name): + return self.bundle / f"{name}.itdb" + def setup_nano(self, guid="0123456789ABCDEF"): # Keep the device's unusual plist arrays verbatim, including AlbumArt2. - capture = (ROOT.parents[1] / "docs/sysinfo-captures/nano-7g-16gb-usb.xml").read_text() capture = re.sub(r"(FireWireGUID\s*)[^<]+", - lambda m: m[1] + guid, capture) + lambda m: m[1] + guid, CAPTURE.read_text()) (self.control / "Device/SysInfoExtended").write_text(capture) (self.control / "Device/SysInfo").write_text("ModelNumStr: MD480\n") - def run_helper(self, operation, success=True): + def setup_synthetic(self, model, db_version, commands=None): + # Synthetic capabilities: no Nano 5/6 SysInfoExtended capture is available. + properties = {"FireWireGUID": "0123456789ABCDEF", "SQLiteDB": True, "DBVersion": db_version} + if commands is not None: + properties[POST_PROCESS] = commands + (self.control / "Device/SysInfoExtended").write_bytes(plistlib.dumps(properties)) + (self.control / "Device/SysInfo").write_text(f"ModelNumStr: {model}\n") + + def run_helper(self, operation, success=True, preexec_fn=None): result = subprocess.run([str(self.helper), str(self.mount), operation], - capture_output=True, text=True) + capture_output=True, text=True, preexec_fn=preexec_fn) if success: self.assertEqual(result.returncode, 0, result.stdout + result.stderr) else: @@ -67,20 +115,99 @@ def check_library(self, count): self.assertEqual(db.execute("SELECT title FROM item").fetchall(), [("Synthetic track",)] * count) + def check_schema(self): + """iTunes' columns and version markers.""" + library = self.db("Library") + for table, names in APPLE_COLUMNS.items(): + self.assertLessEqual(names, columns(library, table), table) + self.assertEqual(rows(library, "PRAGMA user_version"), [(26,)]) + self.assertEqual(rows(self.db("Genius"), "PRAGMA user_version"), [(2,)]) + self.assertEqual(rows(library, "SELECT * FROM version_info"), [(1, 1, 117, 0, 0, 1109, 1)]) + def test_nano_6_create_resave_delete(self): - # Synthetic capabilities: no Nano 6 SysInfoExtended capture is available. - properties = {"FireWireGUID": "0123456789ABCDEF", "SQLiteDB": True, "DBVersion": 5} - (self.control / "Device/SysInfoExtended").write_bytes(plistlib.dumps(properties)) - (self.control / "Device/SysInfo").write_text("ModelNumStr: MC526\n") + self.setup_synthetic("MC526", 5, self.commands) for operation, count in (("create", 1), ("resave", 1), ("delete", 0)): self.run_helper(operation) self.check_library(count) + self.check_schema() def test_nano_7_create_resave_delete(self): self.setup_nano() for operation, count in (("create", 1), ("resave", 1), ("delete", 0)): self.run_helper(operation) self.check_library(count) + self.check_schema() + + def snapshot(self): + files = {path.name: path.read_bytes() for path in self.bundle.iterdir()} + files["iTunesCDB"] = (self.control / "iTunes/iTunesCDB").read_bytes() + return files + + def test_nano_post_process_commands_are_required(self): + sets, sql = self.commands["UserVersionCommandSets"], self.commands["SQLCommands"] + named_26 = [26 if name == "FakePragmaITunes26" else name for name in sets["25"]["Commands"]] + cases = { + "none": None, + "only set 26": dict(self.commands, UserVersionCommandSets={"26": sets["26"]}), + "missing command": dict(self.commands, SQLCommands={ + key: value for key, value in sql.items() if key != "FakePragmaITunes26"}), + "failing command": dict(self.commands, SQLCommands=dict( + sql, CreateIsHomeVideo="ALTER TABLE no_such_table ADD COLUMN x INTEGER;")), + "non-string command": dict(self.commands, UserVersionCommandSets=dict( + sets, **{"25": dict(sets["25"], Commands=named_26)})), + } + self.setup_synthetic("MC526", 5, self.commands) + self.run_helper("create") + library = self.snapshot() + for name, commands in cases.items(): + with self.subTest(name): + self.setup_synthetic("MC526", 5, commands) + result = self.run_helper("resave", success=False) + self.assertIn("post-process commands", result.stderr) + self.assertEqual(self.snapshot(), library) + + def test_nano_runs_the_first_genre_count_command(self): + # Set 25 lists AddGenreMapCompilationCount twice; only the repeat is skipped. + sql = dict(self.commands["SQLCommands"]) + sql["AddGenreMapCompilationCount"] += "CREATE TABLE genre_count_added (x INTEGER);" + self.setup_synthetic("MC526", 5, dict(self.commands, SQLCommands=sql)) + self.run_helper("create") + self.assertEqual(rows(self.db("Library"), "SELECT name FROM sqlite_master " + "WHERE name = 'genre_count_added'"), + [("genre_count_added",)]) + + def test_nano_failed_insert_or_commit_fails_the_write(self): + # A playlist with the master playlist's pid breaks a key; a file size + # limit makes the COMMIT of Library.itdb fail, as on a full disk. + self.setup_nano() + self.run_helper("create") + library = self.snapshot() + limit = self.db("Library").stat().st_size + 65536 + + def full_disk(): + signal.signal(signal.SIGXFSZ, signal.SIG_IGN) + resource.setrlimit(resource.RLIMIT_FSIZE, (limit, limit)) + + for operation, preexec_fn, generated in (("clash", None, "Dynamic.itdb"), + ("grow", full_disk, "Library.itdb")): + with self.subTest(operation): + result = self.run_helper(operation, success=False, preexec_fn=preexec_fn) + self.assertIn(f"during {generated} generation", result.stderr) + self.assertEqual(self.snapshot(), library) + + def test_nano_5g_keeps_libgpod_layout(self): + # SQLite with HASH72 instead of HashAB: none of the nano 6G/7G layout. + self.setup_synthetic("MC027", 4, self.commands) + (self.control / "Device/HashInfo").write_bytes( + b"HASHv0" + bytes.fromhex("0123456789ABCDEF").ljust(20, b"\0") + bytes(range(28))) + self.run_helper("create") + library = self.db("Library") + self.assertIn("date_backed_up", columns(library, "item")) + for table, names in APPLE_COLUMNS.items(): + self.assertFalse(names & columns(library, table), table) + self.assertEqual(rows(library, "PRAGMA user_version"), [(0,)]) + self.assertEqual(rows(self.db("Genius"), "PRAGMA user_version"), [(0,)]) + self.assertEqual(rows(library, "SELECT major, minor FROM version_info"), [(2, 48)]) def test_invalid_or_ios_guid_is_rejected(self): for guid in ("", "0123456789ABCDEG", "0123456789ABCDE", "01" * 20): diff --git a/tools/prebuild/libgpod-nano.patch b/tools/prebuild/libgpod-nano.patch index cc8b46ba..5a293300 100644 --- a/tools/prebuild/libgpod-nano.patch +++ b/tools/prebuild/libgpod-nano.patch @@ -72,6 +72,590 @@ return TRUE; } +--- a/src/itdb_device.c ++++ b/src/itdb_device.c +@@ -1413,6 +1413,16 @@ itdb_device_supports_sqlite_db (const Itdb_Device *device) + } + } + ++/* iPod nano 6G/7G: the firmware browses the HashAB-signed SQLite library ++ and expects it in the layout iTunes writes */ ++G_GNUC_INTERNAL gboolean ++itdb_device_is_hashab_nano (const Itdb_Device *device) ++{ ++ return itdb_device_supports_sqlite_db (device) ++ && !itdb_device_is_iphone_family (device) ++ && itdb_device_get_checksum_type (device) == ITDB_CHECKSUM_HASHAB; ++} ++ + G_GNUC_INTERNAL gboolean + itdb_device_supports_compressed_itunesdb (const Itdb_Device *device) + { +--- a/src/itdb_device.h ++++ b/src/itdb_device.h +@@ -176,6 +176,7 @@ G_GNUC_INTERNAL const char *itdb_device_get_firewire_id (const Itdb_Device *devi + G_GNUC_INTERNAL gboolean itdb_device_supports_sparse_artwork (const Itdb_Device *device); + G_GNUC_INTERNAL gboolean itdb_device_supports_compressed_itunesdb (const Itdb_Device *device); + G_GNUC_INTERNAL gboolean itdb_device_supports_sqlite_db (const Itdb_Device *device); ++G_GNUC_INTERNAL gboolean itdb_device_is_hashab_nano (const Itdb_Device *device); + G_GNUC_INTERNAL gboolean itdb_device_get_storage_info (Itdb_Device *device, guint64 *capacity, guint64 *free); + G_GNUC_INTERNAL gboolean itdb_device_write_checksum (Itdb_Device *device, + unsigned char *itdb_data, +--- a/src/itdb_sqlite_queries.h ++++ b/src/itdb_sqlite_queries.h +@@ -77,6 +77,22 @@ static const char Library_create[] = + "END;" \ + "COMMIT;"; + ++/** columns iTunes' 'Library.itdb' has in addition, for iPod nano 6G/7G; ++ * the device's post-process commands add is_home_video */ ++static const char Library_nano_columns[] = ++ "ALTER TABLE item ADD COLUMN is_tone INTEGER DEFAULT 0;" \ ++ "ALTER TABLE item ADD COLUMN is_book INTEGER DEFAULT 0;" \ ++ "ALTER TABLE item ADD COLUMN is_itunes_u INTEGER DEFAULT 0;" \ ++ "ALTER TABLE item ADD COLUMN is_digital_booklet INTEGER DEFAULT 0;" \ ++ "ALTER TABLE item ADD COLUMN chosen_by_auto_fill INTEGER DEFAULT 0;" \ ++ "ALTER TABLE item ADD COLUMN collection_description TEXT;" \ ++ "ALTER TABLE item ADD COLUMN copyright TEXT;" \ ++ "ALTER TABLE avformat_info ADD COLUMN channels INTEGER DEFAULT 0;" \ ++ "ALTER TABLE video_info ADD COLUMN is_hd INTEGER;" \ ++ "ALTER TABLE store_info ADD COLUMN xid TEXT;" \ ++ "ALTER TABLE store_info ADD COLUMN flavor TEXT;" \ ++ "ALTER TABLE container ADD COLUMN smart_evaluation_order INTEGER;"; ++ + /** creation statement for 'Locations.itdb' */ + static const char Locations_create[] = + "BEGIN TRANSACTION;" \ +--- a/src/itdb_sqlite.c ++++ b/src/itdb_sqlite.c +@@ -190,6 +190,7 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) + struct stat fst; + int idx = 0; + GList *gl = NULL; ++ gboolean nano = itdb_device_is_hashab_nano(itdb->device); + + dbf = g_build_filename(outpath, "Dynamic.itdb", NULL); + printf("[%s] Processing '%s'\n", __func__, dbf); +@@ -227,7 +228,11 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) + goto leave; + } + +- sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); ++ if (SQLITE_OK != sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL) && nano) { ++ fprintf(stderr, "[%s] BEGIN failed: %s\n", __func__, sqlite3_errmsg(db)); ++ res = -1; ++ goto leave; ++ } + + if (itdb->tracks) { + printf("[%s] - processing %d tracks\n", __func__, g_list_length(itdb->tracks)); +@@ -277,6 +282,8 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) + /* expected result */ + } else { + fprintf(stderr, "[%s] 1 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + } + if (stmt) { +@@ -333,19 +340,23 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) + /* expected result */ + } else { + fprintf(stderr, "[%s] 2 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + } + +- sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL); +- +- if (stmt) { +- sqlite3_finalize(stmt); +- stmt = NULL; ++ if (SQLITE_OK != sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL) && nano) { ++ fprintf(stderr, "[%s] COMMIT failed: %s\n", __func__, sqlite3_errmsg(db)); ++ res = -1; ++ goto leave; + } + + res = 0; + printf("[%s] done.\n", __func__); + leave: ++ if (stmt) { ++ sqlite3_finalize(stmt); ++ } + if (db) { + sqlite3_close(db); + } +@@ -363,6 +374,7 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) + sqlite3 *db = NULL; + sqlite3_stmt *stmt_chapter = NULL; + char *errmsg = NULL; ++ gboolean nano = itdb_device_is_hashab_nano(itdb->device); + struct stat fst; + GList *gl = NULL; + +@@ -399,7 +411,11 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) + } + } + +- sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); ++ if (SQLITE_OK != sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL) && nano) { ++ fprintf(stderr, "[%s] BEGIN failed: %s\n", __func__, sqlite3_errmsg(db)); ++ res = -1; ++ goto leave; ++ } + if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"chapter\" VALUES(?,?);", -1, &stmt_chapter, NULL)) { + fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); + goto leave; +@@ -450,9 +466,15 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) + fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); + } + g_byte_array_free(chapter_blob, TRUE); ++ if (res != SQLITE_DONE && nano) ++ goto leave; + } + } +- sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL); ++ if (SQLITE_OK != sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL) && nano) { ++ fprintf(stderr, "[%s] COMMIT failed: %s\n", __func__, sqlite3_errmsg(db)); ++ res = -1; ++ goto leave; ++ } + + res = 0; + printf("[%s] done.\n", __func__); +@@ -510,6 +532,12 @@ static int mk_Genius(Itdb_iTunesDB *itdb, const char *outpath) + goto leave; + } + } ++ /* iTunes' schema level of Genius.itdb on nano 6G/7G */ ++ if (itdb_device_is_hashab_nano(itdb->device) ++ && SQLITE_OK != sqlite3_exec(db, "PRAGMA user_version = 2;", NULL, NULL, NULL)) { ++ fprintf(stderr, "[%s] could not set user_version: %s\n", __func__, sqlite3_errmsg(db)); ++ goto leave; ++ } + + sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); + +@@ -615,6 +643,7 @@ static int mk_Library(Itdb_iTunesDB *itdb, + GHashTable *genre_map = NULL; + GHashTable **orders = NULL; + guint32 genre_index; ++ gboolean nano = itdb_device_is_hashab_nano(itdb->device); + printf("library_persistent_id = 0x%016"G_GINT64_MODIFIER"x\n", itdb->priv->pid); + + dbf = g_build_filename(outpath, "Library.itdb", NULL); +@@ -652,8 +681,16 @@ static int mk_Library(Itdb_iTunesDB *itdb, + } + goto leave; + } ++ if (nano && SQLITE_OK != sqlite3_exec(db, Library_nano_columns, NULL, NULL, NULL)) { ++ fprintf(stderr, "[%s] could not add columns: %s\n", __func__, sqlite3_errmsg(db)); ++ goto leave; ++ } + +- sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); ++ if (SQLITE_OK != sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL) && nano) { ++ fprintf(stderr, "[%s] BEGIN failed: %s\n", __func__, sqlite3_errmsg(db)); ++ res = -1; ++ goto leave; ++ } + + fprintf(stderr, "[%s] compiling SQL statements\n", __func__); + if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"version_info\" VALUES(?,?,?,?,?,?,?);", -1, &stmt_version_info, NULL)) { +@@ -665,7 +702,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + goto leave; + } + if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"container\" " +- "VALUES(?,?,?,?,:name,?,?,?,?,?,?,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);", -1, &stmt_container, NULL)) { ++ "(pid,distinguished_kind,date_created,date_modified,name,name_order,parent_pid,media_kinds," ++ "workout_template_id,is_hidden,smart_is_folder) VALUES(?,?,?,?,:name,?,?,?,?,?,?);", -1, &stmt_container, NULL)) { + fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); + goto leave; + } +@@ -688,7 +726,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, + fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); + goto leave; + } +- if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"avformat_info\" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?);", -1, &stmt_avformat_info, NULL)) { ++ if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"avformat_info\" " ++ "(item_pid,sub_id,audio_format,bit_rate,sample_rate,duration,gapless_heuristic_info,gapless_encoding_delay," ++ "gapless_encoding_drain,gapless_last_frame_resynch,analysis_inhibit_flags,audio_fingerprint," ++ "volume_normalization_energy) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?);", -1, &stmt_avformat_info, NULL)) { + fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); + goto leave; + } +@@ -708,7 +749,12 @@ static int mk_Library(Itdb_iTunesDB *itdb, + fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); + goto leave; + } +- if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"video_info\" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", -1, &stmt_video_info, NULL)) { ++ if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"video_info\" " ++ "(item_pid,has_alternate_audio,has_subtitles,characteristics_valid,has_closed_captions,is_self_contained," ++ "is_compressed,is_anamorphic,season_number,audio_language,audio_track_index,audio_track_id," ++ "subtitle_language,subtitle_track_index,subtitle_track_id,series_name,sort_series_name,episode_id," ++ "episode_sort_id,network_name,extended_content_rating,movie_info) " ++ "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", -1, &stmt_video_info, NULL)) { + fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); + goto leave; + } +@@ -721,9 +767,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* major */ + /* FIXME: the versioning scheme needs to be updated in libgpod using */ + /* major+minor version, this is a workaround */ +- sqlite3_bind_int(stmt_version_info, ++idx, (itdb->version >= 0x28) ? 2 : 1); ++ /* nano 6G/7G: what iTunes writes (1.117), whatever the iTunesDB version */ ++ sqlite3_bind_int(stmt_version_info, ++idx, (nano || itdb->version < 0x28) ? 1 : 2); + /* minor */ +- sqlite3_bind_int(stmt_version_info, ++idx, itdb->version); ++ sqlite3_bind_int(stmt_version_info, ++idx, nano ? 117 : itdb->version); + /* compatibility */ + /* This has an unknown meaning. use 0. */ + sqlite3_bind_int(stmt_version_info, ++idx, 0); +@@ -740,6 +787,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 2 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + + printf("[%s] - inserting into \"genre_map\"\n", __func__); +@@ -809,6 +858,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + + types |= track->mediatype; +@@ -863,6 +914,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 4 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + } + +@@ -911,6 +964,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 3 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + + /* for each track: */ +@@ -1088,6 +1143,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 5 sqlite3_step returned %d: %s\n", __func__, res, sqlite3_errmsg(db)); ++ if (nano) ++ goto leave; + } + + /* printf("[%s] -- inserting into \"avformat_info\"\n", __func__); */ +@@ -1144,6 +1201,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 7 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + + /* this is done by a trigger, so we don't need to do this :-D */ +@@ -1191,6 +1250,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + } + +@@ -1227,6 +1288,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + } + +@@ -1252,6 +1315,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + } + /* if it's a movie, music video or tv show */ +@@ -1314,13 +1379,19 @@ static int mk_Library(Itdb_iTunesDB *itdb, + /* expected result */ + } else { + fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + } + } + + sqlite3_exec(db, "UPDATE album SET artwork_item_pid = (SELECT item.pid FROM item WHERE item.artwork_cache_id != 0 AND item.album_pid = album.pid LIMIT 1);", NULL, NULL, NULL); + +- sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL); ++ if (SQLITE_OK != sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL) && nano) { ++ fprintf(stderr, "[%s] COMMIT failed: %s\n", __func__, sqlite3_errmsg(db)); ++ res = -1; ++ goto leave; ++ } + + res = 0; + printf("[%s] done.\n", __func__); +@@ -1385,6 +1456,7 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu + sqlite3_stmt *stmt = NULL; + char *errmsg = NULL; + int idx = 0; ++ gboolean nano = itdb_device_is_hashab_nano(itdb->device); + + dbf = g_build_filename(outpath, "Locations.itdb", NULL); + printf("[%s] Processing '%s'\n", __func__, dbf); +@@ -1415,7 +1487,11 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu + goto leave; + } + +- sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); ++ if (SQLITE_OK != sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL) && nano) { ++ fprintf(stderr, "[%s] BEGIN failed: %s\n", __func__, sqlite3_errmsg(db)); ++ res = -1; ++ goto leave; ++ } + + if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"base_location\" (id, path) VALUES (?,?);", -1, &stmt, NULL)) { + fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); +@@ -1438,6 +1514,8 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu + /* expected result */ + } else { + fprintf(stderr, "[%s] sqlite3_step returned %d\n", __func__, res); ++ if (nano) ++ goto leave; + } + + if (itdb_device_is_iphone_family(itdb->device)) { +@@ -1489,7 +1567,6 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu + int i = 0; + int cnt = 0; + int pos = 0; +- int res; + + if (!track->ipod_path) { + continue; +@@ -1556,12 +1633,18 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu + free(ipod_path); + ipod_path = NULL; + } ++ if (res != SQLITE_DONE && nano) ++ goto leave; + } + } else { + printf("[%s] No tracks available, none written.\n", __func__); + } + +- sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL); ++ if (SQLITE_OK != sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL) && nano) { ++ fprintf(stderr, "[%s] COMMIT failed: %s\n", __func__, sqlite3_errmsg(db)); ++ res = -1; ++ goto leave; ++ } + + res = 0; + printf("[%s] done.\n", __func__); +@@ -1752,7 +1835,28 @@ static void sqlite_func_iphone_sort_section(sqlite3_context *context, int argc, + sqlite3_result_int(context, res); + } + +-static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, const char *uuid) ++/* Set 25 adds two columns a second time: libgpod's item_stats has ++ rental_expired already, and AddGenreMapCompilationCount is listed ++ twice. Only the repeats are skipped. */ ++static gboolean is_repeated_column(sqlite3 *db, const char *key, gboolean genre_count_added) ++{ ++ sqlite3_stmt *stmt = NULL; ++ gboolean present; ++ ++ if (strcmp(key, "AddGenreMapCompilationCount") == 0) ++ return genre_count_added; ++ if (strcmp(key, "CreateRentalExpiredColumn") != 0) ++ return FALSE; ++ present = sqlite3_prepare_v2(db, "SELECT rental_expired FROM item_stats;", -1, &stmt, NULL) == SQLITE_OK; ++ sqlite3_finalize(stmt); ++ return present; ++} ++ ++/* iPod nano 6G/7G need the commands to reach the schema level their ++ firmware expects: there a missing or failing command fails the write, ++ other devices only log it. */ ++static int run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, const char *uuid, ++ GError **error) + { + plist_t plist_node = NULL; + plist_t ppc_dict = NULL; +@@ -1760,6 +1864,8 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, + const gchar *otherdbs[] = {"Dynamic.itdb", "Extras.itdb", "Genius.itdb", "Locations.itdb", NULL}; + int res; + sqlite3 *db = NULL; ++ gboolean nano = itdb_device_is_hashab_nano(itdb->device); ++ gchar *failure = NULL; + + #ifdef HAVE_LIBIMOBILEDEVICE + if (itdb_device_is_iphone_family(itdb->device)) { +@@ -1846,7 +1952,9 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, + plist_dict_next_item(curnode, iter, &key, &subnode); + while (subnode) { + unsigned long int intval = strtoul(key, NULL, 0); +- if ((intval > 0) && (intval > maxver)) { ++ /* nano 6G/7G: set 25 fits libgpod's schema plus ++ Library_nano_columns and raises it to iTunes' level 26 */ ++ if (nano ? (intval == 25) : ((intval > 0) && (intval > maxver))) { + user_ver_cmds = subnode; + maxver = intval; + } +@@ -1874,6 +1982,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, + gchar *attach_str = NULL; + char *errmsg = NULL; + guint32 i = 0, cnt = 0, ok_cnt = 0; ++ gboolean genre_count_added = FALSE; + + plist_dict_new_iter(sql_cmds, &iter); + if (iter) { +@@ -1900,6 +2009,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, + + if (SQLITE_OK != sqlite3_open((const char*)dbf, &db)) { + fprintf(stderr, "Error opening database '%s': %s\n", dbf, sqlite3_errmsg(db)); ++ failure = g_strdup_printf("cannot open '%s'", dbf); + g_free(dbf); + goto leave; + } +@@ -1910,12 +2020,14 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, + while (otherdbs[i]) { + errmsg = NULL; + dbf = g_build_filename(outpath, otherdbs[i], NULL); +- attach_str = g_strdup_printf("ATTACH DATABASE '%s' AS '%s';", dbf, otherdbs[i]); ++ attach_str = sqlite3_mprintf("ATTACH DATABASE %Q AS %Q;", dbf, otherdbs[i]); + g_free(dbf); + res = sqlite3_exec(db, attach_str, NULL, NULL, &errmsg); +- g_free(attach_str); ++ sqlite3_free(attach_str); + if (res != SQLITE_OK) { + printf("[%s] WARNING: Could not attach database '%s': %s\n", __func__, otherdbs[i], errmsg); ++ if (!failure) ++ failure = g_strdup_printf("cannot attach '%s': %s", otherdbs[i], errmsg); + } + if (errmsg) { + free(errmsg); +@@ -1930,28 +2042,39 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, + cnt = plist_array_get_size(user_ver_cmds); + printf("[%s] Running %d post process commands now\n", __func__, cnt); + +- sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); +- for (i=0; i Date: Tue, 27 Oct 2026 20:16:59 +0100 Subject: [PATCH 08/21] feat(libgpod): write the iPod nano 6G/7G firmware categories to SQLite The nano 6G/7G firmware browses six hidden smart containers: Music, Videos, Movies, TV Shows, Audiobooks and Rentals. iTunes keeps them as dataset 5 playlists in the iTunesCDB and as container rows in Library.itdb. libgpod parsed and rewrote the dataset 5 playlists but wrote no SQLite rows for them. On devices that sign with HashAB and are not iOS devices: - Dataset 5 headers keep +0x52, the category kind that SQLite stores as distinguished_kind, and +0x54 as read. libgpod copied +0x50 into +0x52, which turns the Videos category (+0x50 0, +0x52 1) into an ordinary playlist for iTunes. - Library.itdb gets one hidden, live smart container per category, with iTunes' smart settings and the rules as smart_criteria, serialised by the existing mhod 51 writer. The firmware evaluates the rules itself, so no members are written; Music gets the master playlist's media kinds, as in iTunes. Dynamic.itdb gets their container_ui rows. - A library that lacks a category, such as a new one, gets it with iTunes' rules. - The Rentals rule has an action libgpod cannot classify; it is written as read instead of passing through itdb_splr_validate(), which only warned about it. The roundtrip checks the rows and rules in both files, that they stay the same over resaves, that missing categories are added while the others keep their ids, Music's media kinds with audio and a movie, that the three header fields round-trip independently, and that a nano 5G still writes dataset 5 as before. --- .changeset/nano-firmware-categories.md | 5 + tools/nano7/README.md | 7 +- tools/nano7/libgpod-roundtrip.c | 28 +- tools/nano7/test_libgpod.py | 186 +++++++++++- tools/prebuild/libgpod-nano.patch | 385 ++++++++++++++++++++++--- 5 files changed, 561 insertions(+), 50 deletions(-) create mode 100644 .changeset/nano-firmware-categories.md diff --git a/.changeset/nano-firmware-categories.md b/.changeset/nano-firmware-categories.md new file mode 100644 index 00000000..63d9e187 --- /dev/null +++ b/.changeset/nano-firmware-categories.md @@ -0,0 +1,5 @@ +--- +'@podkit/libgpod-node': patch +--- + +For the iPod nano 6th and 7th generation, libgpod now writes the hidden category containers (Music, Videos, …) and iTunes' database schema level that the nano 7th generation firmware needs to show a library, and keeps the Videos category's identity in the `iTunesCDB`. Such a write fails, instead of leaving a library the device cannot browse, when the device's SQLite post-process commands are missing or fail or when a database cannot be written completely. diff --git a/tools/nano7/README.md b/tools/nano7/README.md index 44d3930f..837405ac 100644 --- a/tools/nano7/README.md +++ b/tools/nano7/README.md @@ -42,8 +42,11 @@ synthetic capabilities. It checks the integrity of the five SQLite databases, both signatures, rejection of malformed or iOS-sized GUIDs, and an unsigned `iTunesDB` for a 5th generation iPod. -For the nano 6th and 7th generation the checks also cover iTunes' schema -level: its additional columns, `user_version` 26 (`Library.itdb`) and 2 +For the nano 6th and 7th generation the checks also cover the six hidden +category containers the firmware browses (Music, Videos, Movies, TV Shows, +Audiobooks, Rentals) in `Library.itdb` and in the `iTunesCDB` dataset 5, +whose header fields are kept as read, and iTunes' schema level: its +additional columns, `user_version` 26 (`Library.itdb`) and 2 (`Genius.itdb`) and the `version_info` row. The device's SQLite post-process command set 25 completes that schema. A missing set, a missing or failing command, or an insert or commit that fails while the databases are generated diff --git a/tools/nano7/libgpod-roundtrip.c b/tools/nano7/libgpod-roundtrip.c index 11a36f24..df766a35 100644 --- a/tools/nano7/libgpod-roundtrip.c +++ b/tools/nano7/libgpod-roundtrip.c @@ -1,6 +1,7 @@ -/* Create, resave or delete one synthetic track with libgpod, give it a 1 MiB - * comment (grow) or the second playlist the master playlist's id (clash). - * Usage: libgpod-roundtrip create|resave|delete|grow|clash */ +/* Create, resave or delete one synthetic track with libgpod, add a second + * one, a movie (add), give the first a 1 MiB comment (grow) or the second + * playlist the master playlist's id (clash). + * Usage: libgpod-roundtrip create|resave|delete|add|grow|clash */ #include #include @@ -43,16 +44,27 @@ int main(int argc, char **argv) g_printerr("parse: %s\n", error ? error->message : "unknown error"); return 4; } - if (g_list_length(db->tracks) != 1) + GList *item; + for (item = db->tracks; item; item = item->next) + if (strcmp(((Itdb_Track *)item->data)->title, "Synthetic track") == 0) + break; + if (!item) return 5; - track = db->tracks->data; - if (strcmp(track->title, "Synthetic track") != 0) - return 6; + track = item->data; if (strcmp(argv[2], "delete") == 0) { - GList *item; for (item = db->playlists; item; item = item->next) itdb_playlist_remove_track(item->data, track); itdb_track_remove(track); + } else if (strcmp(argv[2], "add") == 0) { + track = itdb_track_new(); + track->title = g_strdup("Second track"); + track->filetype = g_strdup("MPEG-4 video file"); + track->ipod_path = g_strdup(":iPod_Control:Music:F00:TEST2.m4v"); + track->mediatype = ITDB_MEDIATYPE_MOVIE; + track->size = 32000; + track->transferred = TRUE; + itdb_track_add(db, track, -1); + itdb_playlist_add_track(itdb_playlist_mpl(db), track, -1); } else if (strcmp(argv[2], "grow") == 0) { g_free(track->comment); track->comment = g_strnfill(1 << 20, 'x'); diff --git a/tools/nano7/test_libgpod.py b/tools/nano7/test_libgpod.py index c122c452..ea2ef8ef 100644 --- a/tools/nano7/test_libgpod.py +++ b/tools/nano7/test_libgpod.py @@ -11,12 +11,23 @@ import subprocess import tempfile import unittest +import zlib from build import build, ROOT from probe import HashAB, guid_bytes, verify_cbk, verify_cdb CAPTURE = ROOT.parents[1] / "docs/sysinfo-captures/nano-7g-16gb-usb.xml" POST_PROCESS = "com.apple.mobile.iTunes.SQLMusicLibraryPostProcessCommands" +# Firmware categories as iTunes writes them for a nano 7G: distinguished_kind +# (mhyp +0x52) -> (+0x50, rule 1 mask, rule 2 action, rule 2 mask). +CATEGORIES = { + 4: (4, 0x1021B1, 0x02000400, 0x208004), + 1: (0, 0x000C42, 0x02000400, 0x20A004), + 2: (2, 0x000402, 0x02000400, 0x20A004), + 3: (3, 0x000040, 0x02000400, 0x20A004), + 5: (5, 0x000008, 0x02000400, 0x20A004), + 7: (7, 0x000042, 0x00000800, 0x008000), +} APPLE_COLUMNS = { "item": {"is_home_video", "is_tone", "is_book", "is_itunes_u", "is_digital_booklet", "chosen_by_auto_fill", "collection_description", "copyright"}, @@ -36,6 +47,16 @@ def columns(path, table): return {row[1] for row in rows(path, f"PRAGMA table_info({table})")} +def category_slst(kind): + """The SLst payload (big-endian) of a category's two media kind rules.""" + _, mask1, action2, mask2 = CATEGORIES[kind] + out = b"SLst" + struct.pack(">III", 0x00010001, 2, 0) + bytes(120) + for action, mask in ((0x400, mask1), (action2, mask2)): + out += struct.pack(">II", 0x3C, action) + bytes(44) + struct.pack(">I", 0x44) + out += struct.pack(">QQQQQQ", mask, 0, 1, mask, 0, 1) + bytes(20) + return out + + def post_process_commands(): # The capture is not strictly valid XML elsewhere; take only this dictionary. text = CAPTURE.read_text() @@ -49,6 +70,59 @@ def post_process_commands(): return plistlib.loads(f'{body}'.encode()) +def read_cdb(path): + """(header, uncompressed database) of an iTunesCDB or iTunesDB.""" + data = path.read_bytes() + header = struct.unpack_from("sortorder = get32lint (cts, mhyp_seek+44); + if (header_len >= 0x6C) { + plitem->priv->mhsd5_type = get16lint (cts, mhyp_seek+0x50); ++ plitem->priv->distinguished_kind = get16lint (cts, mhyp_seek+0x52); ++ plitem->priv->unk_0x54 = get32lint (cts, mhyp_seek+0x54); + } + + mhod_seek = mhyp_seek + header_len; +@@ -4640,7 +4642,12 @@ static void mk_mhod (FExport *fexp, MHODData *mhod) + g_return_if_fail (splr); + ft = itdb_splr_get_field_type (splr); + /* printf ("%p: field: %d ft: %d\n", splr, splr->field, ft);*/ +- itdb_splr_validate (splr); ++ /* the nano 6G/7G Rentals category has a rule libgpod cannot ++ classify; it is written as read */ ++ if (!(itdb_device_is_hashab_nano (fexp->itdb->device) ++ && splr->field == ITDB_SPLFIELD_VIDEO_KIND ++ && splr->action == ITDB_SPLACTION_BINARY_UNKNOWN1)) ++ itdb_splr_validate (splr); + put32bint (cts, splr->field); + put32bint (cts, splr->action); + put32_n0 (cts, 11); /* unknown */ +@@ -5529,13 +5536,20 @@ static gboolean write_playlist (FExport *fexp, + put32_n0 (cts, 8); /* ? */ + /* +0x50 */ + put16lint (cts, pl->priv->mhsd5_type); +- put16lint (cts, pl->priv->mhsd5_type); /* same as 0x50 ? */ +- /* +0x54 */ +- if ((pl->priv->mhsd5_type == ITDB_PLAYLIST_MHSD5_MOVIE_RENTALS) +- || (pl->priv->mhsd5_type == ITDB_PLAYLIST_MHSD5_RINGTONES)){ +- put32lint (cts, 1); /* unknown, 1 for Movie rentals + Ringtones */ ++ if (itdb_device_is_hashab_nano (fexp->itdb->device)) { ++ /* +0x52 is the category kind, which is not +0x50 for Videos */ ++ put16lint (cts, pl->priv->distinguished_kind); ++ /* +0x54 */ ++ put32lint (cts, pl->priv->unk_0x54); + } else { +- put32lint (cts, 0); /* 0 otherwise */ ++ put16lint (cts, pl->priv->mhsd5_type); /* same as 0x50 ? */ ++ /* +0x54 */ ++ if ((pl->priv->mhsd5_type == ITDB_PLAYLIST_MHSD5_MOVIE_RENTALS) ++ || (pl->priv->mhsd5_type == ITDB_PLAYLIST_MHSD5_RINGTONES)){ ++ put32lint (cts, 1); /* unknown, 1 for Movie rentals + Ringtones */ ++ } else { ++ put32lint (cts, 0); /* 0 otherwise */ ++ } + } + put32_n0 (cts, 5); /* ? */ + +@@ -5991,6 +6005,8 @@ static gboolean itdb_write_file_internal (Itdb_iTunesDB *itdb, + g_return_val_if_fail (itdb->device, FALSE); + g_return_val_if_fail (filename || itdb->filename, FALSE); + ++ itdb_playlist_add_missing_categories (itdb); ++ + if (!filename) filename = itdb->filename; + + /* set endianess flag */ +@@ -8254,6 +8270,28 @@ G_GNUC_INTERNAL GByteArray *itdb_chapterdata_build_chapter_blob(Itdb_Chapterdata + return chapter_blob; + } + ++/* The SLst payload of mhod 51 (without the mhod header), which the SQLite ++ library stores as container.smart_criteria */ ++G_GNUC_INTERNAL GByteArray *itdb_splrules_build_blob (Itdb_iTunesDB *itdb, ++ Itdb_SPLRules *splrules) ++{ ++ FExport fexp = { 0 }; ++ MHODData mhod = { 0 }; ++ GByteArray *blob; ++ ++ fexp.itdb = itdb; ++ fexp.wcontents = wcontents_new (""); ++ mhod.valid = TRUE; ++ mhod.type = MHOD_ID_SPLRULES; ++ mhod.data.splrules = splrules; ++ mk_mhod (&fexp, &mhod); ++ blob = g_byte_array_sized_new (fexp.wcontents->pos - 24); ++ g_byte_array_append (blob, (guint8 *)fexp.wcontents->contents + 24, ++ fexp.wcontents->pos - 24); ++ wcontents_free (fexp.wcontents); ++ return blob; ++} ++ + + /*------------------------------------------------------------------*\ + * * +--- a/src/itdb_playlist.c ++++ b/src/itdb_playlist.c +@@ -1355,6 +1355,74 @@ void itdb_playlist_add_mhsd5_playlist (Itdb_iTunesDB *itdb, Itdb_Playlist *pl, + itdb_playlist_add_internal (itdb, pl, pos, &itdb->priv->mhsd5_playlists); + } + ++/* The hidden smart playlists in mhsd type 5 that an iPod nano 6G/7G ++ firmware browses as Music, Videos, ... (iTunes' rules: the first selects ++ media kinds, the second excludes some) */ ++static const struct { ++ guint16 kind; /* mhyp +0x52 */ ++ Itdb_Playlist_Mhsd5_Type mhsd5_type; ++ guint32 sortorder; ++ const gchar *name; ++ guint64 mask1; ++ guint32 action2; ++ guint64 mask2; ++} categories[] = { ++ {4, ITDB_PLAYLIST_MHSD5_MUSIC, 5, "Music", 0x1021B1, ++ ITDB_SPLACTION_NOT_BINARY_AND, 0x208004}, ++ {1, ITDB_PLAYLIST_MHSD5_NONE, 1, "Videos", 0x000C42, ++ ITDB_SPLACTION_NOT_BINARY_AND, 0x20A004}, ++ {2, ITDB_PLAYLIST_MHSD5_MOVIES, 1, "Movies", 0x000402, ++ ITDB_SPLACTION_NOT_BINARY_AND, 0x20A004}, ++ {3, ITDB_PLAYLIST_MHSD5_TV_SHOWS, 1, "TV Shows", 0x000040, ++ ITDB_SPLACTION_NOT_BINARY_AND, 0x20A004}, ++ {5, ITDB_PLAYLIST_MHSD5_AUDIOBOOKS, 1, "Audiobooks", 0x000008, ++ ITDB_SPLACTION_NOT_BINARY_AND, 0x20A004}, ++ {7, ITDB_PLAYLIST_MHSD5_MOVIE_RENTALS, 1, "Rentals", 0x000042, ++ ITDB_SPLACTION_BINARY_UNKNOWN1, 0x008000}, ++}; ++ ++/* Add the categories a nano 6G/7G library lacks, e.g. a new one. */ ++G_GNUC_INTERNAL void itdb_playlist_add_missing_categories (Itdb_iTunesDB *itdb) ++{ ++ guint i, j; ++ ++ if (!itdb_device_is_hashab_nano (itdb->device)) ++ return; ++ for (i = 0; i < G_N_ELEMENTS (categories); i++) { ++ Itdb_Playlist *pl; ++ GList *gl; ++ ++ for (gl = itdb->priv->mhsd5_playlists; gl; gl = gl->next) { ++ pl = gl->data; ++ if (pl->priv->distinguished_kind == categories[i].kind) ++ break; ++ } ++ if (gl) ++ continue; ++ pl = itdb_playlist_new (categories[i].name, FALSE); ++ pl->is_spl = TRUE; ++ pl->sortorder = categories[i].sortorder; ++ pl->splpref.liveupdate = TRUE; ++ pl->splpref.checkrules = TRUE; ++ pl->splpref.limittype = ITDB_LIMITTYPE_SONGS; ++ pl->splpref.limitsort = ITDB_LIMITSORT_RANDOM; ++ pl->splpref.limitvalue = 25; ++ pl->splrules.unk004 = 0x00010001; ++ pl->splrules.match_operator = ITDB_SPLMATCH_AND; ++ for (j = 0; j < 2; j++) { ++ Itdb_SPLRule *splr = g_new0 (Itdb_SPLRule, 1); ++ splr->field = ITDB_SPLFIELD_VIDEO_KIND; ++ splr->action = j ? categories[i].action2 : ITDB_SPLACTION_BINARY_AND; ++ splr->fromvalue = splr->tovalue = j ? categories[i].mask2 : categories[i].mask1; ++ splr->fromunits = splr->tounits = 1; ++ pl->splrules.rules = g_list_append (pl->splrules.rules, splr); ++ } ++ pl->priv->mhsd5_type = categories[i].mhsd5_type; ++ pl->priv->distinguished_kind = categories[i].kind; ++ itdb_playlist_add_mhsd5_playlist (itdb, pl, -1); ++ } ++} ++ + /** + * itdb_playlist_add: + * @itdb: an #Itdb_iTunesDB +--- a/src/itdb_private.h ++++ b/src/itdb_private.h +@@ -210,11 +210,18 @@ struct _Itdb_Track_Private { + + struct _Itdb_Playlist_Private { + Itdb_Playlist_Mhsd5_Type mhsd5_type; ++ /* mhyp +0x52 and +0x54 of a dataset 5 playlist. +0x52 is the SQLite ++ container.distinguished_kind, which differs from +0x50 for Videos. */ ++ guint16 distinguished_kind; ++ guint32 unk_0x54; + }; + + G_GNUC_INTERNAL void itdb_playlist_add_mhsd5_playlist(Itdb_iTunesDB *itdb, + Itdb_Playlist *pl, + gint32 pos); ++G_GNUC_INTERNAL void itdb_playlist_add_missing_categories (Itdb_iTunesDB *itdb); ++G_GNUC_INTERNAL GByteArray *itdb_splrules_build_blob (Itdb_iTunesDB *itdb, ++ Itdb_SPLRules *splrules); + G_GNUC_INTERNAL gboolean itdb_spl_action_known (ItdbSPLAction action); + G_GNUC_INTERNAL void itdb_splr_free (Itdb_SPLRule *splr); + G_GNUC_INTERNAL const gchar *itdb_photodb_get_mountpoint (Itdb_PhotoDB *photodb); --- a/src/itdb_sqlite_queries.h +++ b/src/itdb_sqlite_queries.h @@ -77,6 +77,22 @@ static const char Library_create[] = @@ -128,15 +316,42 @@ "BEGIN TRANSACTION;" \ --- a/src/itdb_sqlite.c +++ b/src/itdb_sqlite.c -@@ -190,6 +190,7 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -180,6 +180,25 @@ static void destroy_key_orders(GHashTable **orders) + g_free(orders); + } + ++/* iPod nano 6G/7G: the dataset 5 playlists that are the firmware's ++ categories, stored as hidden smart containers */ ++static GList *category_containers(Itdb_iTunesDB *itdb) ++{ ++ GList *categories = NULL; ++ GList *gl; ++ ++ if (!itdb_device_is_hashab_nano(itdb->device)) ++ return NULL; ++ for (gl = itdb->priv->mhsd5_playlists; gl; gl = gl->next) { ++ Itdb_Playlist *pl = gl->data; ++ guint16 kind = pl->priv->distinguished_kind; ++ /* Videos 1, Movies 2, TV Shows 3, Music 4, Audiobooks 5, Rentals 7 */ ++ if (kind < 8 && ((1 << kind) & 0xBE)) ++ categories = g_list_append(categories, pl); ++ } ++ return categories; ++} ++ + static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) + { + int res = -1; +@@ -190,6 +209,8 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) struct stat fst; int idx = 0; GList *gl = NULL; ++ GList *containers = NULL; + gboolean nano = itdb_device_is_hashab_nano(itdb->device); dbf = g_build_filename(outpath, "Dynamic.itdb", NULL); printf("[%s] Processing '%s'\n", __func__, dbf); -@@ -227,7 +228,11 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -227,7 +248,11 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) goto leave; } @@ -149,7 +364,7 @@ if (itdb->tracks) { printf("[%s] - processing %d tracks\n", __func__, g_list_length(itdb->tracks)); -@@ -277,6 +282,8 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -277,6 +302,8 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) /* expected result */ } else { fprintf(stderr, "[%s] 1 sqlite3_step returned %d\n", __func__, res); @@ -158,7 +373,19 @@ } } if (stmt) { -@@ -333,19 +340,23 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -293,8 +320,9 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) + goto leave; + } + +- printf("[%s] - processing %d playlists\n", __func__, g_list_length(itdb->playlists)); +- for (gl = itdb->playlists; gl; gl = gl->next) { ++ containers = g_list_concat(g_list_copy(itdb->playlists), category_containers(itdb)); ++ printf("[%s] - processing %d playlists\n", __func__, g_list_length(containers)); ++ for (gl = containers; gl; gl = gl->next) { + Itdb_Playlist *pl = (Itdb_Playlist*)gl->data; + + res = sqlite3_reset(stmt); +@@ -333,19 +361,24 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) /* expected result */ } else { fprintf(stderr, "[%s] 2 sqlite3_step returned %d\n", __func__, res); @@ -181,13 +408,14 @@ res = 0; printf("[%s] done.\n", __func__); leave: ++ g_list_free(containers); + if (stmt) { + sqlite3_finalize(stmt); + } if (db) { sqlite3_close(db); } -@@ -363,6 +374,7 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) +@@ -363,6 +396,7 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) sqlite3 *db = NULL; sqlite3_stmt *stmt_chapter = NULL; char *errmsg = NULL; @@ -195,7 +423,7 @@ struct stat fst; GList *gl = NULL; -@@ -399,7 +411,11 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) +@@ -399,7 +433,11 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) } } @@ -208,7 +436,7 @@ if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"chapter\" VALUES(?,?);", -1, &stmt_chapter, NULL)) { fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; -@@ -450,9 +466,15 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) +@@ -450,9 +488,15 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); } g_byte_array_free(chapter_blob, TRUE); @@ -225,7 +453,7 @@ res = 0; printf("[%s] done.\n", __func__); -@@ -510,6 +532,12 @@ static int mk_Genius(Itdb_iTunesDB *itdb, const char *outpath) +@@ -510,6 +554,12 @@ static int mk_Genius(Itdb_iTunesDB *itdb, const char *outpath) goto leave; } } @@ -238,15 +466,27 @@ sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); -@@ -615,6 +643,7 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -606,15 +656,19 @@ static int mk_Library(Itdb_iTunesDB *itdb, + sqlite3_stmt *stmt_artist = NULL; + sqlite3_stmt *stmt_composer = NULL; + sqlite3_stmt *stmt_video_info = NULL; ++ sqlite3_stmt *stmt_category = NULL; + char *errmsg = NULL; + struct stat fst; + GList *gl = NULL; ++ GList *categories = category_containers(itdb); + Itdb_Playlist *dev_playlist = NULL; + int idx = 0; + int pos = 0; GHashTable *genre_map = NULL; GHashTable **orders = NULL; guint32 genre_index; ++ guint32 master_types = 0; + gboolean nano = itdb_device_is_hashab_nano(itdb->device); printf("library_persistent_id = 0x%016"G_GINT64_MODIFIER"x\n", itdb->priv->pid); dbf = g_build_filename(outpath, "Library.itdb", NULL); -@@ -652,8 +681,16 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -652,8 +706,16 @@ static int mk_Library(Itdb_iTunesDB *itdb, } goto leave; } @@ -264,17 +504,26 @@ fprintf(stderr, "[%s] compiling SQL statements\n", __func__); if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"version_info\" VALUES(?,?,?,?,?,?,?);", -1, &stmt_version_info, NULL)) { -@@ -665,7 +702,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -665,7 +727,17 @@ static int mk_Library(Itdb_iTunesDB *itdb, goto leave; } if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"container\" " - "VALUES(?,?,?,?,:name,?,?,?,?,?,?,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);", -1, &stmt_container, NULL)) { + "(pid,distinguished_kind,date_created,date_modified,name,name_order,parent_pid,media_kinds," + "workout_template_id,is_hidden,smart_is_folder) VALUES(?,?,?,?,:name,?,?,?,?,?,?);", -1, &stmt_container, NULL)) { ++ fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); ++ goto leave; ++ } ++ if (categories && SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"container\" " ++ "(pid,distinguished_kind,date_created,date_modified,name,name_order,parent_pid,media_kinds," ++ "workout_template_id,is_hidden,smart_is_folder,smart_is_dynamic,smart_is_filtered,smart_is_genius," ++ "smart_enabled_only,smart_is_limited,smart_limit_kind,smart_limit_order,smart_evaluation_order," ++ "smart_limit_value,smart_reverse_limit_order,smart_criteria) " ++ "VALUES(?,?,?,?,?,?,0,?,0,1,0,?,?,0,?,?,2,?,7,?,?,?);", -1, &stmt_category, NULL)) { fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } -@@ -688,7 +726,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -688,7 +760,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } @@ -286,7 +535,7 @@ fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } -@@ -708,7 +749,12 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -708,7 +783,12 @@ static int mk_Library(Itdb_iTunesDB *itdb, fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } @@ -300,7 +549,7 @@ fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } -@@ -721,9 +767,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -721,9 +801,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* major */ /* FIXME: the versioning scheme needs to be updated in libgpod using */ /* major+minor version, this is a workaround */ @@ -313,7 +562,7 @@ /* compatibility */ /* This has an unknown meaning. use 0. */ sqlite3_bind_int(stmt_version_info, ++idx, 0); -@@ -740,6 +787,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -740,6 +821,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 2 sqlite3_step returned %d\n", __func__, res); @@ -322,7 +571,7 @@ } printf("[%s] - inserting into \"genre_map\"\n", __func__); -@@ -809,6 +858,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -809,10 +892,15 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -331,16 +580,63 @@ } types |= track->mediatype; -@@ -863,6 +914,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, + } ++ if (pl->type == 1) { ++ master_types = types; ++ } + + printf("[%s] - inserting playlist '%s' into \"container\"\n", __func__, pl->name); + res = sqlite3_reset(stmt_container); +@@ -863,6 +951,48 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 4 sqlite3_step returned %d\n", __func__, res); + if (nano) + goto leave; ++ } ++ } ++ ++ /* the firmware evaluates the categories' rules itself: no members */ ++ for (gl = categories; gl; gl = gl->next) { ++ Itdb_Playlist *pl = gl->data; ++ Itdb_SPLPref *pref = &pl->splpref; ++ GByteArray *criteria = itdb_splrules_build_blob(itdb, &pl->splrules); ++ ++ printf("[%s] - inserting category '%s' into \"container\"\n", __func__, pl->name); ++ sqlite3_reset(stmt_category); ++ idx = 0; ++ sqlite3_bind_int64(stmt_category, ++idx, pl->id); ++ sqlite3_bind_int(stmt_category, ++idx, pl->priv->distinguished_kind); ++ /* date_created, date_modified */ ++ sqlite3_bind_int(stmt_category, ++idx, timeconv(pl->timestamp)); ++ sqlite3_bind_int(stmt_category, ++idx, timeconv(pl->timestamp)); ++ sqlite3_bind_text(stmt_category, ++idx, pl->name, -1, SQLITE_STATIC); ++ /* name_order */ ++ sqlite3_bind_int(stmt_category, ++idx, pos++); ++ /* media_kinds: iTunes gives Music (4) the master's */ ++ sqlite3_bind_int(stmt_category, ++idx, (pl->priv->distinguished_kind == 4) ? master_types : 0); ++ /* smart_is_dynamic, smart_is_filtered */ ++ sqlite3_bind_int(stmt_category, ++idx, pref->liveupdate ? 1 : 0); ++ sqlite3_bind_int(stmt_category, ++idx, pref->checkrules ? 1 : 0); ++ /* smart_enabled_only, smart_is_limited */ ++ sqlite3_bind_int(stmt_category, ++idx, pref->matchcheckedonly ? 1 : 0); ++ sqlite3_bind_int(stmt_category, ++idx, pref->checklimits ? 1 : 0); ++ /* smart_limit_order, smart_limit_value, smart_reverse_limit_order */ ++ sqlite3_bind_int(stmt_category, ++idx, pref->limitsort & 0xff); ++ sqlite3_bind_int(stmt_category, ++idx, pref->limitvalue); ++ sqlite3_bind_int(stmt_category, ++idx, (pref->limitsort & 0x80000000) ? 1 : 0); ++ /* smart_criteria */ ++ sqlite3_bind_blob(stmt_category, ++idx, criteria->data, criteria->len, SQLITE_TRANSIENT); ++ g_byte_array_free(criteria, TRUE); ++ ++ res = sqlite3_step(stmt_category); ++ if (res != SQLITE_DONE) { ++ fprintf(stderr, "[%s] category sqlite3_step returned %d\n", __func__, res); ++ goto leave; } } -@@ -911,6 +964,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -911,6 +1041,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 3 sqlite3_step returned %d\n", __func__, res); @@ -349,7 +645,7 @@ } /* for each track: */ -@@ -1088,6 +1143,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1088,6 +1220,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 5 sqlite3_step returned %d: %s\n", __func__, res, sqlite3_errmsg(db)); @@ -358,7 +654,7 @@ } /* printf("[%s] -- inserting into \"avformat_info\"\n", __func__); */ -@@ -1144,6 +1201,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1144,6 +1278,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 7 sqlite3_step returned %d\n", __func__, res); @@ -367,7 +663,7 @@ } /* this is done by a trigger, so we don't need to do this :-D */ -@@ -1191,6 +1250,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1191,6 +1327,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -376,7 +672,7 @@ } } -@@ -1227,6 +1288,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1227,6 +1365,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -385,7 +681,7 @@ } } -@@ -1252,6 +1315,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1252,6 +1392,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -394,7 +690,7 @@ } } /* if it's a movie, music video or tv show */ -@@ -1314,13 +1379,19 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1314,13 +1456,19 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -415,7 +711,18 @@ res = 0; printf("[%s] done.\n", __func__); -@@ -1385,6 +1456,7 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1365,6 +1513,10 @@ leave: + if (stmt_video_info) { + sqlite3_finalize(stmt_video_info); + } ++ if (stmt_category) { ++ sqlite3_finalize(stmt_category); ++ } ++ g_list_free(categories); + if (genre_map) { + g_hash_table_destroy(genre_map); + } +@@ -1385,6 +1537,7 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu sqlite3_stmt *stmt = NULL; char *errmsg = NULL; int idx = 0; @@ -423,7 +730,7 @@ dbf = g_build_filename(outpath, "Locations.itdb", NULL); printf("[%s] Processing '%s'\n", __func__, dbf); -@@ -1415,7 +1487,11 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1415,7 +1568,11 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu goto leave; } @@ -436,7 +743,7 @@ if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"base_location\" (id, path) VALUES (?,?);", -1, &stmt, NULL)) { fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); -@@ -1438,6 +1514,8 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1438,6 +1595,8 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu /* expected result */ } else { fprintf(stderr, "[%s] sqlite3_step returned %d\n", __func__, res); @@ -445,7 +752,7 @@ } if (itdb_device_is_iphone_family(itdb->device)) { -@@ -1489,7 +1567,6 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1489,7 +1648,6 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu int i = 0; int cnt = 0; int pos = 0; @@ -453,7 +760,7 @@ if (!track->ipod_path) { continue; -@@ -1556,12 +1633,18 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1556,12 +1714,18 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu free(ipod_path); ipod_path = NULL; } @@ -473,7 +780,7 @@ res = 0; printf("[%s] done.\n", __func__); -@@ -1752,7 +1835,28 @@ static void sqlite_func_iphone_sort_section(sqlite3_context *context, int argc, +@@ -1752,7 +1916,28 @@ static void sqlite_func_iphone_sort_section(sqlite3_context *context, int argc, sqlite3_result_int(context, res); } @@ -503,7 +810,7 @@ { plist_t plist_node = NULL; plist_t ppc_dict = NULL; -@@ -1760,6 +1864,8 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1760,6 +1945,8 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, const gchar *otherdbs[] = {"Dynamic.itdb", "Extras.itdb", "Genius.itdb", "Locations.itdb", NULL}; int res; sqlite3 *db = NULL; @@ -512,7 +819,7 @@ #ifdef HAVE_LIBIMOBILEDEVICE if (itdb_device_is_iphone_family(itdb->device)) { -@@ -1846,7 +1952,9 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1846,7 +2033,9 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, plist_dict_next_item(curnode, iter, &key, &subnode); while (subnode) { unsigned long int intval = strtoul(key, NULL, 0); @@ -523,7 +830,7 @@ user_ver_cmds = subnode; maxver = intval; } -@@ -1874,6 +1982,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1874,6 +2063,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, gchar *attach_str = NULL; char *errmsg = NULL; guint32 i = 0, cnt = 0, ok_cnt = 0; @@ -531,7 +838,7 @@ plist_dict_new_iter(sql_cmds, &iter); if (iter) { -@@ -1900,6 +2009,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1900,6 +2090,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, if (SQLITE_OK != sqlite3_open((const char*)dbf, &db)) { fprintf(stderr, "Error opening database '%s': %s\n", dbf, sqlite3_errmsg(db)); @@ -539,7 +846,7 @@ g_free(dbf); goto leave; } -@@ -1910,12 +2020,14 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1910,12 +2101,14 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, while (otherdbs[i]) { errmsg = NULL; dbf = g_build_filename(outpath, otherdbs[i], NULL); @@ -556,7 +863,7 @@ } if (errmsg) { free(errmsg); -@@ -1930,28 +2042,39 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1930,28 +2123,39 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, cnt = plist_array_get_size(user_ver_cmds); printf("[%s] Running %d post process commands now\n", __func__, cnt); @@ -599,7 +906,7 @@ } } g_hash_table_foreach(sqlcmd_map, free_key_val_strings, NULL); -@@ -1959,19 +2082,28 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1959,19 +2163,28 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, printf("[%s] %d out of %d post process commands successfully executed\n", __func__, ok_cnt, cnt); /* TODO perhaps we want to roll back when an error has occured ? */ @@ -629,7 +936,7 @@ } printf("[%s] done.\n", __func__); -@@ -1983,7 +2115,14 @@ leave: +@@ -1983,7 +2196,14 @@ leave: if (plist_node) { plist_free(plist_node); } @@ -645,7 +952,7 @@ } static int cbk_calc_sha1_one_block(FILE *f, unsigned char sha1[20]) -@@ -2167,7 +2306,9 @@ static int build_itdb_files(Itdb_iTunesDB *itdb, +@@ -2167,7 +2387,9 @@ static int build_itdb_files(Itdb_iTunesDB *itdb, return -1; } From 27f33d0710da4d560e6086eeb1bc2d2287e6802b Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 20:29:36 +0100 Subject: [PATCH 09/21] feat(libgpod): keep iPod nano 6G/7G play data and exact dates across writes The nano 6G/7G firmware records plays, skips, ratings, bookmarks and whether a track was started in Dynamic.itdb, and reports plays and skips in Play Counts as well (the count minus play_count_user_original). libgpod rebuilt Dynamic.itdb from the iTunesDB and Play Counts only, so a write dropped what the device had recorded there. SQLite dates were also off by the host's offset, and an unset one was written as -978307200 minus the offset. On devices that sign with HashAB and are not iOS devices: - itdb_parse() reads the device's Dynamic.itdb by item pid after the Play Counts merge. Counts take the larger of the two totals, as the parser has added the Play Counts delta already; ratings, bookmarks and recent counts take the device's value, and play and skip dates the later of the two reports. A track the device marked as played, which it does when a track is started or skipped, stays marked. The exact UTC dates of items and playlists come from Library.itdb and Locations.itdb when they can be read. Edits made after parsing are written as usual, and the post-process commands reset the *_original counts to the totals. - A write keeps the audio and subtitle track choices the device recorded for items still in the library; the post-process commands would reset them. It fails, rather than replace the file, when Dynamic.itdb could not be read at parse time. - SQLite dates are UTC seconds since 2001 and an unset one is 0; the value libgpod wrote for an unset date before reads as unset. Locations.itdb records the date added, as iTunes does, and an unset one stays 0. - Play Counts dates use the database's fixed offset and the file's "no date" value sets none; a reported skip date is taken over like the play date. The roundtrip covers plays counted once over repeated writes, Play Counts dates with and without a Dynamic.itdb row and when the two reports differ, activity recorded only in Dynamic.itdb including the four track choice columns and the played mark, host edits that change or clear it, kept, removed and new items, a Dynamic.itdb without track choice columns, dates in four host time zones, and an unreadable Dynamic.itdb. Both generations stay read-only. --- tools/nano7/README.md | 12 + tools/nano7/libgpod-roundtrip.c | 17 +- tools/nano7/test_libgpod.py | 186 ++++++++++++- tools/prebuild/libgpod-nano.patch | 433 ++++++++++++++++++++++++++---- 4 files changed, 595 insertions(+), 53 deletions(-) diff --git a/tools/nano7/README.md b/tools/nano7/README.md index 837405ac..bef0fd3b 100644 --- a/tools/nano7/README.md +++ b/tools/nano7/README.md @@ -53,6 +53,18 @@ command, or an insert or commit that fails while the databases are generated fails the write and leaves the library unchanged. A nano 5th generation keeps libgpod's previous layout. +A write keeps what a nano 6th/7th generation recorded in `Dynamic.itdb`. +Plays, skips, ratings, bookmarks, recent counts and their dates become the +tracks' values when the library is opened, so changes the host makes +afterwards win; a play reported in both `Dynamic.itdb` and `Play Counts` +counts once, and where their dates differ the later one is kept. The chosen +audio and subtitle tracks, which libgpod does not represent, are carried over +for the items that remain. SQLite dates are exact UTC whatever the host's time +zone, and an unset one is 0; exact item and playlist dates need a readable +`Library.itdb` and `Locations.itdb`, otherwise the `iTunesCDB`'s local times +are used. A `Dynamic.itdb` that cannot be read stops the write. The checks +cover these cases. + ## Checking a copied device library A nano 7th generation synced by macOS carries no `SysInfoExtended` file, so diff --git a/tools/nano7/libgpod-roundtrip.c b/tools/nano7/libgpod-roundtrip.c index df766a35..32816b07 100644 --- a/tools/nano7/libgpod-roundtrip.c +++ b/tools/nano7/libgpod-roundtrip.c @@ -1,7 +1,9 @@ /* Create, resave or delete one synthetic track with libgpod, add a second * one, a movie (add), give the first a 1 MiB comment (grow) or the second - * playlist the master playlist's id (clash). - * Usage: libgpod-roundtrip create|resave|delete|add|grow|clash */ + * playlist the master playlist's id (clash), set the first one's rating to 60 + * and bookmark to 3000 ms (rate) or clear its play data (clear). + * Usage: libgpod-roundtrip + * create|resave|delete|add|grow|clash|rate|clear */ #include #include @@ -33,6 +35,8 @@ int main(int argc, char **argv) track->bitrate = 128; track->size = 32000; track->transferred = TRUE; + track->time_modified = 1784116800; /* 2026-07-15 12:00 UTC */ + track->time_added = 1768478400; /* 2026-01-15 12:00 UTC */ itdb_track_add(db, track, -1); itdb_playlist_add_track(master, track, -1); master = itdb_playlist_new("Test playlist", FALSE); @@ -70,6 +74,15 @@ int main(int argc, char **argv) track->comment = g_strnfill(1 << 20, 'x'); } else if (strcmp(argv[2], "clash") == 0) { ((Itdb_Playlist *)g_list_last(db->playlists)->data)->id = itdb_playlist_mpl(db)->id; + } else if (strcmp(argv[2], "rate") == 0) { + track->rating = 60; + track->bookmark_time = 3000; + } else if (strcmp(argv[2], "clear") == 0) { + track->playcount = track->skipcount = 0; + track->recent_playcount = track->recent_skipcount = 0; + track->time_played = track->last_skipped = 0; + track->rating = 0; + track->bookmark_time = 0; } else if (strcmp(argv[2], "resave") != 0) { return 2; } diff --git a/tools/nano7/test_libgpod.py b/tools/nano7/test_libgpod.py index ea2ef8ef..4302d291 100644 --- a/tools/nano7/test_libgpod.py +++ b/tools/nano7/test_libgpod.py @@ -10,6 +10,7 @@ import struct import subprocess import tempfile +import time import unittest import zlib @@ -17,7 +18,12 @@ from probe import HashAB, guid_bytes, verify_cbk, verify_cdb CAPTURE = ROOT.parents[1] / "docs/sysinfo-captures/nano-7g-16gb-usb.xml" +MAC_EPOCH = 2082844800 +CORE_DATA_EPOCH = 978307200 +# The synthetic track's dates (libgpod-roundtrip.c) +MODIFIED, ADDED = 1784116800, 1768478400 POST_PROCESS = "com.apple.mobile.iTunes.SQLMusicLibraryPostProcessCommands" +TRACK_CHOICES = "audio_language, subtitle_language, audio_track_id, subtitle_track_id" # Firmware categories as iTunes writes them for a nano 7G: distinguished_kind # (mhyp +0x52) -> (+0x50, rule 1 mask, rule 2 action, rule 2 mask). CATEGORIES = { @@ -47,6 +53,20 @@ def columns(path, table): return {row[1] for row in rows(path, f"PRAGMA table_info({table})")} +def host_offset(tz): + """Seconds east of UTC of the zone tz now, as libgpod's host time conversion uses it.""" + saved = os.environ.get("TZ") + os.environ["TZ"] = tz + time.tzset() + offset = time.localtime().tm_gmtoff + if saved is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = saved + time.tzset() + return offset + + def category_slst(kind): """The SLst payload (big-endian) of a category's two media kind rules.""" _, mask1, action2, mask2 = CATEGORIES[kind] @@ -165,9 +185,9 @@ def setup_synthetic(self, model, db_version, commands=None): (self.control / "Device/SysInfoExtended").write_bytes(plistlib.dumps(properties)) (self.control / "Device/SysInfo").write_text(f"ModelNumStr: {model}\n") - def run_helper(self, operation, success=True, preexec_fn=None): - result = subprocess.run([str(self.helper), str(self.mount), operation], - capture_output=True, text=True, preexec_fn=preexec_fn) + def run_helper(self, operation, success=True, preexec_fn=None, tz="Europe/Berlin"): + result = subprocess.run([str(self.helper), str(self.mount), operation], capture_output=True, + text=True, preexec_fn=preexec_fn, env=dict(os.environ, TZ=tz)) if success: self.assertEqual(result.returncode, 0, result.stdout + result.stderr) else: @@ -393,6 +413,166 @@ def test_nano_5g_keeps_libgpod_layout(self): self.assertEqual(rows(self.db("Genius"), "PRAGMA user_version"), [(0,)]) self.assertEqual(rows(library, "SELECT major, minor FROM version_info"), [(2, 48)]) + def stats(self, columns="play_count_user, play_count_recent, play_count_user_original", pid=None): + return rows(self.db("Dynamic"), f"SELECT item_pid, {columns} FROM item_stats " + "WHERE ? IS NULL OR item_pid = ? ORDER BY item_pid", (pid, pid)) + + def set_stats(self, values, pid=None): + """What the firmware records in Dynamic.itdb.""" + with contextlib.closing(sqlite3.connect(self.db("Dynamic"))) as db, db: + db.execute(f"UPDATE item_stats SET {values} WHERE ? IS NULL OR item_pid = ?", (pid, pid)) + + def cdb_track(self, offset, size="playcount = get32lint (cts, seek); + mac_time = get32lint (cts, seek+4); + playcount->time_played = device_time_mac_to_time_t (fimp->itdb->device, mac_time); ++ playcount->mac_played = mac_time; + playcount->bookmark_time = get32lint (cts, seek+8); + + /* rating only exists if the entry length is at least 0x10 */ +@@ -1009,6 +1010,7 @@ static gboolean playcounts_read (FImport *fimp, FContents *cts) + { + playcount->skipcount = get32lint (cts, seek+20); + mac_time = get32lint (cts, seek+24); ++ playcount->mac_skipped = mac_time; + playcount->last_skipped = device_time_mac_to_time_t (fimp->itdb->device, + mac_time); + +@@ -2269,6 +2271,8 @@ static glong get_playlist (FImport *fimp, guint mhsd_type, glong mhyp_seek) plitem->sortorder = get32lint (cts, mhyp_seek+44); if (header_len >= 0x6C) { plitem->priv->mhsd5_type = get16lint (cts, mhyp_seek+0x50); @@ -112,7 +128,51 @@ } mhod_seek = mhyp_seek + header_len; -@@ -4640,7 +4642,12 @@ static void mk_mhod (FExport *fexp, MHODData *mhod) +@@ -2437,6 +2441,15 @@ static glong get_playlist (FImport *fimp, guint mhsd_type, glong mhyp_seek) + } + + ++/* iPod nano 6G/7G write Play Counts dates with the database's fixed ++ offset, and "none" as 2001-01-01 00:00 shifted by it */ ++static time_t play_counts_time (Itdb_iTunesDB *itdb, guint32 mac_time) ++{ ++ if (mac_time == 2082844800u + 978307200u + (guint32) itdb->tzoffset) ++ return 0; ++ return (time_t) mac_time - 2082844800 - itdb->tzoffset; ++} ++ + /* returns a pointer to the next header or -1 on error. fimp->error is + set appropriately. If no "mhit" header is found at the location + specified, -1 is returned but no error is set. */ +@@ -2729,6 +2742,19 @@ static glong get_mhit (FImport *fimp, glong mhit_seek) + playcount = g_hash_table_lookup (fimp->pcounts2, &track->dbid); + free_playcount = FALSE; + } ++ if (playcount && itdb_device_is_hashab_nano (fimp->itdb->device)) ++ { ++ /* only the Play Counts file sets mac_played and mac_skipped */ ++ if (playcount->mac_played) { ++ playcount->time_played = play_counts_time (fimp->itdb, playcount->mac_played); ++ track->priv->reported_played = playcount->time_played; ++ } ++ if (playcount->mac_skipped) { ++ track->priv->reported_skipped = play_counts_time (fimp->itdb, playcount->mac_skipped); ++ if (track->priv->reported_skipped) ++ track->last_skipped = track->priv->reported_skipped; ++ } ++ } + if (playcount) + { + if (playcount->rating != NO_PLAYCOUNT) +@@ -3314,6 +3340,7 @@ itdb_parse_internal (Itdb_iTunesDB *itdb, gboolean compressed, GError **error) + { + if (read_OTG_playlists (fimp)) + { ++ itdb_sqlite_read_device_state (fimp->itdb); + success = TRUE; + } + } +@@ -4640,7 +4667,12 @@ static void mk_mhod (FExport *fexp, MHODData *mhod) g_return_if_fail (splr); ft = itdb_splr_get_field_type (splr); /* printf ("%p: field: %d ft: %d\n", splr, splr->field, ft);*/ @@ -126,7 +186,7 @@ put32bint (cts, splr->field); put32bint (cts, splr->action); put32_n0 (cts, 11); /* unknown */ -@@ -5529,13 +5536,20 @@ static gboolean write_playlist (FExport *fexp, +@@ -5529,13 +5561,20 @@ static gboolean write_playlist (FExport *fexp, put32_n0 (cts, 8); /* ? */ /* +0x50 */ put16lint (cts, pl->priv->mhsd5_type); @@ -153,7 +213,7 @@ } put32_n0 (cts, 5); /* ? */ -@@ -5991,6 +6005,8 @@ static gboolean itdb_write_file_internal (Itdb_iTunesDB *itdb, +@@ -5991,6 +6030,8 @@ static gboolean itdb_write_file_internal (Itdb_iTunesDB *itdb, g_return_val_if_fail (itdb->device, FALSE); g_return_val_if_fail (filename || itdb->filename, FALSE); @@ -162,7 +222,7 @@ if (!filename) filename = itdb->filename; /* set endianess flag */ -@@ -8254,6 +8270,28 @@ G_GNUC_INTERNAL GByteArray *itdb_chapterdata_build_chapter_blob(Itdb_Chapterdata +@@ -8254,6 +8295,28 @@ G_GNUC_INTERNAL GByteArray *itdb_chapterdata_build_chapter_blob(Itdb_Chapterdata return chapter_blob; } @@ -270,7 +330,36 @@ * @itdb: an #Itdb_iTunesDB --- a/src/itdb_private.h +++ b/src/itdb_private.h -@@ -210,11 +210,18 @@ struct _Itdb_Track_Private { +@@ -108,6 +108,9 @@ struct playcount { + time_t last_skipped; + gint32 st_unk06; /* unknown field in iTunesStats file */ + gint32 st_unk09; /* unknown field in iTunesStats file */ ++ /* time_played and last_skipped as stored in the Play Counts file */ ++ guint32 mac_played; ++ guint32 mac_skipped; + }; + + /* value to indicate that playcount was not set in struct playcount +@@ -186,6 +189,8 @@ typedef enum _Itdb_Playlist_Mhsd5_Type Itdb_Playlist_Mhsd5_Type; + struct _Itdb_iTunesDB_Private + { + GList *mhsd5_playlists; ++ /* iPod nano 6G/7G: Dynamic.itdb exists but could not be read */ ++ gboolean dynamic_unread; + guint16 platform; + guint16 unk_0x22; + guint64 id_0x24; +@@ -206,15 +211,28 @@ struct _Itdb_Track_Private { + guint32 album_id; + guint32 artist_id; + guint32 composer_id; ++ /* iPod nano 6G/7G: the dates the Play Counts file reported */ ++ time_t reported_played; ++ time_t reported_skipped; ++ /* iPod nano 6G/7G: Dynamic.itdb marks the track as played; the ++ firmware sets this when a track is started or skipped */ ++ gboolean device_played; + }; struct _Itdb_Playlist_Private { Itdb_Playlist_Mhsd5_Type mhsd5_type; @@ -289,6 +378,14 @@ G_GNUC_INTERNAL gboolean itdb_spl_action_known (ItdbSPLAction action); G_GNUC_INTERNAL void itdb_splr_free (Itdb_SPLRule *splr); G_GNUC_INTERNAL const gchar *itdb_photodb_get_mountpoint (Itdb_PhotoDB *photodb); +@@ -230,6 +248,7 @@ G_GNUC_INTERNAL guint64 device_time_time_t_to_mac (Itdb_Device *device, + time_t timet); + G_GNUC_INTERNAL gint itdb_musicdirs_number_by_mountpoint (const gchar *mountpoint); + G_GNUC_INTERNAL int itdb_sqlite_generate_itdbs(FExport *fexp); ++G_GNUC_INTERNAL void itdb_sqlite_read_device_state(Itdb_iTunesDB *itdb); + G_GNUC_INTERNAL gboolean itdb_hashAB_write_hash (const Itdb_Device *device, + unsigned char *itdb_data, + gsize itdb_len, --- a/src/itdb_sqlite_queries.h +++ b/src/itdb_sqlite_queries.h @@ -77,6 +77,22 @@ static const char Library_create[] = @@ -316,7 +413,21 @@ "BEGIN TRANSACTION;" \ --- a/src/itdb_sqlite.c +++ b/src/itdb_sqlite.c -@@ -180,6 +180,25 @@ static void destroy_key_orders(GHashTable **orders) +@@ -46,9 +46,13 @@ + + /** time zone offset in seconds */ + static uint32_t tzoffset = 0; ++/* iPod nano 6G/7G: dates are UTC, and an unset one stays 0 */ ++static gboolean utc_dates = FALSE; + + static uint32_t timeconv(time_t tv) + { ++ if (utc_dates) ++ return tv ? tv - 978307200 : 0; + /* quite strange. this is time_t - 01.01.2001 01:00:00 - tzoffset */ + return tv - 978307200 - tzoffset; + } +@@ -180,6 +184,25 @@ static void destroy_key_orders(GHashTable **orders) g_free(orders); } @@ -342,7 +453,7 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) { int res = -1; -@@ -190,6 +209,8 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -190,6 +213,8 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) struct stat fst; int idx = 0; GList *gl = NULL; @@ -351,7 +462,7 @@ dbf = g_build_filename(outpath, "Dynamic.itdb", NULL); printf("[%s] Processing '%s'\n", __func__, dbf); -@@ -227,7 +248,11 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -227,7 +252,11 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) goto leave; } @@ -364,7 +475,16 @@ if (itdb->tracks) { printf("[%s] - processing %d tracks\n", __func__, g_list_length(itdb->tracks)); -@@ -277,6 +302,8 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -248,7 +277,7 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) + /* item_pid */ + sqlite3_bind_int64(stmt, ++idx, track->dbid); + /* has_been_played */ +- sqlite3_bind_int(stmt, ++idx, (track->playcount > 0) ? 1 : 0); ++ sqlite3_bind_int(stmt, ++idx, (track->playcount > 0 || track->priv->device_played) ? 1 : 0); + /* date_played */ + sqlite3_bind_int(stmt, ++idx, timeconv(track->time_played)); + /* play_count_user */ +@@ -277,6 +306,8 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) /* expected result */ } else { fprintf(stderr, "[%s] 1 sqlite3_step returned %d\n", __func__, res); @@ -373,7 +493,7 @@ } } if (stmt) { -@@ -293,8 +320,9 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -293,8 +324,9 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) goto leave; } @@ -385,7 +505,7 @@ Itdb_Playlist *pl = (Itdb_Playlist*)gl->data; res = sqlite3_reset(stmt); -@@ -333,19 +361,24 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) +@@ -333,19 +365,24 @@ static int mk_Dynamic(Itdb_iTunesDB *itdb, const char *outpath) /* expected result */ } else { fprintf(stderr, "[%s] 2 sqlite3_step returned %d\n", __func__, res); @@ -415,7 +535,7 @@ if (db) { sqlite3_close(db); } -@@ -363,6 +396,7 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) +@@ -363,6 +400,7 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) sqlite3 *db = NULL; sqlite3_stmt *stmt_chapter = NULL; char *errmsg = NULL; @@ -423,7 +543,7 @@ struct stat fst; GList *gl = NULL; -@@ -399,7 +433,11 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) +@@ -399,7 +437,11 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) } } @@ -436,7 +556,7 @@ if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"chapter\" VALUES(?,?);", -1, &stmt_chapter, NULL)) { fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; -@@ -450,9 +488,15 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) +@@ -450,9 +492,15 @@ static int mk_Extras(Itdb_iTunesDB *itdb, const char *outpath) fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); } g_byte_array_free(chapter_blob, TRUE); @@ -453,7 +573,7 @@ res = 0; printf("[%s] done.\n", __func__); -@@ -510,6 +554,12 @@ static int mk_Genius(Itdb_iTunesDB *itdb, const char *outpath) +@@ -510,6 +558,12 @@ static int mk_Genius(Itdb_iTunesDB *itdb, const char *outpath) goto leave; } } @@ -466,7 +586,7 @@ sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); -@@ -606,15 +656,19 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -606,15 +660,19 @@ static int mk_Library(Itdb_iTunesDB *itdb, sqlite3_stmt *stmt_artist = NULL; sqlite3_stmt *stmt_composer = NULL; sqlite3_stmt *stmt_video_info = NULL; @@ -486,7 +606,7 @@ printf("library_persistent_id = 0x%016"G_GINT64_MODIFIER"x\n", itdb->priv->pid); dbf = g_build_filename(outpath, "Library.itdb", NULL); -@@ -652,8 +706,16 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -652,8 +710,16 @@ static int mk_Library(Itdb_iTunesDB *itdb, } goto leave; } @@ -504,7 +624,7 @@ fprintf(stderr, "[%s] compiling SQL statements\n", __func__); if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"version_info\" VALUES(?,?,?,?,?,?,?);", -1, &stmt_version_info, NULL)) { -@@ -665,7 +727,17 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -665,7 +731,17 @@ static int mk_Library(Itdb_iTunesDB *itdb, goto leave; } if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"container\" " @@ -523,7 +643,7 @@ fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } -@@ -688,7 +760,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -688,7 +764,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } @@ -535,7 +655,7 @@ fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } -@@ -708,7 +783,12 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -708,7 +787,12 @@ static int mk_Library(Itdb_iTunesDB *itdb, fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } @@ -549,7 +669,7 @@ fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); goto leave; } -@@ -721,9 +801,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -721,9 +805,10 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* major */ /* FIXME: the versioning scheme needs to be updated in libgpod using */ /* major+minor version, this is a workaround */ @@ -562,7 +682,7 @@ /* compatibility */ /* This has an unknown meaning. use 0. */ sqlite3_bind_int(stmt_version_info, ++idx, 0); -@@ -740,6 +821,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -740,6 +825,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 2 sqlite3_step returned %d\n", __func__, res); @@ -571,7 +691,7 @@ } printf("[%s] - inserting into \"genre_map\"\n", __func__); -@@ -809,10 +892,15 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -809,10 +896,15 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -587,7 +707,7 @@ printf("[%s] - inserting playlist '%s' into \"container\"\n", __func__, pl->name); res = sqlite3_reset(stmt_container); -@@ -863,6 +951,48 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -863,6 +955,48 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 4 sqlite3_step returned %d\n", __func__, res); @@ -636,7 +756,7 @@ } } -@@ -911,6 +1041,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -911,6 +1045,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 3 sqlite3_step returned %d\n", __func__, res); @@ -645,7 +765,7 @@ } /* for each track: */ -@@ -1088,6 +1220,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1088,6 +1224,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 5 sqlite3_step returned %d: %s\n", __func__, res, sqlite3_errmsg(db)); @@ -654,7 +774,7 @@ } /* printf("[%s] -- inserting into \"avformat_info\"\n", __func__); */ -@@ -1144,6 +1278,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1144,6 +1282,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 7 sqlite3_step returned %d\n", __func__, res); @@ -663,7 +783,7 @@ } /* this is done by a trigger, so we don't need to do this :-D */ -@@ -1191,6 +1327,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1191,6 +1331,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -672,7 +792,7 @@ } } -@@ -1227,6 +1365,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1227,6 +1369,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -681,7 +801,7 @@ } } -@@ -1252,6 +1392,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1252,6 +1396,8 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -690,7 +810,7 @@ } } /* if it's a movie, music video or tv show */ -@@ -1314,13 +1456,19 @@ static int mk_Library(Itdb_iTunesDB *itdb, +@@ -1314,13 +1460,19 @@ static int mk_Library(Itdb_iTunesDB *itdb, /* expected result */ } else { fprintf(stderr, "[%s] 8 sqlite3_step returned %d\n", __func__, res); @@ -711,7 +831,7 @@ res = 0; printf("[%s] done.\n", __func__); -@@ -1365,6 +1513,10 @@ leave: +@@ -1365,6 +1517,10 @@ leave: if (stmt_video_info) { sqlite3_finalize(stmt_video_info); } @@ -722,7 +842,7 @@ if (genre_map) { g_hash_table_destroy(genre_map); } -@@ -1385,6 +1537,7 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1385,6 +1541,7 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu sqlite3_stmt *stmt = NULL; char *errmsg = NULL; int idx = 0; @@ -730,7 +850,7 @@ dbf = g_build_filename(outpath, "Locations.itdb", NULL); printf("[%s] Processing '%s'\n", __func__, dbf); -@@ -1415,7 +1568,11 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1415,7 +1572,11 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu goto leave; } @@ -743,7 +863,7 @@ if (SQLITE_OK != sqlite3_prepare_v2(db, "INSERT INTO \"base_location\" (id, path) VALUES (?,?);", -1, &stmt, NULL)) { fprintf(stderr, "[%s] sqlite3_prepare error: %s\n", __func__, sqlite3_errmsg(db)); -@@ -1438,6 +1595,8 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1438,6 +1599,8 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu /* expected result */ } else { fprintf(stderr, "[%s] sqlite3_step returned %d\n", __func__, res); @@ -752,7 +872,7 @@ } if (itdb_device_is_iphone_family(itdb->device)) { -@@ -1489,7 +1648,6 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1489,7 +1652,6 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu int i = 0; int cnt = 0; int pos = 0; @@ -760,7 +880,18 @@ if (!track->ipod_path) { continue; -@@ -1556,12 +1714,18 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu +@@ -1528,8 +1690,8 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu + sqlite3_bind_int(stmt, ++idx, track->filetype_marker); + /* kind_id, should match track->mediatype */ + sqlite3_bind_int(stmt, ++idx, track->mediatype); +- /* date_created */ +- sqlite3_bind_int(stmt, ++idx, timeconv(track->time_modified)); ++ /* date_created: the date added on nano 6G/7G, as iTunes writes it */ ++ sqlite3_bind_int(stmt, ++idx, timeconv(utc_dates ? track->time_added : track->time_modified)); + /* file_size */ + sqlite3_bind_int(stmt, ++idx, track->size); + #if 0 +@@ -1556,12 +1718,18 @@ static int mk_Locations(Itdb_iTunesDB *itdb, const char *outpath, const char *uu free(ipod_path); ipod_path = NULL; } @@ -780,7 +911,7 @@ res = 0; printf("[%s] done.\n", __func__); -@@ -1752,7 +1916,28 @@ static void sqlite_func_iphone_sort_section(sqlite3_context *context, int argc, +@@ -1752,7 +1920,28 @@ static void sqlite_func_iphone_sort_section(sqlite3_context *context, int argc, sqlite3_result_int(context, res); } @@ -810,7 +941,7 @@ { plist_t plist_node = NULL; plist_t ppc_dict = NULL; -@@ -1760,6 +1945,8 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1760,6 +1949,8 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, const gchar *otherdbs[] = {"Dynamic.itdb", "Extras.itdb", "Genius.itdb", "Locations.itdb", NULL}; int res; sqlite3 *db = NULL; @@ -819,7 +950,7 @@ #ifdef HAVE_LIBIMOBILEDEVICE if (itdb_device_is_iphone_family(itdb->device)) { -@@ -1846,7 +2033,9 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1846,7 +2037,9 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, plist_dict_next_item(curnode, iter, &key, &subnode); while (subnode) { unsigned long int intval = strtoul(key, NULL, 0); @@ -830,7 +961,7 @@ user_ver_cmds = subnode; maxver = intval; } -@@ -1874,6 +2063,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1874,6 +2067,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, gchar *attach_str = NULL; char *errmsg = NULL; guint32 i = 0, cnt = 0, ok_cnt = 0; @@ -838,7 +969,7 @@ plist_dict_new_iter(sql_cmds, &iter); if (iter) { -@@ -1900,6 +2090,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1900,6 +2094,7 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, if (SQLITE_OK != sqlite3_open((const char*)dbf, &db)) { fprintf(stderr, "Error opening database '%s': %s\n", dbf, sqlite3_errmsg(db)); @@ -846,7 +977,7 @@ g_free(dbf); goto leave; } -@@ -1910,12 +2101,14 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1910,12 +2105,14 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, while (otherdbs[i]) { errmsg = NULL; dbf = g_build_filename(outpath, otherdbs[i], NULL); @@ -863,7 +994,7 @@ } if (errmsg) { free(errmsg); -@@ -1930,28 +2123,39 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1930,28 +2127,39 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, cnt = plist_array_get_size(user_ver_cmds); printf("[%s] Running %d post process commands now\n", __func__, cnt); @@ -906,7 +1037,7 @@ } } g_hash_table_foreach(sqlcmd_map, free_key_val_strings, NULL); -@@ -1959,19 +2163,28 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, +@@ -1959,19 +2167,28 @@ static void run_post_process_commands(Itdb_iTunesDB *itdb, const char *outpath, printf("[%s] %d out of %d post process commands successfully executed\n", __func__, ok_cnt, cnt); /* TODO perhaps we want to roll back when an error has occured ? */ @@ -936,7 +1067,7 @@ } printf("[%s] done.\n", __func__); -@@ -1983,7 +2196,14 @@ leave: +@@ -1983,7 +2200,14 @@ leave: if (plist_node) { plist_free(plist_node); } @@ -952,7 +1083,7 @@ } static int cbk_calc_sha1_one_block(FILE *f, unsigned char sha1[20]) -@@ -2167,7 +2387,9 @@ static int build_itdb_files(Itdb_iTunesDB *itdb, +@@ -2167,7 +2391,9 @@ static int build_itdb_files(Itdb_iTunesDB *itdb, return -1; } @@ -963,6 +1094,212 @@ if (!mk_Locations_cbk(itdb, outpath)) { g_set_error (error, ITDB_ERROR, ITDB_ERROR_SQLITE, +@@ -2250,6 +2476,187 @@ static void rmdir_recursive(gchar *path) + g_rmdir(path); + } + ++/* SQLite dates are UTC seconds since 2001; 0 is unset, as is the value ++ libgpod used to write for it */ ++static time_t sqlite_time(Itdb_iTunesDB *itdb, gint64 date) ++{ ++ if (date == 0 || date == -978307200 - (gint64)itdb->tzoffset) ++ return 0; ++ return (time_t)(date + 978307200); ++} ++ ++typedef struct { ++ gpointer object; ++ gint64 v[9]; ++} DeviceRow; ++ ++/* The rows of @sql (a pid and up to nine numbers) from the device's ++ @name whose pid is in @objects. None if @name exists but cannot be read ++ completely; *unreadable is then set, if given. */ ++static GArray *read_device_rows(const gchar *itlpdir, const char *name, const char *sql, ++ GHashTable *objects, gboolean *unreadable) ++{ ++ gchar *path = g_build_filename(itlpdir, name, NULL); ++ GArray *rows = g_array_new(FALSE, TRUE, sizeof(DeviceRow)); ++ sqlite3 *db = NULL; ++ sqlite3_stmt *stmt = NULL; ++ int rc = SQLITE_DONE; ++ ++ if (g_file_test(path, G_FILE_TEST_EXISTS)) { ++ rc = sqlite3_open_v2(path, &db, SQLITE_OPEN_READONLY, NULL); ++ if (rc == SQLITE_OK) ++ rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); ++ while (rc == SQLITE_OK || rc == SQLITE_ROW) { ++ DeviceRow row = { NULL }; ++ gint64 pid; ++ int i; ++ ++ rc = sqlite3_step(stmt); ++ if (rc != SQLITE_ROW) ++ break; ++ pid = sqlite3_column_int64(stmt, 0); ++ row.object = g_hash_table_lookup(objects, &pid); ++ for (i = 1; i < sqlite3_column_count(stmt) && i <= 9; i++) ++ row.v[i - 1] = sqlite3_column_int64(stmt, i); ++ if (row.object) ++ g_array_append_val(rows, row); ++ } ++ sqlite3_finalize(stmt); ++ sqlite3_close(db); ++ } ++ g_free(path); ++ if (rc != SQLITE_DONE) { ++ g_array_set_size(rows, 0); ++ if (unreadable) ++ *unreadable = TRUE; ++ } ++ return rows; ++} ++ ++/** ++ * itdb_sqlite_read_device_state: ++ * ++ * iPod nano 6G/7G: take over what the device's SQLite files know beyond the ++ * iTunesDB, so that the tracks show and a write keeps it. The firmware ++ * records plays, skips, ratings, bookmarks and whether a track was started ++ * in Dynamic.itdb, and plays and skips in Play Counts as well, which the ++ * parser has added already: counts take the larger total, the rest the ++ * device's value. Dates are ++ * the exact UTC ones the iTunesDB only has as local wall times. ++ */ ++G_GNUC_INTERNAL void itdb_sqlite_read_device_state(Itdb_iTunesDB *itdb) ++{ ++ GHashTable *tracks, *playlists; ++ GArray *stats, *items, *locations, *containers; ++ GList *lists[2], *gl; ++ gchar *dir, *itlpdir; ++ guint i; ++ ++ if (!itdb_get_mountpoint(itdb) || !itdb_device_is_hashab_nano(itdb->device)) ++ return; ++ dir = itdb_get_itunes_dir(itdb_get_mountpoint(itdb)); ++ if (!dir) ++ return; ++ itlpdir = g_build_filename(dir, "iTunes Library.itlp", NULL); ++ g_free(dir); ++ tracks = g_hash_table_new(g_int64_hash, g_int64_equal); ++ for (gl = itdb->tracks; gl; gl = gl->next) ++ g_hash_table_insert(tracks, &((Itdb_Track *)gl->data)->dbid, gl->data); ++ playlists = g_hash_table_new(g_int64_hash, g_int64_equal); ++ lists[0] = itdb->playlists; ++ lists[1] = itdb->priv->mhsd5_playlists; ++ for (i = 0; i < 2; i++) ++ for (gl = lists[i]; gl; gl = gl->next) ++ g_hash_table_insert(playlists, &((Itdb_Playlist *)gl->data)->id, gl->data); ++ ++ /* only Dynamic.itdb holds data the device recorded; a write must not ++ replace it unread */ ++ stats = read_device_rows(itlpdir, "Dynamic.itdb", ++ "SELECT item_pid, play_count_user, skip_count_user, play_count_recent, " ++ "skip_count_recent, date_played, date_skipped, user_rating, bookmark_time_ms, " ++ "has_been_played FROM item_stats;", tracks, &itdb->priv->dynamic_unread); ++ for (i = 0; i < stats->len; i++) { ++ DeviceRow *row = &g_array_index(stats, DeviceRow, i); ++ Itdb_Track *track = row->object; ++ track->playcount = MAX(track->playcount, row->v[0]); ++ track->skipcount = MAX(track->skipcount, row->v[1]); ++ track->recent_playcount = row->v[2]; ++ track->recent_skipcount = row->v[3]; ++ /* the later date, when Play Counts reported a newer one */ ++ track->time_played = MAX(sqlite_time(itdb, row->v[4]), track->priv->reported_played); ++ track->last_skipped = MAX(sqlite_time(itdb, row->v[5]), track->priv->reported_skipped); ++ track->rating = row->v[6]; ++ track->bookmark_time = row->v[7]; ++ track->priv->device_played = row->v[8] != 0; ++ if (track->priv->device_played) ++ track->mark_unplayed = 0x01; ++ } ++ items = read_device_rows(itlpdir, "Library.itdb", "SELECT pid, date_modified FROM item;", ++ tracks, NULL); ++ for (i = 0; i < items->len; i++) { ++ DeviceRow *row = &g_array_index(items, DeviceRow, i); ++ ((Itdb_Track *)row->object)->time_modified = sqlite_time(itdb, row->v[0]); ++ } ++ locations = read_device_rows(itlpdir, "Locations.itdb", "SELECT item_pid, date_created FROM location;", ++ tracks, NULL); ++ for (i = 0; i < locations->len; i++) { ++ DeviceRow *row = &g_array_index(locations, DeviceRow, i); ++ ((Itdb_Track *)row->object)->time_added = sqlite_time(itdb, row->v[0]); ++ } ++ containers = read_device_rows(itlpdir, "Library.itdb", "SELECT pid, date_created FROM container;", ++ playlists, NULL); ++ for (i = 0; i < containers->len; i++) { ++ DeviceRow *row = &g_array_index(containers, DeviceRow, i); ++ ((Itdb_Playlist *)row->object)->timestamp = sqlite_time(itdb, row->v[0]); ++ } ++ ++ g_array_free(stats, TRUE); ++ g_array_free(items, TRUE); ++ g_array_free(locations, TRUE); ++ g_array_free(containers, TRUE); ++ g_hash_table_destroy(tracks); ++ g_hash_table_destroy(playlists); ++ g_free(itlpdir); ++} ++ ++/* iPod nano 6G/7G: keep the audio and subtitle track choices the device ++ recorded for items still in the library; the post-process commands ++ reset them. */ ++static int keep_device_choices(Itdb_iTunesDB *itdb, const char *outpath, const char *itlpdir, ++ GError **error) ++{ ++ gchar *device_file = g_build_filename(itlpdir, "Dynamic.itdb", NULL); ++ gchar *dbf = g_build_filename(outpath, "Dynamic.itdb", NULL); ++ sqlite3 *db = NULL; ++ int res = 0; ++ ++ if (itdb->priv->dynamic_unread) { ++ g_set_error(error, ITDB_ERROR, ITDB_ERROR_SQLITE, ++ "'%s' could not be read when the library was opened, so writing would lose " ++ "the plays, ratings and bookmarks it holds; remove it to write without them", ++ device_file); ++ res = -1; ++ } else if (g_file_test(device_file, G_FILE_TEST_EXISTS)) { ++ /* The columns are unqualified: one that the device's item_stats ++ lacks names the new row's own, which keeps its value. */ ++ char *sql = sqlite3_mprintf("ATTACH DATABASE %Q AS device;" ++ "UPDATE item_stats SET (audio_language, subtitle_language, audio_track_id, subtitle_track_id) = " ++ "(SELECT audio_language, subtitle_language, audio_track_id, subtitle_track_id " ++ "FROM device.item_stats d WHERE d.item_pid = item_stats.item_pid) " ++ "WHERE item_pid IN (SELECT item_pid FROM device.item_stats);", device_file); ++ if (sqlite3_open(dbf, &db) != SQLITE_OK || sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) { ++ g_set_error(error, ITDB_ERROR, ITDB_ERROR_SQLITE, ++ "cannot keep the track choices in '%s': %s", device_file, sqlite3_errmsg(db)); ++ res = -1; ++ } ++ sqlite3_free(sql); ++ sqlite3_close(db); ++ } ++ g_free(device_file); ++ g_free(dbf); ++ return res; ++} ++ + int itdb_sqlite_generate_itdbs(FExport *fexp) + { + int res = 0; +@@ -2277,6 +2684,7 @@ int itdb_sqlite_generate_itdbs(FExport *fexp) + g_assert(fexp->itdb->playlists != NULL); + + tzoffset = fexp->itdb->tzoffset; ++ utc_dates = itdb_device_is_hashab_nano(fexp->itdb->device); + + tmpdir = g_build_path(g_get_tmp_dir(), tmpnam(NULL), NULL); + if (g_mkdir(tmpdir, 0755) != 0) { +@@ -2289,7 +2697,8 @@ int itdb_sqlite_generate_itdbs(FExport *fexp) + + /* generate itdb files in temporary directory */ + if (build_itdb_files(fexp->itdb, fexp->albums, fexp->artists, fexp->composers, tmpdir, +- itdb_device_get_uuid(fexp->itdb->device), &fexp->error) != 0) { ++ itdb_device_get_uuid(fexp->itdb->device), &fexp->error) != 0 ++ || (utc_dates && keep_device_choices(fexp->itdb, tmpdir, itlpdir, &fexp->error) != 0)) { + g_prefix_error (&fexp->error, "Failed to generate sqlite database: "); + res = -1; + goto leave; --- a/src/Makefile.am +++ b/src/Makefile.am @@ -31,6 +31,33 @@ libgpod_la_CFLAGS = From 9c7d6990c343a64697ed032a53cafb08b2b253b0 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 20:31:52 +0100 Subject: [PATCH 10/21] feat(devices): enable sync for the iPod nano 7th generation With the firmware categories, iTunes' schema level and the device's play data kept, podkit's libgpod writes the iPod nano 7G library in the form its firmware needs: on firmware 1.0.4, the categories together with that schema level made a library show and play, also after a reboot. A nano 7G on that firmware showed and played libraries written by this code, kept the plays, skips and ratings it had recorded, and took added and removed tracks and playlists, so nano_7g becomes syncable with hardware provenance. Its USB product IDs leave the unsupported table, so detection, readiness and sync treat it like the other syncable nanos. Its videos use the ipod-classic profile, as for other generations without a profile of their own. The nano 6G stays read-only: podkit writes its library the same way, but only synthetic tests cover that. Its note and refusal say so now instead of claiming that writes are unsupported. Tests that used the nano 7G as their example of a read-only or refused device use the nano 6G. The generated support matrix, the supported-devices page, the device notes and tools/nano7 follow, and the changeset covers the libgpod-node changes of the previous commits. --- .changeset/nano-7g-sync.md | 7 ++++ devices/ipod.md | 6 +-- docs/adr/adr-030-native-nano-hashab.md | 7 +++- .../device/identity-support-matrix.md | 4 +- docs/formats/generations.md | 14 +++---- packages/devices-ipod/src/classify.test.ts | 8 ++-- packages/devices-ipod/src/identity.test.ts | 24 ++++++------ packages/devices-ipod/src/resolve.test.ts | 12 ++---- packages/devices-ipod/src/support.test.ts | 7 +--- .../devices-ipod/src/tables/generations.ts | 18 +++++---- .../src/tables/unsupported.test.ts | 9 ++--- .../devices-ipod/src/tables/unsupported.ts | 18 ++------- packages/devices-ipod/src/tables/usb-ids.ts | 2 +- packages/devices-ipod/src/types.ts | 4 +- .../docs/devices/supported-devices.mdx | 7 ++-- .../device/capability-summary.test.ts | 10 ++--- .../src/commands/doctor-read-only.test.ts | 38 +++++++++---------- .../src/commands/sync-runner.unit.test.ts | 12 +++--- .../src/device/discovery-permutations.test.ts | 15 ++++---- .../check-readiness-read-only.test.ts | 20 +++++----- .../check-readiness-unsupported.test.ts | 12 +++--- .../__tests__/determine-level.test.ts | 6 +-- tools/nano7/README.md | 8 ++-- 23 files changed, 128 insertions(+), 140 deletions(-) create mode 100644 .changeset/nano-7g-sync.md diff --git a/.changeset/nano-7g-sync.md b/.changeset/nano-7g-sync.md new file mode 100644 index 00000000..c530819b --- /dev/null +++ b/.changeset/nano-7g-sync.md @@ -0,0 +1,7 @@ +--- +'@podkit/libgpod-node': patch +'@podkit/devices-ipod': minor +'podkit': minor +--- + +podkit now syncs the iPod nano 7th generation, confirmed on a device with firmware 1.0.4. Its libgpod build writes the hidden category containers and iTunes' schema level that the nano 7th generation needs to show a library, and keeps the plays, skips, ratings and bookmarks the device records across syncs. The nano 6th generation stays read-only: its library writes are covered only by synthetic tests, which its unsupported-device message now says. diff --git a/devices/ipod.md b/devices/ipod.md index 2b2f112d..8c440af9 100644 --- a/devices/ipod.md +++ b/devices/ipod.md @@ -282,6 +282,7 @@ iPod_Control/ | Shuffle 1-2G | Yes | No | No | No | — | None | | Nano 1-2G | Yes | No | Yes | No | — | 176x132 | | Nano 3-5G | Yes | Yes | Yes | Yes | ipod-nano-3g | Varies | +| Nano 7G | Yes | Yes | Yes | Yes | ipod-classic | 240x432 | | Video 5G | Yes | Yes | Yes | Yes | ipod-video-5g | 320x240 | | Video 5.5G | Yes | Yes | Yes | Yes | ipod-video-5g | 320x240 | | Classic 6G | Yes | Yes | Yes | Yes | ipod-classic | 320x240 | @@ -293,8 +294,7 @@ iPod_Control/ | Generation | Reason | |------------|--------| | Shuffle 3-4G | Buttonless; writing their iTunesSD playback database is unverified on hardware | -| Nano 6G | Reads and archives (hardware-confirmed); syncing is not supported yet | -| Nano 7G | Reads and archives cleanly (hardware-confirmed); the firmware shows no music for a library libgpod writes | +| Nano 6G | Reads and archives (hardware-confirmed); its library writes are covered only by synthetic tests, not by hardware | | Touch (all) | iOS protocol + cryptographic iTunesDB signing | | iPhone (all) | iOS protocol + cryptographic iTunesDB signing | | iPad | iOS protocol + cryptographic iTunesDB signing | @@ -352,4 +352,4 @@ Fully implemented in podkit. Key implementation files: ## Nano 6G/7G signing The nano 6th and 7th generation sign `iTunesCDB` and `Locations.itdb.cbk` with hashAB, which podkit's libgpod builds compute ([ADR-030](../docs/adr/adr-030-native-nano-hashab.md); checks in `tools/nano7/`). -On a nano 7G the firmware shows no music for a library libgpod writes, so both generations stay read-only. +For them it also writes iTunes' category containers and schema level into the SQLite library and keeps the play data the device records (`tools/nano7/README.md`); the nano 7G syncs, the nano 6G stays read-only. diff --git a/docs/adr/adr-030-native-nano-hashab.md b/docs/adr/adr-030-native-nano-hashab.md index b45ef9a4..8121f9a2 100644 --- a/docs/adr/adr-030-native-nano-hashab.md +++ b/docs/adr/adr-030-native-nano-hashab.md @@ -1,6 +1,6 @@ --- title: 'ADR-030: Link an Open HashAB Signer Into libgpod' -description: Build libgpod with the Unlicense dstaley/hashab C sources instead of loading an external hashab blob, so the iPod nano 6th/7th generation databases can be signed. Signing alone does not make those generations syncable; they stay read-only. +description: Build libgpod with the Unlicense dstaley/hashab C sources instead of loading an external hashab blob, so the iPod nano 6th/7th generation databases can be signed. Signing alone does not make those generations syncable. sidebar: order: 31 --- @@ -68,3 +68,8 @@ firmware shows no music. `nano_6g` and `nano_7g` therefore keep `support.access = 'read-only'`; the nano 6th generation's writes are covered only by synthetic tests. Enabling sync for either generation needs a complete library write that is verified on the device. + +The nano 7th generation became syncable once libgpod also writes the +firmware's category containers and iTunes' schema level and keeps the play +data the device records (`tools/nano7/README.md`); the nano 6th generation +stays read-only. diff --git a/docs/architecture/device/identity-support-matrix.md b/docs/architecture/device/identity-support-matrix.md index 2c6ec6e8..80cf7d61 100644 --- a/docs/architecture/device/identity-support-matrix.md +++ b/docs/architecture/device/identity-support-matrix.md @@ -70,8 +70,8 @@ is the fallback on USB *transport* error | iPod nano 1G–2G | **no (2G does not respond; 1G untested)** | **yes** | SCSI (host only) | | iPod nano 3G | **yes** (first USB-capable) | untested | USB | | iPod nano 4G–5G | **yes** | yes | USB | -| iPod nano 6G | yes (inquiry) | yes | Inquiry works but the generation is **read-only**, hardware-confirmed (reads and archives; syncing is not supported yet) | -| iPod nano 7G | yes (inquiry) | yes | Inquiry works; the generation is **read-only**, hardware-confirmed (reads and archives cleanly; the firmware shows no music for a library libgpod writes) | +| iPod nano 6G | yes (inquiry) | yes | Inquiry works but the generation is **read-only**, hardware-confirmed (reads and archives; its library writes are covered only by synthetic tests) | +| iPod nano 7G | yes (inquiry) | yes | USB | | iPod shuffle 1G–2G | no | **yes** | SCSI (host only) | | iPod shuffle 3G–4G | yes | yes | Inquiry works but sync is **refused** (bdhs iTunesSD write unverified; read-only tier) | | iPod touch / iPhone / iPad | no | no | No disk mode at all — out of scope | diff --git a/docs/formats/generations.md b/docs/formats/generations.md index 4487b8ea..a223818b 100644 --- a/docs/formats/generations.md +++ b/docs/formats/generations.md @@ -69,8 +69,8 @@ after changing the table, then regenerate this block from | iPod nano (3rd Generation) | `nano_3g` | syncable | inferred | — | | iPod nano (4th Generation) | `nano_4g` | syncable | inferred | — | | iPod nano (5th Generation) | `nano_5g` | syncable | inferred | — | -| iPod nano (6th Generation) | `nano_6g` | read-only | hardware | Reads and archives (259 tracks confirmed on hardware); syncing is not supported yet. | -| iPod nano (7th Generation) | `nano_7g` | read-only | hardware | Reads iTunesDB (1,414 tracks confirmed) and archives cleanly; the firmware shows no music for a library libgpod writes, so syncing is not supported yet. | +| iPod nano (6th Generation) | `nano_6g` | read-only | hardware | Reads and archives (259 tracks confirmed on hardware); its library writes are covered only by synthetic tests, not by hardware. | +| iPod nano (7th Generation) | `nano_7g` | syncable | hardware | Syncs (confirmed on hardware, firmware 1.0.4); reads iTunesDB (1,414 tracks confirmed on hardware) and archives cleanly. | | iPod shuffle (1st Generation) | `shuffle_1g` | syncable | inferred | — | | iPod shuffle (2nd Generation) | `shuffle_2g` | syncable | inferred | — | | iPod shuffle (3rd Generation) | `shuffle_3g` | read-only | hardware | Reads iTunesDB; writing the bdhs iTunesSD playback DB is unverified on hardware. | @@ -112,13 +112,9 @@ after changing the table, then regenerate this block from `hardware`-confirmed. See [itunessd-bdhs.md](itunessd-bdhs.md). - **nano 6g — `read-only`, hardware-confirmed.** A real nano 6G read and - archived 259 tracks. It signs its databases with hashAB like the nano 7g; - its writes are covered only by synthetic tests. -- **nano 7g — `read-only`, hardware-confirmed.** libgpod opens its classic - iTunesCDB `iTunesDB` fine — a real nano 7G read 1,414 tracks and archived - cleanly. podkit's libgpod build signs its databases with hashAB - ([ADR-030](../adr/adr-030-native-nano-hashab.md)), but the firmware shows no - music for a library libgpod writes, so writes stay refused. + archived 259 tracks. podkit's libgpod writes its library like the nano 7g's + ([ADR-030](../adr/adr-030-native-nano-hashab.md), `tools/nano7/`), but + only synthetic tests cover that. - **iPod touch (all), not-in-libgpod — `none`.** No mountable database, or Apple's proprietary sync protocol that podkit cannot speak over disk mode. diff --git a/packages/devices-ipod/src/classify.test.ts b/packages/devices-ipod/src/classify.test.ts index 442072f0..bca3d9e5 100644 --- a/packages/devices-ipod/src/classify.test.ts +++ b/packages/devices-ipod/src/classify.test.ts @@ -88,14 +88,14 @@ describe('classifyAsIpod — unsupported iPod-family devices', () => { const result = classifyAsIpod({ vendorId: '05ac', productId: '120d' }); expect(result).not.toBeNull(); expect(result!.supported).toBe(false); - expect(result!.unsupportedReason?.headline).toContain('writing its library is not supported'); + expect(result!.unsupportedReason?.headline).toContain('not verified on hardware'); }); - it('classifies iPod nano 7G as unsupported (0x05ac:0x1267)', () => { + it('classifies iPod nano 7G as supported (0x05ac:0x1267)', () => { const result = classifyAsIpod({ vendorId: '05ac', productId: '1267' }); expect(result).not.toBeNull(); - expect(result!.supported).toBe(false); - expect(result!.unsupportedReason?.headline).toMatch(/nano 7th gen/i); + expect(result!.supported).toBe(true); + expect(result!.unsupportedReason).toBeUndefined(); }); it('classifies iPod touch 5G as unsupported via known table (0x05ac:0x12aa)', () => { diff --git a/packages/devices-ipod/src/identity.test.ts b/packages/devices-ipod/src/identity.test.ts index fe7f200f..53b9307c 100644 --- a/packages/devices-ipod/src/identity.test.ts +++ b/packages/devices-ipod/src/identity.test.ts @@ -155,22 +155,22 @@ describe('identify', () => { }); describe('unsupportedReason — unsupported generations', () => { - test('nano 7G via USB PID 0x120e returns unsupportedReason', () => { - const model = identify({ from: 'usb', productId: '0x120e' }); + test('nano 6G via USB PID 0x1266 returns unsupportedReason', () => { + const model = identify({ from: 'usb', productId: '0x1266' }); expect(model).toBeDefined(); - expect(model!.generationId).toBe('nano_7g'); - expect(model!.unsupportedReason).toBeDefined(); - expect(model!.unsupportedReason!.headline).toMatch(/nano 7th gen/i); + expect(model!.generationId).toBe('nano_6g'); + expect(model!.unsupportedReason!.headline).toMatch(/nano 6th gen/i); // touch_* gets 'ios-device'; everything else gets 'unsupported-device'. expect(model!.unsupportedReason!.kind).toBe('unsupported-device'); expect(model!.unsupportedReason!.docsUrl).toContain('supported-devices'); }); - test('nano 7G via USB PID 0x1267 returns unsupportedReason', () => { - const model = identify({ from: 'usb', productId: '0x1267' }); - expect(model).toBeDefined(); - expect(model!.generationId).toBe('nano_7g'); - expect(model!.unsupportedReason).toBeDefined(); + test('nano 7G via either USB PID is syncable', () => { + for (const productId of ['0x120e', '0x1267']) { + const model = identify({ from: 'usb', productId }); + expect(model!.generationId).toBe('nano_7g'); + expect(model!.unsupportedReason).toBeUndefined(); + } }); test('iPod touch 1G returns unsupportedReason (proprietary protocol)', () => { @@ -228,11 +228,11 @@ describe('identify', () => { expect(model!.unsupportedReason!.headline).toContain('unverified on hardware'); }); - test('nano 6G (0x120d) returns unsupportedReason (writes not supported)', () => { + test('nano 6G (0x120d) returns unsupportedReason (writes not verified)', () => { const model = identify({ from: 'usb', productId: '0x120d' }); expect(model).toBeDefined(); expect(model!.generationId).toBe('nano_6g'); - expect(model!.unsupportedReason!.headline).toContain('writing its library is not supported'); + expect(model!.unsupportedReason!.headline).toContain('not verified on hardware'); }); test('supported devices do NOT have unsupportedReason', () => { diff --git a/packages/devices-ipod/src/resolve.test.ts b/packages/devices-ipod/src/resolve.test.ts index f0e64678..01fdb4f1 100644 --- a/packages/devices-ipod/src/resolve.test.ts +++ b/packages/devices-ipod/src/resolve.test.ts @@ -215,12 +215,11 @@ describe('resolveIpodModel — libgpodGeneration axis', () => { // 'unknown' is skipped by the reverse-index (it's in LIBGPOD_FALLBACK_DISPLAY_NAMES, // but the index IS built from the forward mapping entries — 'unknown' IS in // the index pointing to nano_7g (first entry). Test actual behaviour. - // The lookup returns nano_7g (unsupported). + // The lookup returns nano_7g. if (model !== null) { expect(model.generationId).toBe('nano_7g'); - expect(model.unsupportedReason).toBeDefined(); } - // Both null and a valid unsupported model are acceptable for 'unknown'. + // Both null and a nano_7g model are acceptable for 'unknown'. }); }); @@ -331,14 +330,11 @@ describe('resolveIpodModel — cascade priority', () => { // ============================================================================= describe('resolveIpodModel — unsupported generations', () => { - it('returns a model with unsupportedReason for nano_7g (familyId 18)', () => { + it('returns a syncable model for nano_7g (familyId 18)', () => { const model = resolveIpodModel({ familyId: 18 }); expect(model).not.toBeNull(); expect(model!.generationId).toBe('nano_7g'); - expect(model!.unsupportedReason).toBeDefined(); - expect(model!.unsupportedReason!.headline).toMatch(/nano.*7/i); - expect(model!.unsupportedReason!.kind).toBe('unsupported-device'); - expect(model!.unsupportedReason!.docsUrl).toContain('supported-devices'); + expect(model!.unsupportedReason).toBeUndefined(); }); it('returns a model with unsupportedReason for nano_6g via libgpod axis', () => { diff --git a/packages/devices-ipod/src/support.test.ts b/packages/devices-ipod/src/support.test.ts index f45d1d26..bd807bc1 100644 --- a/packages/devices-ipod/src/support.test.ts +++ b/packages/devices-ipod/src/support.test.ts @@ -41,11 +41,8 @@ describe('resolveGenerationSupport', () => { expect(support.verified).toBe('inferred'); }); - it('reports nano 7g as read-only, hardware-verified', () => { - const support = resolveGenerationSupport('nano_7g'); - expect(support.access).toBe('read-only'); - expect(support.verified).toBe('hardware'); - expect(support.note).toBeDefined(); + it('reports nano 7g as syncable', () => { + expect(resolveGenerationSupport('nano_7g').access).toBe('syncable'); }); it('reports a syncable classic/nano generation as syncable, inferred', () => { diff --git a/packages/devices-ipod/src/tables/generations.ts b/packages/devices-ipod/src/tables/generations.ts index cdc13e81..d9c9d58f 100644 --- a/packages/devices-ipod/src/tables/generations.ts +++ b/packages/devices-ipod/src/tables/generations.ts @@ -14,9 +14,8 @@ * is asserted in `capabilities.test.ts` for every generation that has a * libgpod equivalent. Generations that map to libgpod's `unknown` (nano 7G, * touch 5G–7G) are sourced exclusively from this table for capability data. - * Their `support.access` differs by generation: nano 7G is `read-only` - * (hardware-confirmed readable; library writes not supported yet), touch 5G–7G - * carry `support.access: 'none'` (no disk mode at all). + * Their `support.access` differs by generation: nano 7G is `syncable`, touch + * 5G–7G carry `support.access: 'none'` (no disk mode at all). * * @module */ @@ -196,7 +195,7 @@ export const GENERATIONS: Record = { support: { access: 'read-only', verified: 'hardware', - note: 'Reads and archives (259 tracks confirmed on hardware); syncing is not supported yet.', + note: 'Reads and archives (259 tracks confirmed on hardware); its library writes are covered only by synthetic tests, not by hardware.', }, supportsAlac: false, supportsVideo: false, @@ -208,12 +207,15 @@ export const GENERATIONS: Record = { ordinal: 7, checksumType: 'hashAB', // Read on real hardware through libgpod's iTunesCDB parser (1,414 tracks, - // `device archive` succeeded). Its databases are signed (ADR-030), but the - // firmware shows no music for a library libgpod writes, so writes stay refused. + // `device archive` succeeded). Writes are signed (ADR-030) and give the + // SQLite library iTunes' category containers and schema level, which a + // device on firmware 1.0.4 needed to show music. On that firmware a real + // device showed and played libraries written this way, also after a + // reboot, and kept the plays, skips and ratings it had recorded. support: { - access: 'read-only', + access: 'syncable', verified: 'hardware', - note: 'Reads iTunesDB (1,414 tracks confirmed) and archives cleanly; the firmware shows no music for a library libgpod writes, so syncing is not supported yet.', + note: 'Syncs (confirmed on hardware, firmware 1.0.4); reads iTunesDB (1,414 tracks confirmed on hardware) and archives cleanly.', }, supportsAlac: true, supportsVideo: true, diff --git a/packages/devices-ipod/src/tables/unsupported.test.ts b/packages/devices-ipod/src/tables/unsupported.test.ts index de78df83..3aa77326 100644 --- a/packages/devices-ipod/src/tables/unsupported.test.ts +++ b/packages/devices-ipod/src/tables/unsupported.test.ts @@ -36,12 +36,9 @@ describe('lookupUnsupportedReason — nano 6G / 7G', () => { expect(lookupUnsupportedReason('1266')).toContain('nano 6th gen'); }); - test('0x120e iPod nano 7G (0x120x range)', () => { - expect(lookupUnsupportedReason('120e')).toContain('nano 7th gen'); - }); - - test('0x1267 iPod nano 7G (0x126x range)', () => { - expect(lookupUnsupportedReason('1267')).toContain('nano 7th gen'); + test('iPod nano 7G is not listed (syncable)', () => { + expect(lookupUnsupportedReason('120e')).toBeNull(); + expect(lookupUnsupportedReason('1267')).toBeNull(); }); }); diff --git a/packages/devices-ipod/src/tables/unsupported.ts b/packages/devices-ipod/src/tables/unsupported.ts index 575f307d..bf99a691 100644 --- a/packages/devices-ipod/src/tables/unsupported.ts +++ b/packages/devices-ipod/src/tables/unsupported.ts @@ -15,10 +15,8 @@ import { SUPPORTED_DEVICES_DOCS_URL } from '../build-unsupported-reason.js'; * Unsupported categories: * - Shuffle 3G/4G: readable, but writing their `bdhs` playback database has * never been verified on hardware, so podkit does not attempt it. - * - Nano 6G: readable; syncing is not supported yet. - * - Nano 7G: readable — libgpod opens its iTunesCDB fine (hardware-confirmed, - * 1,414 tracks). Its databases are signed (ADR-030), but the firmware shows no - * music for a library libgpod writes. + * - Nano 6G: readable; its library writes are covered only by synthetic tests, + * not by hardware. * - iPod touch (all generations): Apple's proprietary sync protocol; no disk mode. * - iPhone / iPad: Apple's proprietary sync protocol; no disk mode. * - Apple TV, Apple Watch, HomePod: non-iPod Apple USB devices; out of scope. @@ -46,12 +44,7 @@ const SHUFFLE_REASON = 'iPod shuffle 3rd/4th gen can be read but not written: writing its iTunesSD playback database is unverified on hardware.'; const NANO_6G_REASON = - 'iPod nano 6th gen can be read and archived, but not synced yet: writing its library is not supported.'; - -// Valid signatures are not enough: the firmware shows no music for a library -// libgpod writes (ADR-030). -const NANO_7G_REASON = - 'iPod nano 7th gen can be read and archived, but not synced yet: its firmware shows no music for a library podkit writes.'; + 'iPod nano 6th gen can be read and archived, but not synced yet: its library writes are not verified on hardware.'; const itouch = (gen: string) => `iPod touch (${gen}) uses Apple's proprietary sync protocol; podkit only supports iPod disk mode.`; @@ -76,11 +69,6 @@ export const UNSUPPORTED_IPOD_PRODUCT_IDS: Record = { '120d': NANO_6G_REASON, '1266': NANO_6G_REASON, - // ── iPod nano 7G ─────────────────────────────────────────────────────────── - // Not in libgpod 0.8.3 ipod_info_table. USB IDs from linux-usb.org + IPOD_USB_IDS. - '120e': NANO_7G_REASON, - '1267': NANO_7G_REASON, - // ── iPod touch (all generations) ─────────────────────────────────────────── // Apple's proprietary sync protocol; no disk mode on any touch generation. // libgpod has ipod_info_table entries for 1G–4G but they are unreachable diff --git a/packages/devices-ipod/src/tables/usb-ids.ts b/packages/devices-ipod/src/tables/usb-ids.ts index 5e6e1d62..2cca397f 100644 --- a/packages/devices-ipod/src/tables/usb-ids.ts +++ b/packages/devices-ipod/src/tables/usb-ids.ts @@ -14,7 +14,7 @@ * Both ranges map to the same generations. DFU/WTF mode IDs (0x1223, 0x1224, etc.) * are intentionally excluded -- those are recovery-mode endpoints, not disk-mode devices. * - * Devices that podkit cannot sync (e.g. nano 6G, nano 7G, all Touch generations) + * Devices that podkit cannot sync (e.g. nano 6G, all Touch generations) * are still listed here for identification purposes. The access tier lives on * `IpodGeneration.support` (see `tables/generations.ts`) and the canonical * PID-keyed reason map is `tables/unsupported.ts`. diff --git a/packages/devices-ipod/src/types.ts b/packages/devices-ipod/src/types.ts index 84f401d7..bd2d0943 100644 --- a/packages/devices-ipod/src/types.ts +++ b/packages/devices-ipod/src/types.ts @@ -105,9 +105,7 @@ export interface IpodGeneration { * - shuffle_3g/4g: `read-only` — readable iTunesDB, but writing their * `bdhs` iTunesSD has never been confirmed on hardware. * - nano_6g: `read-only` — reads and archives (hardware-confirmed); - * syncing is not supported yet. - * - nano_7g: `read-only` — reads its iTunesDB fine (hardware-confirmed); - * the firmware shows no music for a library libgpod writes. + * its library writes are covered only by synthetic tests. * - touch/iPhone/iPad, not-in-libgpod: `none` — no mountable database or a * protocol podkit cannot speak. */ diff --git a/packages/docs-site/src/content/docs/devices/supported-devices.mdx b/packages/docs-site/src/content/docs/devices/supported-devices.mdx index 3a3c7dd7..650f882a 100644 --- a/packages/docs-site/src/content/docs/devices/supported-devices.mdx +++ b/packages/docs-site/src/content/docs/devices/supported-devices.mdx @@ -111,8 +111,8 @@ podkit uses [libgpod](https://github.com/fadingred/libgpod) for iPod database ma iPod support comes in three tiers, because reading a device and writing to it are different operations: -- **Full** — podkit reads *and* syncs the device. This is every classic, mini, nano (1st–5th gen), photo, video, and shuffle (1st–2nd gen) iPod. -- **Read-only** — podkit reads the library and can [archive](/user-guide/devices/archive) it, but **cannot sync**. The buttonless shuffles (3rd/4th gen) keep a libgpod-readable `iTunesDB`, but writing the `iTunesSD` playback database their firmware plays from has never been shown to leave such a device playable. The nano 6th and 7th gen read and archive cleanly; podkit's libgpod build can sign their databases (hashAB), but complete library writes are not supported yet. Reading and archiving these is entirely safe; only writes are refused. +- **Full** — podkit reads *and* syncs the device. This is every classic, mini, nano (1st–5th and 7th gen), photo, video, and shuffle (1st–2nd gen) iPod. +- **Read-only** — podkit reads the library and can [archive](/user-guide/devices/archive) it, but **cannot sync**. The buttonless shuffles (3rd/4th gen) keep a libgpod-readable `iTunesDB`, but writing the `iTunesSD` playback database their firmware plays from has never been shown to leave such a device playable. The nano 6th gen reads and archives cleanly; podkit's libgpod writes its library like the nano 7th gen's, but that is covered only by synthetic tests, not by a device. Reading and archiving these is entirely safe; only writes are refused. - **Unsupported** — podkit cannot use the device at all. iOS devices (iPod Touch, iPhone, iPad) don't mount as mass storage. ### How We Test iPods @@ -158,7 +158,7 @@ Reach for `podkit device archive` to preserve one of these devices: podkit device archive -d "/Volumes/IPOD" ``` -Why read but not write? The buttonless shuffles (3rd/4th gen) keep a libgpod-readable `iTunesDB` alongside the `iTunesSD` playback database the firmware actually plays from. Reading the `iTunesDB` is fine; writing the `iTunesSD` is not blocked by anything cryptographic, but no such write has been confirmed to leave a playable device, so podkit does not attempt it. The nano 6th and 7th gen read fine (confirmed on real hardware: 259 and 1,414 tracks). Their databases carry a hashAB signature, which podkit's libgpod build now computes, but signing alone is not enough: the nano 7th gen firmware shows no music for a library written by libgpod, and the nano 6th gen write path is only covered by synthetic tests. So, same as the shuffle, the read is permitted and the write is refused. +Why read but not write? The buttonless shuffles (3rd/4th gen) keep a libgpod-readable `iTunesDB` alongside the `iTunesSD` playback database the firmware actually plays from. Reading the `iTunesDB` is fine; writing the `iTunesSD` is not blocked by anything cryptographic, but no such write has been confirmed to leave a playable device, so podkit does not attempt it. The nano 6th gen reads fine (confirmed on real hardware: 259 tracks). Its databases carry a hashAB signature, which podkit's libgpod build computes, and podkit's libgpod writes its library like the nano 7th gen's. That has only been checked with synthetic tests, not on a nano 6th gen, so, same as the shuffle, the read is permitted and the write is refused. ### Unsupported iPods @@ -193,6 +193,7 @@ libgpod cannot implement iTunes' signing mechanism, making iOS device support te | Mini | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | | Nano 1st-2nd | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | | Nano 3rd-5th | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Nano 7th | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | | Shuffle 1st-2nd | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | The ALAC column indicates Apple Lossless support. The `max` quality preset uses this to automatically choose between ALAC (lossless) and high-quality AAC based on device capability. diff --git a/packages/podkit-cli/src/commands/device/capability-summary.test.ts b/packages/podkit-cli/src/commands/device/capability-summary.test.ts index 641a319b..09ee075c 100644 --- a/packages/podkit-cli/src/commands/device/capability-summary.test.ts +++ b/packages/podkit-cli/src/commands/device/capability-summary.test.ts @@ -264,16 +264,16 @@ describe('confirmUnsupportedDeviceAdd', () => { ): IpodIdentityAssessment { return { model: { - displayName: 'iPod nano (7th Generation)', - generationId: 'nano_7g', + displayName: 'iPod nano (6th Generation)', + generationId: 'nano_6g', family: 'iPod nano', - ordinal: 7, + ordinal: 6, checksumType: 'hashAB', source: 'usb', unsupportedReason: { kind: 'unsupported-device', headline: - 'iPod nano (7th Generation) is not supported by podkit (this generation cannot sync).', + 'iPod nano (6th Generation) is not supported by podkit (this generation cannot sync).', docsUrl: 'https://jvgomg.github.io/podkit/devices/supported-devices', }, ...overrides, @@ -337,7 +337,7 @@ describe('confirmUnsupportedDeviceAdd', () => { expect(decision).toBe('add-anyway'); expect(calls).toBe(0); // Canonical message is rendered to stderr (warn) regardless of autoConfirm. - expect(stderr.text()).toContain('iPod nano (7th Generation) is not supported'); + expect(stderr.text()).toContain('iPod nano (6th Generation) is not supported'); }); it('returns "cancelled" when user declines the prompt', async () => { diff --git a/packages/podkit-cli/src/commands/doctor-read-only.test.ts b/packages/podkit-cli/src/commands/doctor-read-only.test.ts index 18eb6d0b..7806abbb 100644 --- a/packages/podkit-cli/src/commands/doctor-read-only.test.ts +++ b/packages/podkit-cli/src/commands/doctor-read-only.test.ts @@ -1,7 +1,7 @@ /** * `podkit doctor` on a read-only device. * - * A read-only generation (shuffle 3G/4G, nano 6G/7G) is readable: podkit + * A read-only generation (shuffle 3G/4G, nano 6G) is readable: podkit * lists its tracks and archives it. Diagnosing it is a read too, so doctor * runs its whole read-only surface — host checks, the readiness cascade, and * the database-health checks — and reports what it finds. @@ -56,10 +56,10 @@ interface FakeStage { details?: Record; } -const NANO_7G_REASON = { +const NANO_6G_REASON = { kind: 'unsupported-device' as const, headline: - 'iPod nano 7th gen can be read and archived, but not synced: writing needs the hashAB signature.', + 'iPod nano 6th gen can be read and archived, but not synced: its library writes are not verified on hardware.', docsUrl: 'https://jvgomg.github.io/podkit/devices/supported-devices/', }; @@ -224,7 +224,7 @@ function makeFakeCore(opts: FakeCoreOptions): unknown { usbModel: { generationId: opts.generationId, displayName: 'Test iPod', - ...(opts.generationId === 'nano_7g' ? { unsupportedReason: NANO_7G_REASON } : {}), + ...(opts.generationId === 'nano_6g' ? { unsupportedReason: NANO_6G_REASON } : {}), }, }; }, @@ -301,10 +301,10 @@ describe('doctor on a read-only device', () => { let seen: string | undefined = 'never-called'; const { out } = makeOut('json'); await runDoctor( - makeContext('nano7g'), + makeContext('nano6g'), out, makeFakeCore({ - generationId: 'nano_7g', + generationId: 'nano_6g', captureRequiredAccess: (requiredAccess) => { seen = requiredAccess; }, @@ -317,10 +317,10 @@ describe('doctor on a read-only device', () => { it('runs its checks and reports the device healthy instead of refusing', async () => { const { out, stdout, exitCode } = makeOut('json'); await runDoctor( - makeContext('nano7g'), + makeContext('nano6g'), out, makeFakeCore({ - generationId: 'nano_7g', + generationId: 'nano_6g', checks: [ check({ id: 'codec-encoders', status: 'pass', scope: 'system' }), check({ id: 'artwork-rebuild', status: 'pass' }), @@ -344,28 +344,28 @@ describe('doctor on a read-only device', () => { it('marks the access tier in the JSON envelope', async () => { const { out, stdout } = makeOut('json'); - await runDoctor(makeContext('nano7g'), out, makeFakeCore({ generationId: 'nano_7g' })); + await runDoctor(makeContext('nano6g'), out, makeFakeCore({ generationId: 'nano_6g' })); expect(stdout.json().access).toBe('read-only'); }); it('announces the tier and why no repairs are offered', async () => { const { out, stdout } = makeOut('text'); - await runDoctor(makeContext('nano7g'), out, makeFakeCore({ generationId: 'nano_7g' })); + await runDoctor(makeContext('nano6g'), out, makeFakeCore({ generationId: 'nano_6g' })); const text = stdout.text(); expect(text).toContain('Read-only device'); - expect(text).toContain(NANO_7G_REASON.headline); + expect(text).toContain(NANO_6G_REASON.headline); expect(text).toContain('podkit device archive'); }); it('reports a repairable finding but withholds the repair command', async () => { const { out, stdout, stderr, exitCode } = makeOut('text'); await runDoctor( - makeContext('nano7g'), + makeContext('nano6g'), out, makeFakeCore({ - generationId: 'nano_7g', + generationId: 'nano_6g', checks: [ check({ id: 'artwork-rebuild', @@ -393,9 +393,9 @@ describe('doctor on a read-only device', () => { it('withholds readiness-stage fixes that write to the device', async () => { const { out, stdout, stderr } = makeOut('text'); await runDoctor( - makeContext('nano7g'), + makeContext('nano6g'), out, - makeFakeCore({ generationId: 'nano_7g', stages: SYSINFO_WARNS }) + makeFakeCore({ generationId: 'nano_6g', stages: SYSINFO_WARNS }) ); const text = `${stdout.text()}\n${stderr.text()}`; @@ -408,11 +408,11 @@ describe('doctor on a read-only device', () => { describe('doctor --repair on a read-only device', () => { it('is refused before any device handle is opened', async () => { - // D476 = iPod nano (7th generation) — a real read-only generation, so + // C526 = iPod nano (6th generation) — a real read-only generation, so // the refusal is pinned to the access tier rather than to a hand-written // reason string. - const model = identify({ from: 'sysinfo', modelNumStr: 'D476' }); - expect(model?.generationId).toBe('nano_7g'); + const model = identify({ from: 'sysinfo', modelNumStr: 'C526' }); + expect(model?.generationId).toBe('nano_6g'); let opened = false; let repaired = false; @@ -447,7 +447,7 @@ describe('doctor --repair on a read-only device', () => { repairableCheck as unknown as Parameters[1], {}, out, - makeContext('nano7g').config, + makeContext('nano6g').config, // No `assessIpodIdentity` override — the refusal must come from the // core surface the production path uses. { loadCore: async () => core as unknown as typeof import('@podkit/core') } diff --git a/packages/podkit-cli/src/commands/sync-runner.unit.test.ts b/packages/podkit-cli/src/commands/sync-runner.unit.test.ts index d6b9779d..d9149bb8 100644 --- a/packages/podkit-cli/src/commands/sync-runner.unit.test.ts +++ b/packages/podkit-cli/src/commands/sync-runner.unit.test.ts @@ -209,15 +209,15 @@ describe('runSync: validation + deps seam', () => { ...real, assessIpodIdentity: async () => ({ model: { - displayName: 'iPod nano (7th Generation)', - generationId: 'nano_7g', + displayName: 'iPod nano (6th Generation)', + generationId: 'nano_6g', family: 'iPod nano', - ordinal: 7, + ordinal: 6, checksumType: 'hashAB', source: 'usb', unsupportedReason: { kind: 'unsupported-device', - headline: 'iPod nano (7th Generation) is not supported by podkit.', + headline: 'iPod nano (6th Generation) is not supported by podkit.', docsUrl: 'https://jvgomg.github.io/podkit/devices/supported-devices', }, }, @@ -242,10 +242,10 @@ describe('runSync: validation + deps seam', () => { expect(exitCode.get()).toBe(1); const err = stdout.json(); expect(err.code).toBe(SyncErrorCodes.DEVICE_UNSUPPORTED); - expect(err.error).toContain('iPod nano (7th Generation) is not supported'); + expect(err.error).toContain('iPod nano (6th Generation) is not supported'); // Error wording must surface the unsupported-device reason without // mentioning libgpod; the latter is an internal detail and would - // confuse end users running into this on an iPod nano 7g. + // confuse end users running into this on an iPod nano 6g. expect(err.error.toLowerCase()).not.toContain('libgpod'); // No track plan generated. expect(openedDevice).toBe(false); diff --git a/packages/podkit-core/src/device/discovery-permutations.test.ts b/packages/podkit-core/src/device/discovery-permutations.test.ts index 754be2dd..2b9527c5 100644 --- a/packages/podkit-core/src/device/discovery-permutations.test.ts +++ b/packages/podkit-core/src/device/discovery-permutations.test.ts @@ -76,11 +76,10 @@ const IPOD_NANO_4G: EnumeratedUsbDevice = { productId: '1263', }; -/** ipod-nano-7g-space-gray persona — unsupported nano 7G (hashAB / no libgpod entry). */ -const IPOD_NANO_7G: EnumeratedUsbDevice = { +/** nano 6G — unsupported for sync (read-only generation). */ +const IPOD_NANO_6G: EnumeratedUsbDevice = { vendorId: '05ac', - productId: '1267', - serialNumber: 'DCYN72R8FJQ1', + productId: '1266', }; /** ipod-mini-2g-pink persona — supported mini (mini_1g via PID 0x1205). */ @@ -253,13 +252,13 @@ describe('multiple iPod descriptors classified in stable order', () => { it('mixed supported / unsupported iPods preserved in input order', () => { const enumerated: EnumeratedUsbDevice[] = [ IPOD_NANO_4G, // supported - IPOD_NANO_7G, // unsupported (nano_7g) + IPOD_NANO_6G, // unsupported (nano_6g) IPOD_MINI, // supported IPOD_TOUCH_5G, // unsupported (iOS-range) ]; const classified = classifyUsbDevices(enumerated); expect(classified).toHaveLength(4); - expect(classified.map((c) => c.device.productId)).toEqual(['1263', '1267', '1205', '12aa']); + expect(classified.map((c) => c.device.productId)).toEqual(['1263', '1266', '1205', '12aa']); expect(classified[0]!.kind === 'ipod' && classified[0]!.supported).toBe(true); expect(classified[1]!.kind === 'ipod' && classified[1]!.supported).toBe(false); expect(classified[2]!.kind === 'ipod' && classified[2]!.supported).toBe(true); @@ -349,10 +348,10 @@ describe('missing serialNumber is omitted (not "" / not null)', () => { // --------------------------------------------------------------------------- describe('realistic mixed-bus snapshot', () => { - it('classifies a representative mix: 5G + unsupported nano 7G + Sony + Echo Mini + Cruzer + Apple unknown', () => { + it('classifies a representative mix: 5G + unsupported nano 6G + Sony + Echo Mini + Cruzer + Apple unknown', () => { const enumerated: EnumeratedUsbDevice[] = [ IPOD_VIDEO_5G, // → supported iPod - IPOD_NANO_7G, // → unsupported iPod + IPOD_NANO_6G, // → unsupported iPod SONY_WALKMAN, // → kind=unsupported (vendor-recognised) ECHO_MINI, // → kind=mass-storage SANDISK_CRUZER, // → kind=unsupported (vendor-recognised, no preset) diff --git a/packages/podkit-core/src/device/readiness/__tests__/check-readiness-read-only.test.ts b/packages/podkit-core/src/device/readiness/__tests__/check-readiness-read-only.test.ts index f4083397..06df4a8f 100644 --- a/packages/podkit-core/src/device/readiness/__tests__/check-readiness-read-only.test.ts +++ b/packages/podkit-core/src/device/readiness/__tests__/check-readiness-read-only.test.ts @@ -1,7 +1,7 @@ /** * `checkReadiness()` on a read-only generation. * - * A `read-only` generation (shuffle 3G/4G, nano 6G/7G) refuses writes but + * A `read-only` generation (shuffle 3G/4G, nano 6G) refuses writes but * reads its `iTunesDB` fine. The cascade's unsupported short-circuit is * therefore intent-dependent: a write caller (sync, init) is refused up * front, while a read caller (doctor's diagnostics) runs the whole cascade @@ -24,8 +24,8 @@ import type { IpodClassification } from '@podkit/devices-ipod'; import { classifyAsIpod } from '@podkit/devices-ipod'; import type { EnumeratedUsbDevice } from '../../usb-enumeration.js'; -/** nano 7G — `read-only`: libgpod parses its database, hashAB signing is unavailable. */ -const NANO_7G_PID = '120e'; +/** nano 6G — `read-only`: libgpod parses its database, writes are unverified on hardware. */ +const NANO_6G_PID = '120d'; /** iPod touch 5G — `none`: no disk mode at all. */ const TOUCH_5G_PID = '12aa'; @@ -66,11 +66,11 @@ describe('checkReadiness() — read-only generations', () => { beforeEach(() => { mountPoint = fs.mkdtempSync(path.join(os.tmpdir(), 'podkit-readiness-read-only-')); // Minimal iPod layout so the mount stage passes and the cascade can - // reach sysinfo / database. D476 = iPod nano (7th generation). + // reach sysinfo / database. C526 = iPod nano (6th generation). const deviceDir = path.join(mountPoint, 'iPod_Control', 'Device'); fs.mkdirSync(deviceDir, { recursive: true }); fs.mkdirSync(path.join(mountPoint, 'iPod_Control', 'Music'), { recursive: true }); - fs.writeFileSync(path.join(deviceDir, 'SysInfo'), 'ModelNumStr: D476\n', 'utf-8'); + fs.writeFileSync(path.join(deviceDir, 'SysInfo'), 'ModelNumStr: C526\n', 'utf-8'); }); afterEach(() => { @@ -79,7 +79,7 @@ describe('checkReadiness() — read-only generations', () => { it('runs the whole cascade for a read intent instead of refusing up front', async () => { const result = await checkReadiness({ - device: makeArm(mountPoint, classify(NANO_7G_PID)), + device: makeArm(mountPoint, classify(NANO_6G_PID)), requiredAccess: 'read', }); @@ -95,13 +95,13 @@ describe('checkReadiness() — read-only generations', () => { expect(byStage.get('database')?.status).not.toBe('skip'); // The read-only tier is still discoverable by the caller — the model // carries the rejection reason even though the cascade continued. - expect(result.usbModel?.generationId).toBe('nano_7g'); - expect(result.usbModel?.unsupportedReason?.headline).toContain('nano 7th gen'); + expect(result.usbModel?.generationId).toBe('nano_6g'); + expect(result.usbModel?.unsupportedReason?.headline).toContain('nano 6th gen'); }); it('refuses a read-only generation for a write intent (the default)', async () => { const result = await checkReadiness({ - device: makeArm(mountPoint, classify(NANO_7G_PID)), + device: makeArm(mountPoint, classify(NANO_6G_PID)), }); expect(result.level).toBe('unsupported'); @@ -131,7 +131,7 @@ describe('checkReadiness() — read-only generations', () => { requiredAccess: 'read', }); - expect(result.deviceModel?.generationId).toBe('nano_7g'); + expect(result.deviceModel?.generationId).toBe('nano_6g'); expect(result.level).not.toBe('unsupported'); const byStage = new Map(result.stages.map((s) => [s.stage, s])); // The database stage ran for real: no libgpod-parsable database exists in diff --git a/packages/podkit-core/src/device/readiness/__tests__/check-readiness-unsupported.test.ts b/packages/podkit-core/src/device/readiness/__tests__/check-readiness-unsupported.test.ts index a082bb36..4462229a 100644 --- a/packages/podkit-core/src/device/readiness/__tests__/check-readiness-unsupported.test.ts +++ b/packages/podkit-core/src/device/readiness/__tests__/check-readiness-unsupported.test.ts @@ -172,15 +172,15 @@ describe('checkReadiness() — post-sysinfo unsupported short-circuit (block-onl }); it('short-circuits to unsupported when sysinfo identifies an unsupported model, even without USB classification', async () => { - // D476 = iPod nano 16GB Yellow (7th Generation) — nano_7g has supported:false - // in the generations table. `identify({ from: 'sysinfo', modelNumStr: 'D476' })` + // C526 = iPod nano 16GB Silver (6th Generation) — nano_6g has supported:false + // in the generations table. `identify({ from: 'sysinfo', modelNumStr: 'C526' })` // sets `unsupportedReason` on the returned IpodModel. // // The block is mounted with a valid iPod_Control directory and a SysInfo // file naming this model. No `usb` field — simulates the doctor fallback // path (`ipodFromBlock`). The pipeline must short-circuit after sysinfo // instead of opening the database. - writeDeviceFiles({ SysInfo: 'ModelNumStr: D476\nVisibleBuildID: 1.0.4\n' }); + writeDeviceFiles({ SysInfo: 'ModelNumStr: C526\nVisibleBuildID: 1.2\n' }); const result = await checkReadiness({ device: { @@ -188,8 +188,8 @@ describe('checkReadiness() — post-sysinfo unsupported short-circuit (block-onl matchedBy: 'block-only', block: { identifier: 'disk9s2', - volumeName: 'NANO7G', - volumeUuid: 'NANO-7G-UUID', + volumeName: 'NANO6G', + volumeUuid: 'NANO-6G-UUID', storage: { sizeBytes: 16_000_000_000 }, isMounted: true, mountPoint, @@ -202,7 +202,7 @@ describe('checkReadiness() — post-sysinfo unsupported short-circuit (block-onl expect(result.unsupported?.headline).toContain('nano'); // deviceModel must be populated (identifies the refused generation) expect(result.deviceModel).toBeDefined(); - expect(result.deviceModel?.generationId).toBe('nano_7g'); + expect(result.deviceModel?.generationId).toBe('nano_6g'); // The database stage must be present but SKIPPED — the pipeline short-circuits // after sysinfo and appends skip rows for remaining stages (same pattern as // the USB-arm short-circuit). The database must NOT have run (status 'fail' diff --git a/packages/podkit-core/src/device/readiness/__tests__/determine-level.test.ts b/packages/podkit-core/src/device/readiness/__tests__/determine-level.test.ts index 2d4e72ea..45fb22f5 100644 --- a/packages/podkit-core/src/device/readiness/__tests__/determine-level.test.ts +++ b/packages/podkit-core/src/device/readiness/__tests__/determine-level.test.ts @@ -50,11 +50,11 @@ describe('determineLevel() — unsupported short-circuit', () => { expect(result4g.unsupported?.kind).toBe('unsupported-device'); }); - it('returns unsupported for nano 7G PIDs (read-only: hashAB signing unavailable)', () => { - const result = determineLevel([], { vendorId: '05ac', productId: '120e' }); + it('returns unsupported for nano 6G PIDs (read-only: writes unverified on hardware)', () => { + const result = determineLevel([], { vendorId: '05ac', productId: '120d' }); expect(result.level).toBe('unsupported'); expect(result.unsupported?.kind).toBe('unsupported-device'); - expect(result.unsupported?.headline).toContain('nano 7th gen'); + expect(result.unsupported?.headline).toContain('nano 6th gen'); }); it('returns unsupported for iOS-range PIDs without explicit table entry', () => { diff --git a/tools/nano7/README.md b/tools/nano7/README.md index bef0fd3b..5ba94323 100644 --- a/tools/nano7/README.md +++ b/tools/nano7/README.md @@ -4,9 +4,11 @@ The iPod nano 6th and 7th generation sign `iTunesCDB` and `iTunes Library.itlp/Locations.itdb.cbk` with HashAB. podkit's libgpod builds link the vendored signer from `tools/prebuild/vendor/hashab` through `tools/prebuild/libgpod-nano.patch` (see -[ADR-030](../../docs/adr/adr-030-native-nano-hashab.md)). Both generations stay -**read-only** in podkit: the nano 7th generation firmware shows no music for a -library libgpod writes, although its signatures verify. +[ADR-030](../../docs/adr/adr-030-native-nano-hashab.md)). Valid signatures +alone do not make a library acceptable to the nano 7th generation firmware; +the patch also gives the SQLite library iTunes' layout (see below), and podkit +syncs that generation. The nano 6th generation stays **read-only**: its writes +are covered only by synthetic tests. These tools need Python 3.11+ and a C compiler, and no pip or npm packages. From 0bcf4fe4f9251f971d05e5e19f5929283d31726f Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 19:00:38 +0100 Subject: [PATCH 11/21] fix(transcode): enforce the device's H.264 profile and level for passthrough checkVideoCompatibility allowed passthrough without comparing the source's H.264 level with the device's, and the sync classifier built its analysis from CollectionVideo with profile, level, bitrate and frame rate blanked, so only container, codec and resolution were checked there. A known level above the device's, or an unknown level, is now a transcode reason. CollectionVideo carries the probed profile, level, bitrate and frame rate, and the classifier passes them on. With the profile now reaching the check, only Baseline (including Constrained Baseline) and, on Main devices, Main pass; High 10 and every other reported profile is transcoded instead of falling through as compatible. A source without a reported profile is still judged by its level alone, as before. The VideoToolbox path gets -level from the profile, as libx264 already did. --- .changeset/video-level-limit.md | 8 ++++ .../content/docs/user-guide/syncing/video.md | 2 +- .../src/sync/video/classifier.test.ts | 1 + .../podkit-core/src/sync/video/classifier.ts | 8 ++-- .../src/video/compatibility.test.ts | 40 +++++++++++++++++-- .../podkit-core/src/video/compatibility.ts | 34 +++++++++++----- .../src/video/directory-adapter.ts | 16 ++++++++ .../podkit-core/src/video/transcode.test.ts | 1 + packages/podkit-core/src/video/transcode.ts | 2 + 9 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 .changeset/video-level-limit.md diff --git a/.changeset/video-level-limit.md b/.changeset/video-level-limit.md new file mode 100644 index 00000000..f5ca82eb --- /dev/null +++ b/.changeset/video-level-limit.md @@ -0,0 +1,8 @@ +--- +"podkit": patch +"@podkit/core": patch +--- + +Keep video within the device's H.264 profile and level + +A video was copied to the iPod untouched when its container, codec, profile and resolution fit, without looking at its H.264 level, and the video sync path did not pass the probed profile, level, bitrate or frame rate to that check at all. A level above the device's (3.0 for the 320x240 iPods, 3.1 for the Classic) or an unknown level now means the file is transcoded, and the probed bitrate and frame rate limits apply too. When the profile is reported, only Baseline, and Main on the Classic, is copied; any other profile, such as High 10, is transcoded. The macOS VideoToolbox encoder is now given the level constraint as well as the profile. diff --git a/packages/docs-site/src/content/docs/user-guide/syncing/video.md b/packages/docs-site/src/content/docs/user-guide/syncing/video.md index 81658c52..37596670 100644 --- a/packages/docs-site/src/content/docs/user-guide/syncing/video.md +++ b/packages/docs-site/src/content/docs/user-guide/syncing/video.md @@ -32,7 +32,7 @@ Mass-storage DAPs (Echo Mini, Rockbox, generic) do not support video sync. If yo | WebM | `.webm` | VP8/VP9 transcoded to H.264 | | WMV | `.wmv` | Windows Media | -All videos are transcoded to iPod-compatible M4V (H.264 video, AAC audio). Files that are already compatible are copied directly without re-encoding. See [Video Transcoding](/user-guide/transcoding/video) for quality presets and device profiles. +All videos are transcoded to iPod-compatible M4V (H.264 video, AAC audio). Files that are already compatible are copied directly without re-encoding: H.264 in a profile and **level** the device decodes (a file whose level is unknown is transcoded), within the device's resolution, bitrate and frame rate, with AAC audio. See [Video Transcoding](/user-guide/transcoding/video) for quality presets and device profiles. ## Supported Content Types diff --git a/packages/podkit-core/src/sync/video/classifier.test.ts b/packages/podkit-core/src/sync/video/classifier.test.ts index 6e8c7a04..3a341b5a 100644 --- a/packages/podkit-core/src/sync/video/classifier.test.ts +++ b/packages/podkit-core/src/sync/video/classifier.test.ts @@ -17,6 +17,7 @@ function makeCollectionVideo(overrides: Partial = {}): Collecti year: 2024, container: 'mkv', videoCodec: 'h264', + videoLevel: '3.0', audioCodec: 'aac', width: 1920, height: 1080, diff --git a/packages/podkit-core/src/sync/video/classifier.ts b/packages/podkit-core/src/sync/video/classifier.ts index 8c9cff70..134893fd 100644 --- a/packages/podkit-core/src/sync/video/classifier.ts +++ b/packages/podkit-core/src/sync/video/classifier.ts @@ -62,12 +62,12 @@ function toSourceAnalysis(video: CollectionVideo): VideoSourceAnalysis { filePath: video.filePath, container: video.container, videoCodec: video.videoCodec, - videoProfile: null, // Not available in CollectionVideo - videoLevel: null, + videoProfile: video.videoProfile ?? null, + videoLevel: video.videoLevel ?? null, width: video.width, height: video.height, - videoBitrate: 0, // Not available in CollectionVideo - frameRate: 0, // Not available in CollectionVideo + videoBitrate: video.videoBitrate ?? 0, + frameRate: video.frameRate ?? 0, audioCodec: video.audioCodec, audioBitrate: 0, audioChannels: 2, // Assume stereo diff --git a/packages/podkit-core/src/video/compatibility.test.ts b/packages/podkit-core/src/video/compatibility.test.ts index 50ac3e45..055fda58 100644 --- a/packages/podkit-core/src/video/compatibility.test.ts +++ b/packages/podkit-core/src/video/compatibility.test.ts @@ -50,6 +50,7 @@ describe('video/compatibility helpers', () => { it('returns true for H.264 baseline on all devices', () => { expect(isCompatibleVideoCodec('h264', 'baseline', ipodClassic)).toBe(true); expect(isCompatibleVideoCodec('h264', 'baseline', ipodVideo5g)).toBe(true); + expect(isCompatibleVideoCodec('h264', 'Constrained Baseline', ipodVideo5g)).toBe(true); }); it('returns true for H.264 main on devices that support main profile', () => { @@ -60,9 +61,11 @@ describe('video/compatibility helpers', () => { expect(isCompatibleVideoCodec('h264', 'main', ipodVideo5g)).toBe(false); }); - it('returns false for H.264 high profile on all devices', () => { - expect(isCompatibleVideoCodec('h264', 'high', ipodClassic)).toBe(false); - expect(isCompatibleVideoCodec('h264', 'high', ipodVideo5g)).toBe(false); + it('returns false for High-family and other unsupported profiles on all devices', () => { + for (const profile of ['high', 'High 10', 'High 4:4:4 Predictive', 'Extended']) { + expect(isCompatibleVideoCodec('h264', profile, ipodClassic)).toBe(false); + expect(isCompatibleVideoCodec('h264', profile, ipodVideo5g)).toBe(false); + } }); it('returns true for H.264 with no profile info (assume compatible)', () => { @@ -159,6 +162,7 @@ describe('video/compatibility passthrough', () => { container: 'mp4', videoCodec: 'h264', videoProfile: 'baseline', + videoLevel: '3.0', width: 320, height: 240, videoBitrate: 600, @@ -171,6 +175,36 @@ describe('video/compatibility passthrough', () => { expect(result.reasons).toHaveLength(0); }); + it('transcode: H.264 level above the device maximum', () => { + const analysis = createAnalysis({ + videoProfile: 'baseline', + videoLevel: '4.0', + width: 320, + height: 240, + videoBitrate: 600, + }); + + const result = checkVideoCompatibility(analysis, ipodVideo5g); + + expect(result.status).toBe('transcode'); + expect(result.reasons).toEqual(['H.264 level exceeds device maximum: 4.0 > 3.0']); + }); + + it('transcode: unknown H.264 level', () => { + const analysis = createAnalysis({ + videoProfile: 'baseline', + videoLevel: null, + width: 320, + height: 240, + videoBitrate: 600, + }); + + const result = checkVideoCompatibility(analysis, ipodVideo5g); + + expect(result.status).toBe('transcode'); + expect(result.reasons).toEqual(['Unknown H.264 level']); + }); + it('passthrough: video without audio stream', () => { const analysis = createAnalysis({ hasAudioStream: false, diff --git a/packages/podkit-core/src/video/compatibility.ts b/packages/podkit-core/src/video/compatibility.ts index 1ac4e6c6..55fca0ed 100644 --- a/packages/podkit-core/src/video/compatibility.ts +++ b/packages/podkit-core/src/video/compatibility.ts @@ -118,30 +118,34 @@ export function isCompatibleVideoCodec( return false; } - // Check profile compatibility + // Check profile compatibility: Baseline plays on every iPod, Main only on + // Main devices, and any other profile (High, High 10, Extended, ...) is + // transcoded if (profile) { const normalizedProfile = profile.toLowerCase(); - // High profile is not compatible with any iPod - if (normalizedProfile === 'high') { - return false; - } - - // Baseline is compatible with all devices - if (normalizedProfile === 'baseline') { + if (normalizedProfile === 'baseline' || normalizedProfile === 'constrained baseline') { return true; } - // Main profile is only compatible with devices that support main if (normalizedProfile === 'main') { return device.videoProfile === 'main'; } + + return false; } // If no profile info, assume compatible (will be verified during playback) return true; } +/** H.264 level ('3', '3.0', '3.1') as a comparable number, or undefined. */ +function parseH264Level(level: string | null | undefined): number | undefined { + if (!level) return undefined; + const value = Number.parseFloat(level); + return Number.isFinite(value) && value > 0 ? value : undefined; +} + /** * Check if an audio codec is compatible (passthrough) */ @@ -264,6 +268,18 @@ export function checkVideoCompatibility( ) { reasons.push(`Incompatible video profile: ${analysis.videoProfile}`); } + + // A level above the device's cannot be decoded; an unknown one cannot be + // shown to be within it, so it is transcoded rather than copied. + const level = parseH264Level(analysis.videoLevel); + const deviceLevel = parseH264Level(device.videoLevel)!; + if (level === undefined) { + reasons.push('Unknown H.264 level'); + } else if (level > deviceLevel) { + reasons.push( + `H.264 level exceeds device maximum: ${analysis.videoLevel} > ${device.videoLevel}` + ); + } } // Resolution check diff --git a/packages/podkit-core/src/video/directory-adapter.ts b/packages/podkit-core/src/video/directory-adapter.ts index 40f18f3a..d01b2041 100644 --- a/packages/podkit-core/src/video/directory-adapter.ts +++ b/packages/podkit-core/src/video/directory-adapter.ts @@ -97,6 +97,18 @@ export interface CollectionVideo { /** Duration in seconds */ duration: number; + + /** H.264 profile, when probed (e.g. 'baseline', 'main', 'high') */ + videoProfile?: string | null; + + /** H.264 level, when probed (e.g. '3.0') */ + videoLevel?: string | null; + + /** Video bitrate in kbps, when probed */ + videoBitrate?: number; + + /** Frame rate in fps, when probed */ + frameRate?: number; } /** @@ -403,6 +415,10 @@ export class VideoDirectoryAdapter implements CollectionAdapter { expect(args).toContain('h264_videotoolbox'); expect(args).toContain('-profile:v'); expect(args).toContain('main'); + expect(args.join(' ')).toContain(`-level ${hardwareSettings.videoLevel}`); expect(args).toContain('-b:v'); expect(args).toContain('2000k'); diff --git a/packages/podkit-core/src/video/transcode.ts b/packages/podkit-core/src/video/transcode.ts index 21dfdf6c..d0abd0f1 100644 --- a/packages/podkit-core/src/video/transcode.ts +++ b/packages/podkit-core/src/video/transcode.ts @@ -188,6 +188,8 @@ export function buildVideoTranscodeArgs( 'h264_videotoolbox', '-profile:v', mapProfileForVideoToolbox(settings.videoProfile), + '-level', + settings.videoLevel, '-b:v', `${settings.targetVideoBitrate}k` ); From 5c0fc960014ed690419a6b8a7af438c5dcde78fd Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 19:00:39 +0100 Subject: [PATCH 12/21] fix(cli): resolve the configured video quality after the device is open deriveSettings runs before the device is open and passes null capabilities, and resolveDeviceVideo returns no quality for unknown capabilities, so sync fell back to `high` whenever no CLI quality flag was given: global and per-device quality/videoQuality (including the env forms) never reached the video plan. runSync now resolves the video chain again with the opened device's capabilities. resolveEffectiveVideoQuality keeps the precedence --video-quality > --quality > device > global > high. An integration test drives runSync against a test iPod Video and reads the video plan's quality line for global, global videoQuality, device videoQuality and both flags; the first three fail without the fix. --- .changeset/video-quality-config.md | 9 ++ .../sync-video-quality.integration.test.ts | 135 ++++++++++++++++++ packages/podkit-cli/src/commands/sync.ts | 33 ++++- 3 files changed, 172 insertions(+), 5 deletions(-) create mode 100644 .changeset/video-quality-config.md create mode 100644 packages/podkit-cli/src/commands/sync-video-quality.integration.test.ts diff --git a/.changeset/video-quality-config.md b/.changeset/video-quality-config.md new file mode 100644 index 00000000..e328d6d7 --- /dev/null +++ b/.changeset/video-quality-config.md @@ -0,0 +1,9 @@ +--- +"podkit": patch +--- + +Apply the configured video quality during sync + +`podkit sync` resolved the video quality before the device was open, when its capabilities are still unknown, and an unknown device resolves to no video quality, so `quality`, `videoQuality` and their per-device and environment-variable forms were ignored and video was always encoded at `high` unless `--quality` or `--video-quality` was passed. The configured chain is now resolved once the device is open; `--video-quality` still overrides `--quality`, which overrides the device and then the global settings. + +**One-time effect:** if you configured a video quality other than `high`, the first sync after upgrading re-transcodes the videos that were encoded at `high`, to the quality you configured. diff --git a/packages/podkit-cli/src/commands/sync-video-quality.integration.test.ts b/packages/podkit-cli/src/commands/sync-video-quality.integration.test.ts new file mode 100644 index 00000000..049e4e4f --- /dev/null +++ b/packages/podkit-cli/src/commands/sync-video-quality.integration.test.ts @@ -0,0 +1,135 @@ +/** + * Video quality reaches the video plan through runSync: configured global + * and device quality apply without a CLI flag, and the flags keep their + * precedence (`--video-quality` > `--quality` > device > global). + * + * The configured chain needs the device's capabilities, which are only + * known after the device is open, so this runs against a real test iPod. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'bun:test'; +import { copyFile, mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTestIpod, TestModels } from '@podkit/gpod-testing'; +import { + ensureFixturesExist, + getVideoFixturesDir, + requireFFmpeg, + requireGpodTool, +} from '@podkit/test-fixtures'; +import { requireLibgpodNode } from '@podkit/libgpod-node'; +import { runSync } from './sync.js'; +import { BufferExitCodeSink, OutputContext } from '../output/index.js'; +import { runWithContext, type CliContext } from '../context.js'; +import { runAction } from '../errors.js'; +import { BufferSink } from '../test-utils/buffer-sink.js'; +import { + DEFAULT_TRANSFORMS_CONFIG, + DEFAULT_VIDEO_TRANSFORMS_CONFIG, + type DeviceConfig, + type PodkitConfig, +} from '../config/index.js'; + +requireFFmpeg(); +requireGpodTool(); +requireLibgpodNode(); +ensureFixturesExist('video'); + +describe('runSync video quality resolution', () => { + let ipod: Awaited>; + let videoDir: string; + + beforeAll(async () => { + ipod = await createTestIpod({ model: TestModels.VIDEO_60GB, name: 'Video iPod' }); + videoDir = await mkdtemp(join(tmpdir(), 'podkit-video-quality-')); + await copyFile( + join(getVideoFixturesDir(), 'movie-with-metadata.mp4'), + join(videoDir, 'Test Movie (2024).mp4') + ); + }); + + afterAll(async () => { + await ipod?.cleanup(); + if (videoDir) await rm(videoDir, { recursive: true, force: true }); + }); + + async function videoQualityLine( + global: Partial, + device: DeviceConfig | undefined, + flags: { quality?: string; videoQuality?: string } = {} + ): Promise { + const config: PodkitConfig = { + artwork: true, + tips: false, + transforms: DEFAULT_TRANSFORMS_CONFIG, + videoTransforms: DEFAULT_VIDEO_TRANSFORMS_CONFIG, + devices: device ? { nano: { ...device, path: ipod.path } as DeviceConfig } : {}, + music: {}, + video: { films: { path: videoDir } }, + ...global, + } as PodkitConfig; + const ctx: CliContext = { + config, + globalOpts: { + json: false, + quiet: false, + verbose: 0, + color: false, + tips: false, + tty: false, + device: device ? 'nano' : ipod.path, + }, + configResult: { config, configPath: undefined, configFileExists: false }, + }; + const stdout = new BufferSink(); + const stderr = new BufferSink(); + const out = new OutputContext({ + mode: 'text', + quiet: false, + verbose: 0, + color: false, + tips: false, + tty: false, + stdout, + stderr, + exitCode: new BufferExitCodeSink(), + }); + await runWithContext(ctx, () => + runAction(out, () => + runSync({ type: ['video'], collection: 'films', dryRun: true, ...flags } as never, out) + ) + ); + const text = stdout.text(); + if (!text.includes('=== Video Sync Plan')) + throw new Error(`no video plan:\n${text}\n${stderr.text()}`); + const section = text.slice(text.indexOf('=== Video Sync Plan')); + return section.split('\n').find((l) => l.startsWith('Quality:')); + } + + it('applies the global quality without a CLI flag', async () => { + expect(await videoQualityLine({ quality: 'low' }, undefined)).toBe('Quality: low'); + }); + + it('applies the global videoQuality over the global quality', async () => { + expect(await videoQualityLine({ quality: 'low', videoQuality: 'medium' }, undefined)).toBe( + 'Quality: medium' + ); + }); + + it('applies the device videoQuality over the global settings', async () => { + expect( + await videoQualityLine({ quality: 'low' }, { videoQuality: 'max' } as DeviceConfig) + ).toBe('Quality: max'); + }); + + it('lets --quality override the configured chain and --video-quality override --quality', async () => { + const device = { videoQuality: 'max' } as DeviceConfig; + expect(await videoQualityLine({ quality: 'low' }, device, { quality: 'medium' })).toBe( + 'Quality: medium' + ); + expect( + await videoQualityLine({ quality: 'low' }, device, { quality: 'medium', videoQuality: 'low' }) + ).toBe('Quality: low'); + }); +}); diff --git a/packages/podkit-cli/src/commands/sync.ts b/packages/podkit-cli/src/commands/sync.ts index f2cc1b0f..dbb86ed5 100644 --- a/packages/podkit-cli/src/commands/sync.ts +++ b/packages/podkit-cli/src/commands/sync.ts @@ -277,6 +277,23 @@ function collectTypes(value: string, previous: string[]): string[] { return [...previous, ...value.split(',').map((v) => v.trim().toLowerCase())]; } +/** + * Video quality for a sync: `--video-quality` → `--quality` → the configured + * device → global chain (`resolved`, null until capabilities are known) → + * `high`. + */ +function resolveEffectiveVideoQuality( + options: Pick, + resolved: VideoQualityPreset | null +): VideoQualityPreset { + return ( + (options.videoQuality as VideoQualityPreset | undefined) ?? + (options.quality as VideoQualityPreset | undefined) ?? + resolved ?? + 'high' + ); +} + const syncTypeOption = new Option( '-t, --type ', 'sync type (repeatable, comma-separated, default: all)' @@ -551,11 +568,7 @@ export async function runSync( : options.quality ? (options.quality as QualityPreset) : resolved.audio.value, - videoQuality: options.videoQuality - ? (options.videoQuality as VideoQualityPreset) - : options.quality - ? (options.quality as VideoQualityPreset) - : (resolved.video.value ?? ('high' as VideoQualityPreset)), + videoQuality: resolveEffectiveVideoQuality(options, resolved.video.value), artwork: options.artwork !== undefined ? options.artwork : (resolved.artwork.value ?? true), skipUpgrades: options.skipUpgrades !== undefined ? options.skipUpgrades : resolved.skipUpgrades.value, @@ -917,6 +930,16 @@ export async function runSync( const deviceSupportsAlac = openResult.deviceSupportsAlac; const deviceCapabilities = openResult.capabilities; + // deriveSettings resolves before the device is open, and the video chain + // needs its capabilities (unknown capabilities resolve to no quality), so + // configured quality would always fall through to `high`. Resolve it again + // now; the CLI flags keep their precedence. + effectiveVideoQuality = resolveEffectiveVideoQuality( + options, + resolveDeviceSettings(config, '', deviceConfig ?? {}, deviceCapabilities, false, false).video + .value + ); + // ----- Acquire per-device sync lock ----- // // Dry-run is read-only by design — it inspects state without writing From 19f08f89013f3a94fd6983d612d50337e2505d53 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 19:01:06 +0100 Subject: [PATCH 13/21] feat(core): add an iPod nano 7th generation video profile The Nano 7 screen is 240x432; video is now fitted into 432x240 with the aspect ratio kept, H.264 Baseline level 3.0, at most 30 fps, AAC-LC in M4V. Bitrates are the 320x240 tiers scaled by the pixel ratio (1.35x): 1000/800/550/400 kbps. 432x240 is an encoding choice, not the decoder's limit: the firmware decodes up to 720x576, but frames larger than the screen only cost space. Because the compatibility check uses the profile's size, a larger file is re-encoded rather than copied; the code and docs say so. libgpod reports the Nano 7 as an unknown generation, which fell back to the iPod Classic profile (640x480 Main). The video presenter now picks the profile from the identity cascade's generation id first and keeps libgpod's generation as the fallback. Nano 6 has no video and is unchanged. An integration test encodes wide, portrait, small and odd-sized High profile sources for the Nano 7 through libx264 and, on macOS, VideoToolbox, and probes Baseline, level <= 3.0, the fitted size, <= 30 fps and AAC. --- .changeset/nano7-video-profile.md | 10 ++ devices/ipod.md | 7 +- packages/demo/src/mock-core.ts | 6 +- .../content/docs/reference/quality-presets.md | 14 +- .../docs/user-guide/transcoding/video.md | 26 +++- .../podkit-cli/src/commands/sync-presenter.ts | 6 + packages/podkit-cli/src/commands/sync.ts | 1 + .../commands/video-presenter.profile.test.ts | 45 ++++++ .../src/commands/video-presenter.ts | 5 +- packages/podkit-core/src/index.ts | 1 + .../video/nano7-output.integration.test.ts | 138 ++++++++++++++++++ packages/podkit-core/src/video/types.test.ts | 78 +++++++++- packages/podkit-core/src/video/types.ts | 75 +++++++++- 13 files changed, 389 insertions(+), 23 deletions(-) create mode 100644 .changeset/nano7-video-profile.md create mode 100644 packages/podkit-cli/src/commands/video-presenter.profile.test.ts create mode 100644 packages/podkit-core/src/video/nano7-output.integration.test.ts diff --git a/.changeset/nano7-video-profile.md b/.changeset/nano7-video-profile.md new file mode 100644 index 00000000..a9698c85 --- /dev/null +++ b/.changeset/nano7-video-profile.md @@ -0,0 +1,10 @@ +--- +"podkit": minor +"@podkit/core": minor +--- + +Add an iPod nano 7th generation video profile + +Videos for a Nano 7 are now encoded for its 240x432 screen held sideways: fitted into 432x240 with the aspect ratio kept (a 16:9 film becomes 426x240), H.264 Baseline level 3.0, at most 30 fps, AAC-LC audio in M4V. Bitrates are the 320x240 tiers scaled by the extra pixels: 1000, 800, 550 and 400 kbps for `max`, `high`, `medium` and `low`. + +The profile is chosen from podkit's own device identification, so it applies even though libgpod reports the Nano 7 as an unknown generation. Before, an unknown generation fell back to the iPod Classic profile (640x480 Main). The Nano 6 has no video playback and is unchanged. diff --git a/devices/ipod.md b/devices/ipod.md index 8c440af9..fa1f6291 100644 --- a/devices/ipod.md +++ b/devices/ipod.md @@ -282,7 +282,7 @@ iPod_Control/ | Shuffle 1-2G | Yes | No | No | No | — | None | | Nano 1-2G | Yes | No | Yes | No | — | 176x132 | | Nano 3-5G | Yes | Yes | Yes | Yes | ipod-nano-3g | Varies | -| Nano 7G | Yes | Yes | Yes | Yes | ipod-classic | 240x432 | +| Nano 7G | Yes | Yes | Yes | Yes | ipod-nano-7g | 240x432 | | Video 5G | Yes | Yes | Yes | Yes | ipod-video-5g | 320x240 | | Video 5.5G | Yes | Yes | Yes | Yes | ipod-video-5g | 320x240 | | Classic 6G | Yes | Yes | Yes | Yes | ipod-classic | 320x240 | @@ -305,6 +305,11 @@ iPod_Control/ |---------|-----------|---------------|-------------------|-----------|-----|---------| | ipod-video-5g | 320x240 | Baseline | 768 kbps | 128 kbps | 30 | Video 5G/5.5G, Nano 3-5G | | ipod-classic | 640x480 | Main | 2500 kbps | 160 kbps | 30 | Classic 6G/6.5G/7G | +| ipod-nano-7g | 432x240 | Baseline 3.0 | 1000 kbps | 128 kbps | 30 | Nano 7G | + +Nano 7G: 432x240 is the screen size, not the decoder's limit (see +`DEVICE_PROFILES` in `packages/podkit-core/src/video/types.ts`). Nano 6G has no +video playback. ## Quirks & Limitations diff --git a/packages/demo/src/mock-core.ts b/packages/demo/src/mock-core.ts index c7db5276..71deee2f 100644 --- a/packages/demo/src/mock-core.ts +++ b/packages/demo/src/mock-core.ts @@ -1627,8 +1627,12 @@ export function getDeviceProfileByGeneration(_generation: string) { return getDeviceProfile('ipod-classic'); } +export function getDeviceProfileByIpodGeneration(_generationId: string) { + return getDeviceProfile('ipod-classic'); +} + export function getDeviceProfileNames(): string[] { - return ['ipod-classic', 'ipod-video-5g', 'ipod-nano-3g']; + return ['ipod-classic', 'ipod-video-5g', 'ipod-nano-3g', 'ipod-nano-7g']; } export function getPresetSettings(_preset: string) { diff --git a/packages/docs-site/src/content/docs/reference/quality-presets.md b/packages/docs-site/src/content/docs/reference/quality-presets.md index 544fc007..3375f349 100644 --- a/packages/docs-site/src/content/docs/reference/quality-presets.md +++ b/packages/docs-site/src/content/docs/reference/quality-presets.md @@ -95,12 +95,14 @@ Notes: ### Bitrate by Device and Preset -| Preset | iPod Classic (640x480) | iPod Video (320x240) | -|--------|------------------------|----------------------| -| max | 640x480 @ 2500 kbps | 320x240 @ 768 kbps | -| high | 640x480 @ 2000 kbps | 320x240 @ 600 kbps | -| medium | 640x480 @ 1500 kbps | 320x240 @ 400 kbps | -| low | 640x480 @ 1000 kbps | 320x240 @ 300 kbps | +| Preset | iPod Classic (640x480) | iPod Video / Nano 3-5 (320x240) | iPod Nano 7 (432x240) | +|--------|------------------------|---------------------------------|-----------------------| +| max | 640x480 @ 2500 kbps | 320x240 @ 768 kbps | 432x240 @ 1000 kbps | +| high | 640x480 @ 2000 kbps | 320x240 @ 600 kbps | 432x240 @ 800 kbps | +| medium | 640x480 @ 1500 kbps | 320x240 @ 400 kbps | 432x240 @ 550 kbps | +| low | 640x480 @ 1000 kbps | 320x240 @ 300 kbps | 432x240 @ 400 kbps | + +The resolution is the largest frame that fits the device's screen with the source's aspect ratio kept; the table shows the full-screen case. For the Nano 7 tiers see [Video Transcoding](/user-guide/transcoding/video#ipod-nano-7th-gen). ### File Size Estimates diff --git a/packages/docs-site/src/content/docs/user-guide/transcoding/video.md b/packages/docs-site/src/content/docs/user-guide/transcoding/video.md index 3cbb4f08..0d93e4c0 100644 --- a/packages/docs-site/src/content/docs/user-guide/transcoding/video.md +++ b/packages/docs-site/src/content/docs/user-guide/transcoding/video.md @@ -60,16 +60,30 @@ All videos are transcoded to iPod-compatible format: | Max Video Bitrate | ~768 kbps | | Audio | AAC-LC, up to 128 kbps | +### iPod Nano (7th Gen) + +| Setting | Value | +|---------|-------| +| Max Resolution | 432x240 | +| Video Profile | H.264 Baseline Profile, level 3.0 | +| Max Video Bitrate | ~1 Mbps | +| Max Frame Rate | 30 fps | +| Audio | AAC-LC, up to 128 kbps | + +432x240 is podkit's encoding choice for the 240x432 screen held sideways, not a playback limit: the Nano 7 decodes up to 720x576, but a larger frame would be scaled down on the device and only cost space, so a larger file is re-encoded rather than copied. Video is fitted with its aspect ratio kept (a 16:9 film becomes 426x240, a 4:3 one 320x240). The bitrates are the 320x240 tiers scaled by the extra pixels (about 1.35 times). The Nano 6 has no video playback. + ## Quality and Resolution Quality settings affect bitrate, not resolution. Resolution is always matched to device capabilities: -| Preset | iPod Classic (640x480) | iPod Video (320x240) | -|--------|------------------------|----------------------| -| max | 640x480 @ 2500 kbps | 320x240 @ 768 kbps | -| high | 640x480 @ 2000 kbps | 320x240 @ 600 kbps | -| medium | 640x480 @ 1500 kbps | 320x240 @ 400 kbps | -| low | 640x480 @ 1000 kbps | 320x240 @ 300 kbps | +| Preset | iPod Classic (640x480) | iPod Video (320x240) | iPod Nano 7 (432x240) | +|--------|------------------------|----------------------|-----------------------| +| max | 640x480 @ 2500 kbps | 320x240 @ 768 kbps | 432x240 @ 1000 kbps | +| high | 640x480 @ 2000 kbps | 320x240 @ 600 kbps | 432x240 @ 800 kbps | +| medium | 640x480 @ 1500 kbps | 320x240 @ 400 kbps | 432x240 @ 550 kbps | +| low | 640x480 @ 1000 kbps | 320x240 @ 300 kbps | 432x240 @ 400 kbps | + +Audio in a video uses 128 kbps for `max` and `high` and 96 kbps for `medium` and `low` on the 320x240 and 432x240 profiles. ## File Size Estimates diff --git a/packages/podkit-cli/src/commands/sync-presenter.ts b/packages/podkit-cli/src/commands/sync-presenter.ts index a95fa552..8dd02d5a 100644 --- a/packages/podkit-cli/src/commands/sync-presenter.ts +++ b/packages/podkit-cli/src/commands/sync-presenter.ts @@ -130,6 +130,12 @@ export interface VideoContentConfig { effectiveVideoTransforms: VideoTransformsConfig; effectiveTransferMode: TransferMode | undefined; forceMetadata: boolean; + /** + * podkit generation id from the identity cascade (`OpenDeviceResult.ipodModel`). + * Picks the video profile ahead of libgpod's generation, which is `unknown` + * for devices libgpod's tables miss (Nano 7). + */ + ipodGenerationId?: string; } /** diff --git a/packages/podkit-cli/src/commands/sync.ts b/packages/podkit-cli/src/commands/sync.ts index dbb86ed5..61d75479 100644 --- a/packages/podkit-cli/src/commands/sync.ts +++ b/packages/podkit-cli/src/commands/sync.ts @@ -1289,6 +1289,7 @@ export async function runSync( effectiveVideoTransforms, effectiveTransferMode, forceMetadata: options.forceMetadata ?? false, + ipodGenerationId: openResult.ipodModel?.generationId, }; // allowEmptyPlaylist is intentionally omitted: video collections cannot diff --git a/packages/podkit-cli/src/commands/video-presenter.profile.test.ts b/packages/podkit-cli/src/commands/video-presenter.profile.test.ts new file mode 100644 index 00000000..49fd1d20 --- /dev/null +++ b/packages/podkit-cli/src/commands/video-presenter.profile.test.ts @@ -0,0 +1,45 @@ +/** + * Video device-profile selection in VideoPresenter.computeDiff: the identity + * cascade's generation id wins over libgpod's generation, which is `unknown` + * for devices libgpod's tables miss. + */ + +import { describe, expect, it } from 'bun:test'; +import * as core from '@podkit/core'; +import type { VideoDeviceProfile } from '@podkit/core'; +import { VideoPresenter } from './video-presenter.js'; +import type { VideoContentConfig } from './sync-presenter.js'; + +function selectedProfile(libgpodGeneration: string, ipodGenerationId?: string): string { + let captured: VideoDeviceProfile | undefined; + const spyCore = { + ...core, + createVideoHandler: (cfg: Parameters[0]) => { + captured = cfg?.deviceProfile; + return core.createVideoHandler(cfg); + }, + } as typeof core; + const device = { + getIpodDatabase: () => ({ getInfo: () => ({ device: { generation: libgpodGeneration } }) }), + }; + const config: VideoContentConfig = { + type: 'video', + effectiveVideoQuality: 'high', + effectiveVideoTransforms: core.DEFAULT_VIDEO_TRANSFORMS_CONFIG, + effectiveTransferMode: undefined, + forceMetadata: false, + ipodGenerationId, + }; + new VideoPresenter().computeDiff([], [], config, device, spyCore); + return captured!.name; +} + +describe('VideoPresenter device profile', () => { + it('uses the Nano 7G profile when the cascade says nano_7g and libgpod says unknown', () => { + expect(selectedProfile('unknown', 'nano_7g')).toBe('ipod-nano-7g'); + }); + + it("falls back to libgpod's generation without a cascade generation id", () => { + expect(selectedProfile('video_1')).toBe('ipod-video-5g'); + }); +}); diff --git a/packages/podkit-cli/src/commands/video-presenter.ts b/packages/podkit-cli/src/commands/video-presenter.ts index eaa8a76d..f487c27a 100644 --- a/packages/podkit-cli/src/commands/video-presenter.ts +++ b/packages/podkit-cli/src/commands/video-presenter.ts @@ -112,7 +112,10 @@ export class VideoPresenter implements ContentTypePresenter>; +} + +describe('Nano 7 video output', () => { + let dir: string; + + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'podkit-nano7-video-')); + for (const s of sources) { + execFileSync('ffmpeg', [ + '-v', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + `testsrc2=size=${s.size}:rate=${s.rate}:duration=2`, + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:duration=2:sample_rate=48000', + ...(s.yuv444 + ? ['-vf', `scale=${s.size.replace('x', ':')}`, '-c:v', 'libx264', '-pix_fmt', 'yuv444p'] + : ['-c:v', 'libx264', '-profile:v', 'high', '-level', '4.1', '-pix_fmt', 'yuv420p']), + '-c:a', + 'aac', + '-shortest', + join(dir, `${s.name}.${s.yuv444 ? 'mkv' : 'mp4'}`), + ]); + } + }); + + afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }); + }); + + for (const encoder of encoders) { + for (const s of sources) { + it(`${encoder.label}: ${s.name} ${s.size}@${s.rate} → Baseline <=3.0, fitted, <=30 fps, AAC`, async () => { + const input = join(dir, `${s.name}.${s.yuv444 ? 'mkv' : 'mp4'}`); + const analysis = await probeVideo(input); + expect(checkVideoCompatibility(analysis, nano7).status).toBe('transcode'); + + const settings = { + ...calculateEffectiveSettings(analysis, 'high', nano7), + useHardwareAcceleration: encoder.hardware, + }; + expect([settings.targetWidth, settings.targetHeight]).toEqual(s.expected); + + const output = join(dir, `${s.name}-${encoder.label}.m4v`); + await transcodeVideo(input, output, settings); + + const streams = probeStreams(output); + const video = streams.find((x) => x.codec_type === 'video')!; + const audio = streams.find((x) => x.codec_type === 'audio')!; + const [num, den] = String(video.avg_frame_rate).split('/').map(Number); + expect({ + codec: video.codec_name, + profile: String(video.profile).toLowerCase().includes('baseline'), + levelOk: Number(video.level) > 0 && Number(video.level) <= 30, + size: [Number(video.width), Number(video.height)], + fpsOk: num! / den! <= 30, + audio: audio?.codec_name, + }).toEqual({ + codec: 'h264', + profile: true, + levelOk: true, + size: s.expected, + fpsOk: true, + audio: 'aac', + }); + }); + } + } +}); diff --git a/packages/podkit-core/src/video/types.test.ts b/packages/podkit-core/src/video/types.test.ts index 5ebe60be..9f4f8bba 100644 --- a/packages/podkit-core/src/video/types.test.ts +++ b/packages/podkit-core/src/video/types.test.ts @@ -9,9 +9,13 @@ import { getDeviceProfile, getDefaultDeviceProfile, getDeviceProfileNames, + getDeviceProfileByGeneration, + getDeviceProfileByIpodGeneration, getPresetSettings, getPresetSettingsWithFallback, } from './types.js'; +import { GENERATIONS } from '@podkit/devices-ipod'; +import { calculateEffectiveSettings } from './quality.js'; import type { VideoQualityPreset, VideoDeviceProfile, @@ -133,7 +137,79 @@ describe('video/types', () => { expect(names).toContain('ipod-classic'); expect(names).toContain('ipod-video-5g'); expect(names).toContain('ipod-nano-3g'); - expect(names).toHaveLength(3); + expect(names).toContain('ipod-nano-7g'); + expect(names).toHaveLength(4); + }); + }); + + describe('getDeviceProfileByIpodGeneration', () => { + it('selects the Nano 7G profile for nano_7g', () => { + expect(getDeviceProfileByIpodGeneration('nano_7g')?.name).toBe('ipod-nano-7g'); + }); + + it('covers every video-capable generation podkit can open, and nothing else', () => { + for (const gen of Object.values(GENERATIONS)) { + const expected = gen.supportsVideo && gen.support.access !== 'none'; + expect([gen.id, getDeviceProfileByIpodGeneration(gen.id) !== undefined]).toEqual([ + gen.id, + expected, + ]); + } + }); + + it('agrees with the libgpod mapping for generations libgpod knows', () => { + const pairs: Array<[string, string]> = [ + ['video_5g', 'video_1'], + ['video_5_5g', 'video_2'], + ['classic_6g', 'classic_1'], + ['classic_7g', 'classic_3'], + ['nano_3g', 'nano_3'], + ['nano_5g', 'nano_5'], + ]; + for (const [podkitId, libgpodId] of pairs) { + expect(getDeviceProfileByIpodGeneration(podkitId)?.name).toBe( + getDeviceProfileByGeneration(libgpodId).name + ); + } + }); + }); + + describe('Nano 7G effective settings', () => { + const source = (width: number, height: number): VideoSourceAnalysis => ({ + filePath: '/x.mkv', + container: 'mkv', + videoCodec: 'h264', + videoProfile: 'high', + videoLevel: '4.1', + width, + height, + videoBitrate: 8000, + frameRate: 60, + audioCodec: 'ac3', + audioBitrate: 384, + audioChannels: 6, + audioSampleRate: 48000, + duration: 60, + hasVideoStream: true, + hasAudioStream: true, + }); + const nano7 = DEVICE_PROFILES['ipod-nano-7g']!; + + it('fits 16:9 into 432x240 with even dimensions, Baseline 3.0, 30 fps', () => { + const s = calculateEffectiveSettings(source(1920, 1080), 'high', nano7); + expect([s.targetWidth, s.targetHeight]).toEqual([426, 240]); + expect(s.videoProfile).toBe('baseline'); + expect(s.videoLevel).toBe('3.0'); + expect(s.frameRate).toBe(30); + expect(s.targetVideoBitrate).toBe(800); + }); + + it('fits 4:3 by height and 2.39:1 by width', () => { + const fourThree = calculateEffectiveSettings(source(1440, 1080), 'max', nano7); + expect([fourThree.targetWidth, fourThree.targetHeight]).toEqual([320, 240]); + const scope = calculateEffectiveSettings(source(1920, 804), 'low', nano7); + expect([scope.targetWidth, scope.targetHeight]).toEqual([432, 180]); + expect(scope.targetVideoBitrate).toBe(400); }); }); diff --git a/packages/podkit-core/src/video/types.ts b/packages/podkit-core/src/video/types.ts index ec9d8901..30f6d071 100644 --- a/packages/podkit-core/src/video/types.ts +++ b/packages/podkit-core/src/video/types.ts @@ -25,6 +25,7 @@ * | iPod Classic | 640x480 | Main 3.1 | * | iPod Video 5G | 320x240 | Baseline 3.0 | * | iPod Nano 3G-5G | 320x240 | Baseline 3.0 | + * | iPod Nano 7G | 432x240 | Baseline 3.0 | */ // ============================================================================= @@ -40,10 +41,9 @@ * * @see DEVICE_PROFILES below for the full profile spec. */ -const LIBGPOD_GENERATION_TO_VIDEO_PROFILE: Record< - string, - 'ipod-video-5g' | 'ipod-classic' | 'ipod-nano-3g' -> = { +type VideoProfileName = 'ipod-video-5g' | 'ipod-classic' | 'ipod-nano-3g' | 'ipod-nano-7g'; + +const LIBGPOD_GENERATION_TO_VIDEO_PROFILE: Record = { // iPod Video 5G/5.5G video_1: 'ipod-video-5g', video_2: 'ipod-video-5g', @@ -72,12 +72,28 @@ const LIBGPOD_GENERATION_TO_VIDEO_PROFILE: Record< * Get the video device profile name for a libgpod generation string. * Returns `undefined` for generations that don't support video. */ -function getVideoProfile( - generation: string -): 'ipod-video-5g' | 'ipod-classic' | 'ipod-nano-3g' | undefined { +function getVideoProfile(generation: string): VideoProfileName | undefined { return LIBGPOD_GENERATION_TO_VIDEO_PROFILE[generation]; } +/** + * Maps podkit's own generation ids (`IpodGenerationId`) to video profiles. + * + * Needed alongside the libgpod map because libgpod reports `unknown` for + * generations its tables miss — the Nano 7 among them — and an unknown + * libgpod generation falls back to the iPod Classic profile. + */ +const IPOD_GENERATION_TO_VIDEO_PROFILE: Record = { + video_5g: 'ipod-video-5g', + video_5_5g: 'ipod-video-5g', + classic_6g: 'ipod-classic', + classic_7g: 'ipod-classic', + nano_3g: 'ipod-nano-3g', + nano_4g: 'ipod-nano-3g', + nano_5g: 'ipod-nano-3g', + nano_7g: 'ipod-nano-7g', +}; + // ============================================================================= // Quality Presets // ============================================================================= @@ -174,6 +190,11 @@ export interface VideoDeviceProfile { * - iPod Classic: 640x480, Main profile, level 3.1 * - iPod Video 5G: 320x240, Baseline profile, level 3.0 * - iPod Nano 3G-5G: 320x240, Baseline profile, level 3.0 + * - iPod Nano 7G: 432x240 (the 240x432 portrait screen, landscape), Baseline + * profile, level 3.0. 432x240 is an encoding choice, not the decoder's + * limit: the firmware decodes up to 720x576 (SysInfoExtended). As the + * compatibility check uses maxWidth/maxHeight, a larger file the device + * could play is still re-encoded to the screen size, which saves space. */ export const DEVICE_PROFILES: Record = { 'ipod-classic': { @@ -220,6 +241,21 @@ export const DEVICE_PROFILES: Record = { maxFrameRate: 30, supportsVideo: true, }, + + 'ipod-nano-7g': { + name: 'ipod-nano-7g', + displayName: 'iPod Nano (7th Gen)', + maxWidth: 432, + maxHeight: 240, + maxVideoBitrate: 1000, + maxAudioBitrate: 128, + videoCodec: 'h264', + videoProfile: 'baseline', + videoLevel: '3.0', + audioCodec: 'aac', + maxFrameRate: 30, + supportsVideo: true, + }, } as const; /** @@ -258,6 +294,21 @@ export function getDeviceProfileByGeneration(generation: string): VideoDevicePro return DEVICE_PROFILES['ipod-classic']!; } +/** + * Get the video device profile for a podkit generation id (`nano_7g`, + * `classic_6g`, ...) as resolved by the identity cascade. + * + * @returns The profile, or `undefined` when the generation has no video + * profile (or is not a known id) — callers then fall back to + * {@link getDeviceProfileByGeneration} with libgpod's view. + */ +export function getDeviceProfileByIpodGeneration( + generationId: string +): VideoDeviceProfile | undefined { + const profileName = IPOD_GENERATION_TO_VIDEO_PROFILE[generationId]; + return profileName ? DEVICE_PROFILES[profileName] : undefined; +} + /** * Get all device profile names */ @@ -451,6 +502,16 @@ export const VIDEO_PRESET_SETTINGS: Record< medium: { videoBitrate: 400, audioBitrate: 96, crf: 26 }, low: { videoBitrate: 300, audioBitrate: 96, crf: 28 }, }, + + // iPod Nano 7th Gen (432x240, Baseline profile). 432x240 is 1.35x the + // pixels of 320x240, so each tier is the 320x240 tier scaled by that and + // rounded; CRF and audio follow the 320x240 tiers. + 'ipod-nano-7g': { + max: { videoBitrate: 1000, audioBitrate: 128, crf: 20 }, + high: { videoBitrate: 800, audioBitrate: 128, crf: 23 }, + medium: { videoBitrate: 550, audioBitrate: 96, crf: 26 }, + low: { videoBitrate: 400, audioBitrate: 96, crf: 28 }, + }, } as const; /** From 469dbbcce424bf2cc073229a5f54fe5d1a5da9fb Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 18:12:28 +0100 Subject: [PATCH 14/21] feat(cli): add --custom-bitrate and document customBitrate below the low preset The presets stop at low (~128 kbps). customBitrate (global, per device, PODKIT_CUSTOM_BITRATE) already replaces the preset bitrate and is the cap the lossy-reduction seam compares against, so no new setting is needed for a 96 kbps library; it was missing a CLI flag and user documentation. sync gains --custom-bitrate (64-320, same range as the config key) overriding the resolved value for one run, and the dry-run quality line names the custom bitrate. A composed test pins the behaviour from resolveMusicConfig through the classifier: FLAC and MP3 above the tolerance become AAC at 96 kbps under reduce=always with 96 recorded in the sync tag, MP3 within the tolerance and MP3 under preserve are copied. Docs cover the reduction interplay and that aac_at is preferred over native aac at low bitrates. --- .changeset/custom-bitrate-flag.md | 9 +++ .../content/docs/reference/cli-commands.md | 1 + .../src/content/docs/reference/config-file.md | 4 +- .../docs/reference/environment-variables.md | 2 +- .../content/docs/reference/quality-presets.md | 8 +++ .../docs/user-guide/transcoding/audio.md | 29 +++++++++ .../src/commands/music-presenter.ts | 8 ++- .../src/commands/sync-aggregation.test.ts | 52 +++++++++++++++ packages/podkit-cli/src/commands/sync.test.ts | 14 ++++- packages/podkit-cli/src/commands/sync.ts | 18 +++++- .../src/sync/music/classifier.test.ts | 63 +++++++++++++++++++ 11 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 .changeset/custom-bitrate-flag.md diff --git a/.changeset/custom-bitrate-flag.md b/.changeset/custom-bitrate-flag.md new file mode 100644 index 00000000..e294e94d --- /dev/null +++ b/.changeset/custom-bitrate-flag.md @@ -0,0 +1,9 @@ +--- +"podkit": minor +--- + +Add `--custom-bitrate ` to `podkit sync` and document `customBitrate` as the way below the `low` preset + +The presets stop at `low` (~128 kbps). The existing `customBitrate` setting (global, per device, or `PODKIT_CUSTOM_BITRATE`) replaces the preset's bitrate with any value from 64 to 320 kbps and is the cap for every lossy target, including lossy reduction of MP3/AAC sources when `[bitrate].reduce` converts. It can now also be set for a single run with `--custom-bitrate`, and `sync --dry-run` shows it on the quality line (`Quality: low (custom bitrate 96 kbps)`). + +For example, `customBitrate = 96` with `[bitrate] reduce = "always"` encodes lossless sources to AAC at 96 kbps and re-encodes MP3s more than 25% above that to AAC at 96 kbps, while MP3s at or near 96 kbps are copied. Without `reduce`, MP3 and AAC files are still copied untouched. Presets are unchanged. diff --git a/packages/docs-site/src/content/docs/reference/cli-commands.md b/packages/docs-site/src/content/docs/reference/cli-commands.md index 17b4a87d..806b98d2 100644 --- a/packages/docs-site/src/content/docs/reference/cli-commands.md +++ b/packages/docs-site/src/content/docs/reference/cli-commands.md @@ -129,6 +129,7 @@ podkit sync [options] | `--encoding ` | Encoding mode: `vbr` (default) or `cbr` | | `--bitrate-reduce ` | Lossy reduction mode for this run: `auto` (default), `always`, or `never`. Overrides the device's [`[bitrate].reduce`](/reference/config-file#lossy-reduction) setting. `auto` follows the transfer mode (`optimized` converts; `fast`/`portable` preserve). `always` reduces all over-cap device-native lossy sources. `never` preserves all device-native lossy sources regardless of cap. | | `--bitrate-tolerance ` | Source-proximity tolerance for lossy reduction on this run (e.g. `0.25` means reduce only when source exceeds cap by more than 25%). Overrides the device's [`[bitrate].tolerance`](/reference/config-file#lossy-reduction) setting. | +| `--custom-bitrate ` | Lossy audio bitrate for this run, 64-320 kbps, replacing the preset's bitrate as the cap. Overrides the [`customBitrate`](/reference/config-file#global-settings) setting. See [Custom Bitrate](/user-guide/transcoding/audio#custom-bitrate). | | `--transfer-mode ` | Transfer mode: `fast` (default), `optimized`, or `portable`. Controls whether extra file data is preserved or stripped during sync | | `--force-transfer-mode` | Re-process all tracks when changing transfer mode | | `--filter ` | Only sync tracks matching pattern | diff --git a/packages/docs-site/src/content/docs/reference/config-file.md b/packages/docs-site/src/content/docs/reference/config-file.md index f1461fb5..0c4375f8 100644 --- a/packages/docs-site/src/content/docs/reference/config-file.md +++ b/packages/docs-site/src/content/docs/reference/config-file.md @@ -110,7 +110,7 @@ These apply to all devices unless overridden at the device level. | `videoQuality` | string | - | Video-specific quality override: `max`, `high`, `medium`, `low`. Overrides `quality` for video. | | `encoding` | string | `"vbr"` | Encoding mode for lossy transcoding: `vbr` (variable bitrate) or `cbr` (constant bitrate). VBR produces better quality per MB; CBR produces predictable file sizes and more reliable preset change detection. Applies to whichever codec the [preference stack](/user-guide/transcoding/codec-preferences) resolves. | | `transferMode` | string | `"fast"` | Transfer mode controlling how extra file data (e.g. embedded artwork) is handled, and — for iPod — whether on-disk file tags are kept in sync with the iTunesDB. See [Transfer Mode](#transfer-mode) below. | -| `customBitrate` | integer | - | Override the preset's target bitrate (64-320 kbps). Ignored when `max` resolves to ALAC. | +| `customBitrate` | integer | - | Replace the preset's bitrate with this value (64-320 kbps). It is the cap for every lossy target, including [lossy reduction](#lossy-reduction) of MP3/AAC sources. Ignored for lossless sources when `max` resolves to ALAC. CLI: `--custom-bitrate`. See [Custom Bitrate](/user-guide/transcoding/audio#custom-bitrate). | | `[bitrate].tolerance` | number | `0.25` | Source-proximity tolerance for lossy reduction on the add path (fraction of the cap). Reduce only when `source > cap × (1 + tolerance)`. Applies only when the reduction axis is `convert`. See [Lossy Reduction](#lossy-reduction). | | `artwork` | boolean | `true` | Include album artwork during sync | | `checkArtwork` | boolean | `false` | Detect artwork changes between syncs (added, removed, or replaced). For Subsonic sources, adds one HTTP request per unique album during scanning. Consider using the `--check-artwork` CLI flag for periodic checks instead of enabling permanently on large libraries. | @@ -362,7 +362,7 @@ quality = "max" # Use --device to specify mount point | `videoQuality` | string | no | global `videoQuality` | Video-specific quality override for this device | | `encoding` | string | no | global `encoding` | Encoding mode override: `vbr` or `cbr` | | `transferMode` | string | no | global `transferMode` | Transfer mode override: `fast`, `optimized`, or `portable`. See [Transfer Mode](#transfer-mode) for the device-specific contract (file-tag writes differ between iPod and mass-storage). | -| `customBitrate` | integer | no | global `customBitrate` | Override the preset's target bitrate for this device | +| `customBitrate` | integer | no | global `customBitrate` | Replace the preset's bitrate for this device (64-320 kbps) | | `bitrate` | table | no | global `[bitrate]` | Lossy reduction block (`[devices..bitrate]`). See [Lossy Reduction](#lossy-reduction). | | `artwork` | boolean | no | global `artwork` | Artwork override for this device | | `checkArtwork` | boolean | no | global `checkArtwork` | Detect changed artwork for this device | diff --git a/packages/docs-site/src/content/docs/reference/environment-variables.md b/packages/docs-site/src/content/docs/reference/environment-variables.md index c59caf40..5ec427c5 100644 --- a/packages/docs-site/src/content/docs/reference/environment-variables.md +++ b/packages/docs-site/src/content/docs/reference/environment-variables.md @@ -18,7 +18,7 @@ podkit settings can be overridden via environment variables. These take preceden | `PODKIT_ENCODING` | Encoding mode: `vbr` or `cbr` (overrides config file `encoding`) | | `PODKIT_TRANSFER_MODE` | Transfer mode: `fast`, `optimized`, or `portable`. Controls whether extra file data is preserved or stripped (overrides config file `transferMode`) | | `PODKIT_FORCE_TRANSFER_MODE` | Force re-processing of all tracks when changing transfer mode (`true`/`false`) | -| `PODKIT_CUSTOM_BITRATE` | Override target bitrate for AAC encoding, 64-320 kbps (overrides config file `customBitrate`) | +| `PODKIT_CUSTOM_BITRATE` | Replace the preset's bitrate for lossy encoding, 64-320 kbps (overrides config file `customBitrate`; `--custom-bitrate` overrides this) | | `PODKIT_BITRATE_REDUCE` | Lossy reduction mode: `auto`, `always`, or `never` (overrides config file `[bitrate].reduce`) | | `PODKIT_BITRATE_TOLERANCE` | Source-proximity tolerance for lossy reduction, as a fraction of the cap (e.g. `0.25`; overrides config file `[bitrate].tolerance`) | | `PODKIT_FORCE_TRANSCODE` | Force re-transcoding of all lossless-source tracks (`true`/`false`) | diff --git a/packages/docs-site/src/content/docs/reference/quality-presets.md b/packages/docs-site/src/content/docs/reference/quality-presets.md index 3375f349..0a21fd17 100644 --- a/packages/docs-site/src/content/docs/reference/quality-presets.md +++ b/packages/docs-site/src/content/docs/reference/quality-presets.md @@ -48,9 +48,14 @@ For a 4-minute song: | **high** | ~256 kbps | ~7.5 MB | | **medium** | ~192 kbps | ~5.6 MB | | **low** | ~128 kbps | ~3.8 MB | +| `customBitrate = 96` | ~96 kbps | ~2.9 MB | VBR file sizes vary based on content complexity. CBR sizes are exact. +### Custom Bitrate + +`customBitrate` (config, per device, `PODKIT_CUSTOM_BITRATE`, or `--custom-bitrate` for one run) replaces the preset's bitrate with any value from 64 to 320 kbps. It is the cap for every lossy target, including lossy reduction of MP3/AAC sources when `[bitrate].reduce` converts. See [Custom Bitrate](/user-guide/transcoding/audio#custom-bitrate). + ### VBR vs CBR | Mode | Description | @@ -69,6 +74,7 @@ podkit picks the best AAC encoder your FFmpeg offers -- `aac_at` (macOS AudioToo | high (and max AAC fallback) | 256k | 5 | 256k | | medium | 192k | 4 | 192k | | low | 128k | 3 | 128k | +| `customBitrate = 96` | 96k | 2 | 96k | When `max` resolves to ALAC (on capable devices with lossless sources), the ALAC encoder is used instead. When `max` falls back to AAC, it uses the same encoder settings as `high`. @@ -82,6 +88,8 @@ Notes: One limit remains: 64 kbps is the lowest rate this encoder offers for stereo, so a target below that cannot be honoured on macOS. +- Below 128 kbps the encoders differ most in quality. `aac_at` is the best of the three there; FFmpeg's native `aac` is clearly weaker at 96 kbps, so a low custom bitrate is best encoded on macOS or with `libfdk_aac`. + ## Video Presets ### Preset Summary diff --git a/packages/docs-site/src/content/docs/user-guide/transcoding/audio.md b/packages/docs-site/src/content/docs/user-guide/transcoding/audio.md index ec40bfa7..cbbdd1be 100644 --- a/packages/docs-site/src/content/docs/user-guide/transcoding/audio.md +++ b/packages/docs-site/src/content/docs/user-guide/transcoding/audio.md @@ -39,6 +39,9 @@ podkit sync --quality low # Set unified quality, but override audio specifically podkit sync --quality medium --audio-quality high + +# A bitrate below the presets, for this run only +podkit sync --custom-bitrate 96 ``` ### Config File @@ -48,8 +51,33 @@ podkit sync --quality medium --audio-quality high quality = "high" # Unified quality for audio and video audioQuality = "high" # Audio-specific override: max | high | medium | low encoding = "vbr" # Encoding mode: vbr (default) or cbr +customBitrate = 96 # Optional: replaces the preset's bitrate (64-320 kbps) +``` + +## Custom Bitrate + +The presets stop at `low` (~128 kbps). To fit more music on a small player, set `customBitrate` to the AAC bitrate you want, in kbps (64-320). It works globally, per device, through `PODKIT_CUSTOM_BITRATE`, or for one run with `--custom-bitrate`: + +```toml +[devices.nano] +quality = "low" +customBitrate = 96 + +[devices.nano.bitrate] +reduce = "always" # also re-encode MP3/AAC sources above the cap ``` +`customBitrate` replaces the preset's bitrate everywhere the preset's bitrate would apply, so it is the cap for every lossy target: + +- **Lossless sources** (FLAC, WAV, ALAC) are transcoded to AAC at the custom bitrate. With `max` on a device that plays ALAC, lossless sources still become ALAC and the custom bitrate only applies to lossy targets. +- **Incompatible lossy sources** (OGG, Opus) are transcoded at no more than the custom bitrate. +- **MP3 and AAC sources** follow the [reduction axis](#bitrate-cap-enforcement-for-lossy-sources), as with any cap. Under `reduce = "always"` (or `auto` with `transferMode = "optimized"`), a source more than the tolerance above the cap is re-encoded to AAC at the custom bitrate: at 96 kbps and the default tolerance of 0.25, a 192 kbps MP3 is reduced and a 112 kbps MP3 is copied. Under `never` (or `auto` with `fast`/`portable`, the default) MP3 and AAC files are copied untouched, so setting a custom bitrate alone never re-encodes them. +- **The bitrate is recorded in each track's sync tag**, so changing it later is detected like a preset change: lossless-source tracks are re-transcoded at the new bitrate, and lossy sources follow the [cap rules](#bitrate-cap-enforcement-for-lossy-sources) (reduced tracks above a lowered cap are re-encoded under `convert`; below a raised cap they are reported, and `--force-transcode` lifts them). + +`sync --dry-run` shows it on the quality line, e.g. `Quality: low (custom bitrate 96 kbps)`. + +Low bitrates are where the AAC encoders differ most. On macOS podkit uses Apple's AudioToolbox encoder (`aac_at`), which holds up well at 96 kbps and is the one to use. FFmpeg's native `aac` encoder, the only one on a stock Linux FFmpeg, is noticeably weaker at low bitrates; if you sync from Linux, prefer 128 kbps or more, or use an FFmpeg built with `libfdk_aac`. See [AAC Encoders](#aac-encoders). + ## Example Scenarios **Scenario 1: Audiophile with iPod Classic** @@ -157,6 +185,7 @@ A few details worth knowing: | **high** | ~256 kbps | ~7.5 MB | | **medium** | ~192 kbps | ~5.6 MB | | **low** | ~128 kbps | ~3.8 MB | +| `customBitrate = 96` | ~96 kbps | ~2.9 MB | VBR file sizes vary based on content complexity. CBR sizes are exact. The `max` preset produces ALAC (lossless) on devices that support it, otherwise AAC at the same quality as `high`. diff --git a/packages/podkit-cli/src/commands/music-presenter.ts b/packages/podkit-cli/src/commands/music-presenter.ts index 8dc7f1cd..d3e9efd3 100644 --- a/packages/podkit-cli/src/commands/music-presenter.ts +++ b/packages/podkit-cli/src/commands/music-presenter.ts @@ -128,6 +128,12 @@ function resolveDisplayQuality(config: MusicContentConfig): string { return config.effectiveQuality; } +function formatQualityLine(config: MusicContentConfig): string { + const quality = resolveDisplayQuality(config); + if (config.effectiveCustomBitrate === undefined || quality === 'lossless') return quality; + return `${quality} (custom bitrate ${config.effectiveCustomBitrate} kbps)`; +} + /** * Find the lossless codec that will be used for transcoding (first device-supported * non-source entry in the lossless preference stack). @@ -444,7 +450,7 @@ export class MusicPresenter implements ContentTypePresenter { expect(stdout.text).toContain('--force-transcode'); }); }); + +describe('music dry-run quality line', () => { + class BufferSink implements OutputSink { + chunks: string[] = []; + write(chunk: string): boolean { + this.chunks.push(chunk); + return true; + } + get text(): string { + return this.chunks.join(''); + } + } + + function renderQualityLine(config: MusicContentConfig): string { + const stdout = new BufferSink(); + const out = new OutputContext({ + mode: 'text', + quiet: false, + verbose: 0, + color: false, + tips: false, + tty: false, + stdout, + stderr: new BufferSink(), + }); + const ctx = createMusicDryRunCtx([]); + new MusicPresenter().renderDryRunText( + out, + '/music', + '/ipod', + ctx.diff as never, + ctx.plan as never, + ctx.summary as never, + null, + true, + false, + config, + ctx.core as never, + [] + ); + return stdout.text.split('\n').find((line) => line.startsWith('Quality:')) ?? ''; + } + + it('names a custom bitrate next to the preset', () => { + const config = { ...makeMusicContentConfig('low'), effectiveCustomBitrate: 96 }; + expect(renderQualityLine(config)).toBe('Quality: low (custom bitrate 96 kbps)'); + }); + + it('shows the bare preset without a custom bitrate', () => { + expect(renderQualityLine(makeMusicContentConfig('low'))).toBe('Quality: low'); + }); +}); diff --git a/packages/podkit-cli/src/commands/sync.test.ts b/packages/podkit-cli/src/commands/sync.test.ts index 764c1eab..b879617e 100644 --- a/packages/podkit-cli/src/commands/sync.test.ts +++ b/packages/podkit-cli/src/commands/sync.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { formatDuration, renderProgressBar, syncCommand } from './sync.js'; +import { formatDuration, parseCustomBitrate, renderProgressBar, syncCommand } from './sync.js'; import { formatBytes } from './display-utils.js'; describe('sync command', () => { @@ -61,6 +61,18 @@ describe('sync command', () => { expect(toleranceOption?.defaultValue).toBeUndefined(); }); + it('has --custom-bitrate value option bounded like the customBitrate config key', () => { + const option = syncCommand.options.find((opt) => opt.long === '--custom-bitrate'); + expect(option?.attributeName()).toBe('customBitrate'); + expect(option?.defaultValue).toBeUndefined(); + expect(parseCustomBitrate('96')).toBe(96); + expect(parseCustomBitrate('64')).toBe(64); + expect(parseCustomBitrate('320')).toBe(320); + for (const bad of ['63', '321', '96.5', 'abc', '']) { + expect(() => parseCustomBitrate(bad)).toThrow('between 64 and 320'); + } + }); + it('no longer has the removed --bitrate-sync option', () => { const options = syncCommand.options; expect(options.find((opt) => opt.long === '--bitrate-sync')).toBeUndefined(); diff --git a/packages/podkit-cli/src/commands/sync.ts b/packages/podkit-cli/src/commands/sync.ts index 61d75479..2e325c98 100644 --- a/packages/podkit-cli/src/commands/sync.ts +++ b/packages/podkit-cli/src/commands/sync.ts @@ -134,6 +134,7 @@ interface SyncOptions { encoding?: string; bitrateReduce?: string; bitrateTolerance?: number; + customBitrate?: number; transferMode?: string; filter?: string; artwork?: boolean; @@ -294,6 +295,15 @@ function resolveEffectiveVideoQuality( ); } +/** Same range as the `customBitrate` config key. */ +export function parseCustomBitrate(value: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 64 || parsed > 320) { + throw new InvalidArgumentError('Must be an integer between 64 and 320 (kbps).'); + } + return parsed; +} + const syncTypeOption = new Option( '-t, --type ', 'sync type (repeatable, comma-separated, default: all)' @@ -343,6 +353,12 @@ export const syncCommand = new Command('sync') return parsed; }) ) + .addOption( + new Option( + '--custom-bitrate ', + 'lossy audio bitrate in kbps (64-320) that replaces the preset bitrate as the cap — overrides config for this run' + ).argParser(parseCustomBitrate) + ) .addOption( new Option( '--transfer-mode ', @@ -578,7 +594,7 @@ export async function runSync( transferMode: options.transferMode ? (options.transferMode as import('@podkit/core').TransferMode) : resolved.transferMode.value, - customBitrate: resolved.customBitrate.value, + customBitrate: options.customBitrate ?? resolved.customBitrate.value, // `--bitrate-reduce` / `--bitrate-tolerance` override the resolved // device → global → default chain for this run only. Neither flag has a // Commander default, so an unpassed flag is absent here and the resolved diff --git a/packages/podkit-core/src/sync/music/classifier.test.ts b/packages/podkit-core/src/sync/music/classifier.test.ts index cb1dd3dc..72cdd810 100644 --- a/packages/podkit-core/src/sync/music/classifier.test.ts +++ b/packages/podkit-core/src/sync/music/classifier.test.ts @@ -3,6 +3,10 @@ import { MusicTrackClassifier, classifierFromConfig } from './classifier.js'; import type { ClassifierContext } from './classifier.js'; import { resolveReductionAxis } from '../engine/lossy-reduction.js'; import { makeMockCollectionTrack } from '../../test-utils/tracks.js'; +import { resolveMusicConfig } from './config.js'; +import { expectedSyncTagFromClassification } from './handler.js'; +import type { FFmpegTranscoder } from '../../transcode/ffmpeg.js'; +import type { DeviceCapabilities } from '@podkit/device-types'; // ============================================================================= // Test Fixtures @@ -767,3 +771,62 @@ describe('MusicTrackClassifier', () => { }); }); }); + +describe('custom bitrate as the lossy cap (resolveMusicConfig → classifier)', () => { + const capabilities: DeviceCapabilities = { + artworkSources: ['database'], + artworkMaxResolution: 240, + supportedAudioCodecs: ['aac', 'mp3', 'alac', 'wav', 'aiff'], + supportsVideo: true, + audioNormalization: 'soundcheck', + supportsAlbumArtistBrowsing: false, + }; + + function classify(reduce: 'always' | 'never', track: Parameters[0]) { + const resolved = resolveMusicConfig({ + quality: 'low', + customBitrate: 96, + reduce, + capabilities, + transcoder: {} as FFmpegTranscoder, + }); + expect(resolved.presetBitrate).toBe(96); + const ctx = classifierFromConfig(resolved); + const classification = new MusicTrackClassifier(ctx).classify(makeTrack(track)); + return { + action: classification.action, + tag: expectedSyncTagFromClassification(classification, { + customBitrate: 96, + resolvedLossyCodec: ctx.resolvedLossyCodec, + }), + }; + } + + const mp3 = (bitrate: number) => ({ + fileType: 'mp3' as const, + filePath: `/music/${bitrate}.mp3`, + codec: 'mp3', + lossless: false, + bitrate, + }); + + test('FLAC → AAC at 96 kbps, recorded in the sync tag', () => { + const { action, tag } = classify('never', {}); + expect(action).toMatchObject({ type: 'transcode', preset: { bitrateOverride: 96 } }); + expect(tag?.bitrate).toBe(96); + }); + + test('MP3 192 under convert → reduced to AAC 96, recorded in the sync tag', () => { + const { action, tag } = classify('always', mp3(192)); + expect(action).toEqual({ type: 'transcode', preset: { name: 'low', bitrateOverride: 96 } }); + expect(tag).toMatchObject({ quality: 'low', bitrate: 96 }); + }); + + test('MP3 112 under convert stays a copy: within the 25% tolerance of 96', () => { + expect(classify('always', mp3(112)).action).toEqual({ type: 'direct-copy' }); + }); + + test('MP3 192 under preserve stays a copy: the cap alone never re-encodes', () => { + expect(classify('never', mp3(192)).action).toEqual({ type: 'direct-copy' }); + }); +}); From 83ff66f96ff7e002b910616181b76484aba4a7b3 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 18:12:56 +0100 Subject: [PATCH 15/21] feat(transcode): encode MP3 at a custom or reduced bitrate in LAME's ABR mode With an MP3 codec preference, a custom bitrate or reduction target set bitrateKbps but the preset's VBR quality still won: buildMp3Args emitted only -q:a, which has no bitrate bound, while the sync tag recorded the numeric target. A 96 kbps override could therefore produce a file well above 96. buildTranscodePreset drops the preset quality for MP3 when the preset carries a bitrateOverride, and buildMp3Args encodes VBR without a quality in LAME's average-bitrate mode (-abr 1 -b:a). ABR holds the target as an average, not as a hard ceiling. Plain presets keep -q:a; CBR keeps -b:a. An integration test encodes white noise through the 96 kbps override and measures the stream bitrate. --- .changeset/custom-bitrate-flag.md | 3 +++ .../src/content/docs/reference/quality-presets.md | 5 ++--- .../content/docs/user-guide/transcoding/audio.md | 3 +-- .../podkit-core/src/sync/music/pipeline-display.ts | 8 +++++++- .../src/transcode/ffmpeg.integration.test.ts | 14 ++++++++++++++ packages/podkit-core/src/transcode/ffmpeg.test.ts | 7 +++++++ packages/podkit-core/src/transcode/ffmpeg.ts | 10 ++++++---- 7 files changed, 40 insertions(+), 10 deletions(-) diff --git a/.changeset/custom-bitrate-flag.md b/.changeset/custom-bitrate-flag.md index e294e94d..2e987845 100644 --- a/.changeset/custom-bitrate-flag.md +++ b/.changeset/custom-bitrate-flag.md @@ -1,5 +1,6 @@ --- "podkit": minor +"@podkit/core": patch --- Add `--custom-bitrate ` to `podkit sync` and document `customBitrate` as the way below the `low` preset @@ -7,3 +8,5 @@ Add `--custom-bitrate ` to `podkit sync` and document `customBitrate` as t The presets stop at `low` (~128 kbps). The existing `customBitrate` setting (global, per device, or `PODKIT_CUSTOM_BITRATE`) replaces the preset's bitrate with any value from 64 to 320 kbps and is the cap for every lossy target, including lossy reduction of MP3/AAC sources when `[bitrate].reduce` converts. It can now also be set for a single run with `--custom-bitrate`, and `sync --dry-run` shows it on the quality line (`Quality: low (custom bitrate 96 kbps)`). For example, `customBitrate = 96` with `[bitrate] reduce = "always"` encodes lossless sources to AAC at 96 kbps and re-encodes MP3s more than 25% above that to AAC at 96 kbps, while MP3s at or near 96 kbps are copied. Without `reduce`, MP3 and AAC files are still copied untouched. Presets are unchanged. + +MP3 output now follows a numeric target too: with a custom bitrate or a lossy reduction target, LAME encodes in average-bitrate (ABR) mode with that bitrate as its average instead of the preset's `-q:a` quality level, which has no bitrate bound. MP3s already encoded with a custom bitrate are not re-encoded; `--force-transcode` re-encodes those made from lossless sources. diff --git a/packages/docs-site/src/content/docs/reference/quality-presets.md b/packages/docs-site/src/content/docs/reference/quality-presets.md index 0a21fd17..b24fb41c 100644 --- a/packages/docs-site/src/content/docs/reference/quality-presets.md +++ b/packages/docs-site/src/content/docs/reference/quality-presets.md @@ -52,9 +52,7 @@ For a 4-minute song: VBR file sizes vary based on content complexity. CBR sizes are exact. -### Custom Bitrate - -`customBitrate` (config, per device, `PODKIT_CUSTOM_BITRATE`, or `--custom-bitrate` for one run) replaces the preset's bitrate with any value from 64 to 320 kbps. It is the cap for every lossy target, including lossy reduction of MP3/AAC sources when `[bitrate].reduce` converts. See [Custom Bitrate](/user-guide/transcoding/audio#custom-bitrate). +For a bitrate below `low`, see [Custom Bitrate](/user-guide/transcoding/audio#custom-bitrate). ### VBR vs CBR @@ -88,6 +86,7 @@ Notes: One limit remains: 64 kbps is the lowest rate this encoder offers for stereo, so a target below that cannot be honoured on macOS. +- MP3 output (when the [codec preference](/user-guide/transcoding/codec-preferences) resolves to MP3) uses LAME's `-q:a` quality scale for a plain preset. When a numeric cap applies — a `customBitrate` or a lossy reduction target — podkit switches LAME to average-bitrate mode (`-abr 1 -b:a`) with that bitrate as the target average, because `-q:a` has no bitrate bound; the sync tag records the target as the track's bitrate. ABR holds the average, so short passages can run above it. `encoding = "cbr"` uses `-b:a` as before. - Below 128 kbps the encoders differ most in quality. `aac_at` is the best of the three there; FFmpeg's native `aac` is clearly weaker at 96 kbps, so a low custom bitrate is best encoded on macOS or with `libfdk_aac`. ## Video Presets diff --git a/packages/docs-site/src/content/docs/user-guide/transcoding/audio.md b/packages/docs-site/src/content/docs/user-guide/transcoding/audio.md index cbbdd1be..d68aaea8 100644 --- a/packages/docs-site/src/content/docs/user-guide/transcoding/audio.md +++ b/packages/docs-site/src/content/docs/user-guide/transcoding/audio.md @@ -67,7 +67,7 @@ customBitrate = 96 reduce = "always" # also re-encode MP3/AAC sources above the cap ``` -`customBitrate` replaces the preset's bitrate everywhere the preset's bitrate would apply, so it is the cap for every lossy target: +`customBitrate` replaces the preset's bitrate everywhere the preset's bitrate would apply, so it is the cap for every lossy target (for MP3 output it is the target average of LAME's ABR mode, which can briefly run above it; see [Encoder Mapping](/reference/quality-presets#encoder-mapping)): - **Lossless sources** (FLAC, WAV, ALAC) are transcoded to AAC at the custom bitrate. With `max` on a device that plays ALAC, lossless sources still become ALAC and the custom bitrate only applies to lossy targets. - **Incompatible lossy sources** (OGG, Opus) are transcoded at no more than the custom bitrate. @@ -185,7 +185,6 @@ A few details worth knowing: | **high** | ~256 kbps | ~7.5 MB | | **medium** | ~192 kbps | ~5.6 MB | | **low** | ~128 kbps | ~3.8 MB | -| `customBitrate = 96` | ~96 kbps | ~2.9 MB | VBR file sizes vary based on content complexity. CBR sizes are exact. The `max` preset produces ALAC (lossless) on devices that support it, otherwise AAC at the same quality as `high`. diff --git a/packages/podkit-core/src/sync/music/pipeline-display.ts b/packages/podkit-core/src/sync/music/pipeline-display.ts index 6ed3e8b5..b9227044 100644 --- a/packages/podkit-core/src/sync/music/pipeline-display.ts +++ b/packages/podkit-core/src/sync/music/pipeline-display.ts @@ -193,7 +193,13 @@ export function buildTranscodePreset( bitrateKbps: preset.bitrateOverride ?? getCodecPresetBitrate(preset.targetCodec, preset.name) ?? 256, encoding: encodingMode ?? 'vbr', - quality: getCodecVbrQuality(preset.targetCodec, preset.name), + // A numeric target (custom bitrate, reduction target) must reach the + // encoder: LAME's -q:a is a quality index with no bitrate bound, so MP3 + // drops it and encodes at an average bitrate instead. + quality: + preset.bitrateOverride !== undefined && preset.targetCodec === 'mp3' + ? undefined + : getCodecVbrQuality(preset.targetCodec, preset.name), }; return config; } diff --git a/packages/podkit-core/src/transcode/ffmpeg.integration.test.ts b/packages/podkit-core/src/transcode/ffmpeg.integration.test.ts index ad5630fc..fcc90617 100644 --- a/packages/podkit-core/src/transcode/ffmpeg.integration.test.ts +++ b/packages/podkit-core/src/transcode/ffmpeg.integration.test.ts @@ -21,6 +21,8 @@ import { FFmpegNotFoundError, TranscodeError, } from './ffmpeg.js'; +import type { EncoderConfig } from './ffmpeg.js'; +import { buildTranscodePreset } from '../sync/music/pipeline-display.js'; import type { TranscodeProgress } from './types.js'; import { AAC_PRESETS } from './types.js'; @@ -828,6 +830,18 @@ describe('quality preset bitrate ceiling', () => { expect(await streamBitrateKbps(highPath)).toBeGreaterThan(await streamBitrateKbps(lowPath)); }); + it('targets an MP3 average: a 96 kbps override encodes at about 96 kbps', async () => { + const config = buildTranscodePreset( + { name: 'high', targetCodec: 'mp3', bitrateOverride: 96 }, + 'vbr' + ); + const outputPath = join(ceilingDir, 'ceiling-mp3-96.mp3'); + await transcoder.transcode(hardSource, outputPath, config as EncoderConfig); + + // LAME's ABR tracks the average closely on noise; 5% covers frame padding. + expect(await streamBitrateKbps(outputPath)).toBeLessThanOrEqual(Math.round(96 * 1.05)); + }); + it('honours a planner-reduced target below the preset cap', async () => { // ADR-023 §3: a resolved EncoderConfig target is a ceiling in its own // right — a preserve/convert reduction must not come out at the preset. diff --git a/packages/podkit-core/src/transcode/ffmpeg.test.ts b/packages/podkit-core/src/transcode/ffmpeg.test.ts index 3341cbe3..5761fdbd 100644 --- a/packages/podkit-core/src/transcode/ffmpeg.test.ts +++ b/packages/podkit-core/src/transcode/ffmpeg.test.ts @@ -1017,6 +1017,13 @@ describe('buildMp3Args', () => { expect(args).not.toContain('-b:a'); }); + it('generates average-bitrate args for VBR with a numeric target and no preset quality', () => { + const config: EncoderConfig = { codec: 'mp3', bitrateKbps: 96, encoding: 'vbr' }; + const args = buildMp3Args(input, output, config); + expect(args.join(' ')).toContain('-abr 1 -b:a 96k'); + expect(args).not.toContain('-q:a'); + }); + it('generates CBR args with -b:a bitrate', () => { const config: EncoderConfig = { codec: 'mp3', bitrateKbps: 256, encoding: 'cbr' }; const args = buildMp3Args(input, output, config); diff --git a/packages/podkit-core/src/transcode/ffmpeg.ts b/packages/podkit-core/src/transcode/ffmpeg.ts index 81f76302..ea97e2f9 100644 --- a/packages/podkit-core/src/transcode/ffmpeg.ts +++ b/packages/podkit-core/src/transcode/ffmpeg.ts @@ -572,8 +572,10 @@ export function buildOpusArgs( /** * Build FFmpeg command arguments for MP3 encoding. * - * MP3 uses libmp3lame. VBR uses `-q:a` quality scale (0=best, 9=worst). - * CBR uses `-b:a` with target bitrate. + * MP3 uses libmp3lame. VBR with a preset quality uses the `-q:a` scale + * (0=best, 9=worst); VBR without one — a numeric target — uses LAME's average + * bitrate mode (`-abr 1 -b:a`), which holds the target as an average, not as a + * hard ceiling. CBR uses `-b:a`. * * @param input - Input file path * @param output - Output file path @@ -593,10 +595,10 @@ export function buildMp3Args( // Quality settings based on encoding mode if (config.encoding === 'vbr' && config.quality !== undefined) { - // VBR mode — use -q:a (LAME VBR quality: 0=best, 9=worst) args.push('-q:a', String(config.quality)); + } else if (config.encoding === 'vbr') { + args.push('-abr', '1', '-b:a', `${config.bitrateKbps}k`); } else { - // CBR mode args.push('-b:a', `${config.bitrateKbps}k`); } From a9b4e9d05dab7c4367a62bb7fe1c0f7f42e03460 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 18:19:09 +0100 Subject: [PATCH 16/21] feat(libgpod-node): expose the iTunesDB sort fields Track, TrackInput and updateTrack gain sortTitle, sortArtist, sortAlbum, sortAlbumArtist and sortComposer, mapped to libgpod's sort_* strings in TrackToObject, addTrack and updateTrack. An empty string clears a field, as for the other strings. A test adds, updates, saves and reopens a database and reads the values back. --- .../libgpod-node/native/gpod_converters.cc | 7 ++++ .../libgpod-node/native/track_operations.cc | 26 +++++++++++++++ .../src/__tests__/tracks.integration.test.ts | 33 +++++++++++++++++++ packages/libgpod-node/src/types.ts | 23 +++++++++++++ packages/podkit-core/src/ipod/track.test.ts | 5 +++ 5 files changed, 94 insertions(+) diff --git a/packages/libgpod-node/native/gpod_converters.cc b/packages/libgpod-node/native/gpod_converters.cc index 507ada65..1a07a6fe 100644 --- a/packages/libgpod-node/native/gpod_converters.cc +++ b/packages/libgpod-node/native/gpod_converters.cc @@ -138,6 +138,13 @@ Napi::Object TrackToObject(Napi::Env env, const Itdb_Track* track) { obj.Set("comment", GcharToValue(env, track->comment)); obj.Set("grouping", GcharToValue(env, track->grouping)); + // Sort overrides (NULL: the firmware sorts by the plain field) + obj.Set("sortTitle", GcharToValue(env, track->sort_title)); + obj.Set("sortArtist", GcharToValue(env, track->sort_artist)); + obj.Set("sortAlbum", GcharToValue(env, track->sort_album)); + obj.Set("sortAlbumArtist", GcharToValue(env, track->sort_albumartist)); + obj.Set("sortComposer", GcharToValue(env, track->sort_composer)); + // Track info obj.Set("trackNumber", Napi::Number::New(env, track->track_nr)); obj.Set("totalTracks", Napi::Number::New(env, track->tracks)); diff --git a/packages/libgpod-node/native/track_operations.cc b/packages/libgpod-node/native/track_operations.cc index 40bbe1b0..0e137edb 100644 --- a/packages/libgpod-node/native/track_operations.cc +++ b/packages/libgpod-node/native/track_operations.cc @@ -65,6 +65,12 @@ Napi::Value DatabaseWrapper::AddTrack(const Napi::CallbackInfo& info) { track->comment = GetOptionalString(input, "comment"); track->grouping = GetOptionalString(input, "grouping"); + track->sort_title = GetOptionalString(input, "sortTitle"); + track->sort_artist = GetOptionalString(input, "sortArtist"); + track->sort_album = GetOptionalString(input, "sortAlbum"); + track->sort_albumartist = GetOptionalString(input, "sortAlbumArtist"); + track->sort_composer = GetOptionalString(input, "sortComposer"); + track->track_nr = GetOptionalInt32(input, "trackNumber", 0); track->tracks = GetOptionalInt32(input, "totalTracks", 0); track->cd_nr = GetOptionalInt32(input, "discNumber", 0); @@ -391,6 +397,26 @@ Napi::Value DatabaseWrapper::UpdateTrack(const Napi::CallbackInfo& info) { g_free(track->grouping); track->grouping = ValueToGchar(fields.Get("grouping")); } + if (fields.Has("sortTitle")) { + g_free(track->sort_title); + track->sort_title = ValueToGchar(fields.Get("sortTitle")); + } + if (fields.Has("sortArtist")) { + g_free(track->sort_artist); + track->sort_artist = ValueToGchar(fields.Get("sortArtist")); + } + if (fields.Has("sortAlbum")) { + g_free(track->sort_album); + track->sort_album = ValueToGchar(fields.Get("sortAlbum")); + } + if (fields.Has("sortAlbumArtist")) { + g_free(track->sort_albumartist); + track->sort_albumartist = ValueToGchar(fields.Get("sortAlbumArtist")); + } + if (fields.Has("sortComposer")) { + g_free(track->sort_composer); + track->sort_composer = ValueToGchar(fields.Get("sortComposer")); + } // Update numeric fields if (fields.Has("trackNumber") && fields.Get("trackNumber").IsNumber()) { diff --git a/packages/libgpod-node/src/__tests__/tracks.integration.test.ts b/packages/libgpod-node/src/__tests__/tracks.integration.test.ts index 6efcf7fe..17c64f17 100644 --- a/packages/libgpod-node/src/__tests__/tracks.integration.test.ts +++ b/packages/libgpod-node/src/__tests__/tracks.integration.test.ts @@ -282,6 +282,39 @@ describe('libgpod-node track updates (updateTrack)', () => { }); }); + it('persists sort fields through add, update, save and reopen', async () => { + await withTestIpod(async (ipod) => { + const db = Database.openSync(ipod.path); + const handle = db.addTrack({ + title: 'The Title', + sortTitle: 'Title, The', + sortArtist: 'Band, The', + sortAlbum: 'Album, The', + sortAlbumArtist: 'Artist, The', + }); + db.updateTrack(handle, { sortComposer: 'Bach, Johann Sebastian', sortAlbum: '' }); + db.saveSync(); + db.close(); + + const reopened = Database.openSync(ipod.path); + const track = reopened.getTrack(reopened.getTracks()[0]!); + expect({ + sortTitle: track.sortTitle, + sortArtist: track.sortArtist, + sortAlbum: track.sortAlbum, + sortAlbumArtist: track.sortAlbumArtist, + sortComposer: track.sortComposer, + }).toEqual({ + sortTitle: 'Title, The', + sortArtist: 'Band, The', + sortAlbum: null, + sortAlbumArtist: 'Artist, The', + sortComposer: 'Bach, Johann Sebastian', + }); + reopened.close(); + }); + }); + it('can update all numeric fields', async () => { await withTestIpod(async (ipod) => { const db = Database.openSync(ipod.path); diff --git a/packages/libgpod-node/src/types.ts b/packages/libgpod-node/src/types.ts index 1a6ce5f4..555bf229 100644 --- a/packages/libgpod-node/src/types.ts +++ b/packages/libgpod-node/src/types.ts @@ -241,6 +241,18 @@ export interface Track { /** Grouping */ grouping: string | null; + // Sort overrides: when set, the firmware sorts by these instead of the plain field + /** Title used for sorting */ + sortTitle: string | null; + /** Artist used for sorting */ + sortArtist: string | null; + /** Album used for sorting */ + sortAlbum: string | null; + /** Album artist used for sorting */ + sortAlbumArtist: string | null; + /** Composer used for sorting */ + sortComposer: string | null; + // Track info /** Track number on disc */ trackNumber: number; @@ -350,6 +362,17 @@ export interface TrackInput { /** Grouping */ grouping?: string; + /** Title used for sorting */ + sortTitle?: string; + /** Artist used for sorting */ + sortArtist?: string; + /** Album used for sorting */ + sortAlbum?: string; + /** Album artist used for sorting */ + sortAlbumArtist?: string; + /** Composer used for sorting */ + sortComposer?: string; + /** Track number */ trackNumber?: number; /** Total tracks */ diff --git a/packages/podkit-core/src/ipod/track.test.ts b/packages/podkit-core/src/ipod/track.test.ts index 39eba4c9..888d7172 100644 --- a/packages/podkit-core/src/ipod/track.test.ts +++ b/packages/podkit-core/src/ipod/track.test.ts @@ -30,6 +30,11 @@ function createMockTrack(overrides: Partial = {}): Track { composer: 'Test Composer', comment: 'Test Comment', grouping: 'Test Grouping', + sortTitle: null, + sortArtist: null, + sortAlbum: null, + sortAlbumArtist: null, + sortComposer: null, trackNumber: 1, totalTracks: 10, discNumber: 1, From f3e00d758d78e0826c17209e510898c671c47bf7 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 18:19:37 +0100 Subject: [PATCH 17/21] feat(sync): sync composer, grouping, track/disc totals and sort fields to iPods The directory adapter reads composer (multi-value tags joined with ", "), grouping, total tracks/discs and the five sort fields from music-metadata; blank values read as unset. The fields travel through the add path, metadata-correction detection, the planner and the pipeline to the device, and a file replacement writes them as well. Detection is gated on DeviceTrack.storesExtendedTags, set by IpodTrack only: mass-storage tracks do not read these fields back, so comparing them there would flag the same tracks on every sync. A tag removed at the source clears the device field ('' for strings, 0 for totals). The extended fields are compared case-sensitively, since nothing else rewrites them and a case-only sort fix is a real change. CollectionTrack.unreportedTagFields lists the fields a source cannot report; those are neither compared nor cleared. The Subsonic adapter lists all of them for now. totalDiscs also reaches the add collision check. The comment field stays reserved for the sync tag. --- .changeset/extended-tag-fidelity.md | 15 +++ .../docs/user-guide/collections/directory.md | 3 +- .../content/docs/user-guide/syncing/music.md | 10 +- .../docs/user-guide/syncing/upgrades.md | 2 +- .../adapters/directory.integration.test.ts | 100 ++++++++++++++++ .../src/adapters/directory.test.ts | 20 ++++ .../podkit-core/src/adapters/directory.ts | 23 ++++ .../podkit-core/src/adapters/interface.ts | 39 +++++++ packages/podkit-core/src/adapters/subsonic.ts | 4 + packages/podkit-core/src/device/adapter.ts | 22 ++++ .../src/device/mass-storage-adapter.test.ts | 32 +++++ packages/podkit-core/src/ipod/track.ts | 13 +++ packages/podkit-core/src/ipod/types.ts | 30 +++++ packages/podkit-core/src/sync/engine/types.ts | 9 ++ .../podkit-core/src/sync/engine/upgrades.ts | 43 ++++++- .../src/sync/music/handler.test.ts | 104 +++++++++++++++++ .../podkit-core/src/sync/music/handler.ts | 19 +-- .../src/sync/music/pipeline.test.ts | 109 ++++++++++++++++++ .../podkit-core/src/sync/music/pipeline.ts | 5 + .../podkit-core/src/sync/music/planner.ts | 15 +++ .../podkit-core/src/sync/music/transfer.ts | 23 ++++ packages/podkit-core/src/types.ts | 9 ++ .../test-fixtures/src/mini-tracks.ts | 8 ++ 23 files changed, 637 insertions(+), 20 deletions(-) create mode 100644 .changeset/extended-tag-fidelity.md diff --git a/.changeset/extended-tag-fidelity.md b/.changeset/extended-tag-fidelity.md new file mode 100644 index 00000000..197fd2d1 --- /dev/null +++ b/.changeset/extended-tag-fidelity.md @@ -0,0 +1,15 @@ +--- +"podkit": minor +"@podkit/core": minor +"@podkit/libgpod-node": minor +--- + +Sync composer, grouping, track/disc totals and sort fields to iPods + +The directory adapter now reads composer, grouping, total tracks, total discs and the sort fields (sort title, artist, album, album artist, composer) from file tags. They are written to the iPod database when a track is added, and a change to any of them at the source is detected as a metadata correction and updated in place without copying the file again. A tag removed at the source is cleared on the device. A field the source cannot report is left untouched rather than cleared; Subsonic sources do not report these fields yet. + +Mass-storage change detection is unchanged: podkit does not compare these fields there. A multi-disc track added to a mass-storage device now gets the `Album (disc N)` folder its path template asks for, since the total disc count reaches it. The comment field still carries podkit's sync tag; file comments are not copied. + +`@podkit/libgpod-node` adds `sortTitle`, `sortArtist`, `sortAlbum`, `sortAlbumArtist` and `sortComposer` to `Track`, `TrackInput` and `updateTrack`. + +**One-time effect:** the first sync after upgrading updates the iPod database entry of every track whose files carry these tags, and clears these fields on the iPod for tracks whose files lack them, since a folder collection is authoritative for its tags. No audio is copied or re-encoded for it. diff --git a/packages/docs-site/src/content/docs/user-guide/collections/directory.md b/packages/docs-site/src/content/docs/user-guide/collections/directory.md index 91757a07..2cf3b0cd 100644 --- a/packages/docs-site/src/content/docs/user-guide/collections/directory.md +++ b/packages/docs-site/src/content/docs/user-guide/collections/directory.md @@ -39,7 +39,8 @@ path = "/path/to/your/music" podkit extracts the following metadata from audio files: - **Core fields:** title, artist, album (required for matching) -- **Extended fields:** album artist, genre, year, track number, disc number, composer +- **Extended fields:** album artist, genre, year, track number and total tracks, disc number and total discs, composer, grouping, compilation +- **Sort fields:** sort title, sort artist, sort album, sort album artist, sort composer (e.g. `TITLESORT`/`ARTISTSORT` in FLAC, `TSOT`/`TSOP`/`TSOA`/`TSO2`/`TSOC` in MP3) - **Technical info:** duration, bitrate, sample rate, file size - **Identifiers:** MusicBrainz IDs, AcoustID (for future use) diff --git a/packages/docs-site/src/content/docs/user-guide/syncing/music.md b/packages/docs-site/src/content/docs/user-guide/syncing/music.md index 88ded67f..4ee2edfa 100644 --- a/packages/docs-site/src/content/docs/user-guide/syncing/music.md +++ b/packages/docs-site/src/content/docs/user-guide/syncing/music.md @@ -27,10 +27,18 @@ The target codec is chosen automatically based on your [codec preference stack]( podkit preserves all standard metadata through syncing and transcoding: - **Core fields:** Title, artist, album, album artist -- **Track info:** Track number, disc number, year, genre, composer +- **Track info:** Track number and total tracks, disc number and total discs, year, genre, composer, grouping, compilation +- **Sort fields:** Sort title, artist, album, album artist and composer, so an iPod lists "The Beatles" under B when your files say so - **Technical info:** Duration, bitrate, sample rate - **Album artwork:** Embedded artwork is synced to the device (see [Artwork](/user-guide/syncing/artwork)) +Composer, grouping, the totals and the sort fields are written to iPods, which keep them in the device database; a change to any of them in your files updates the device on the next sync without copying the file again. Mass-storage devices read tags from the files themselves, so for them podkit does not compare these fields. + +Limitations: + +- **The comment field is podkit's.** podkit stores its sync tag in the iPod comment field and does not copy your files' comments onto the iPod. +- **Subsonic sources** do not provide these fields; podkit leaves them as they are on the iPod. + ### Track Matching podkit matches tracks by **artist + album + title**. This means: diff --git a/packages/docs-site/src/content/docs/user-guide/syncing/upgrades.md b/packages/docs-site/src/content/docs/user-guide/syncing/upgrades.md index fa80d403..67b7b3ad 100644 --- a/packages/docs-site/src/content/docs/user-guide/syncing/upgrades.md +++ b/packages/docs-site/src/content/docs/user-guide/syncing/upgrades.md @@ -28,7 +28,7 @@ podkit recognizes several types of improvements: | **Artwork updated** | Album artwork changed in source (requires `--check-artwork`) | Metadata updated (no file transfer) | | **Artwork removed** | Artwork removed from source files | Artwork removed from iPod | | **Sound Check update** | ReplayGain tags added to collection | Metadata updated (no file transfer) | -| **Metadata correction** | Genre, year, or track numbers fixed | Metadata updated (no file transfer) | +| **Metadata correction** | Genre, year, track/disc numbers, album artist or compilation fixed; on iPods also composer, grouping, track/disc totals and sort fields | Metadata updated (no file transfer) | File-replacement upgrades (format, quality, preset, artwork added) require transferring a new audio file to the iPod. Metadata-only updates (artwork updated, artwork removed, Sound Check, metadata corrections) are instant since they only touch the iPod database. diff --git a/packages/podkit-core/src/adapters/directory.integration.test.ts b/packages/podkit-core/src/adapters/directory.integration.test.ts index 88026b0d..1a51ae91 100644 --- a/packages/podkit-core/src/adapters/directory.integration.test.ts +++ b/packages/podkit-core/src/adapters/directory.integration.test.ts @@ -12,6 +12,8 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { ensureFixturesExist, + generateMiniFlac, + generateMiniMp3, getMultiFormatFixturesDir, getMultiFormatEmbeddedFixturesDir, requireFFmpeg, @@ -265,6 +267,104 @@ describe('DirectoryAdapter — AIFF multi-format fixture', () => { }); }); +describe('DirectoryAdapter — composer, grouping, totals and sort tags', () => { + let dir: string; + const common = { + composer: 'Johann Sebastian Bach', + grouping: 'Cantatas', + album_artist: 'The Ensemble', + disc: '1/2', + track: '3/12', + }; + + beforeAll(async () => { + dir = join(tmpdir(), `podkit-tags-${Date.now()}`); + await mkdir(dir, { recursive: true }); + generateMiniFlac(dir, { + filename: 'tags.flac', + title: 'The Wall', + artist: 'The Band', + tags: { + ...common, + TRACKTOTAL: '12', + DISCTOTAL: '2', + TITLESORT: 'Wall, The', + ARTISTSORT: 'Band, The', + ALBUMSORT: 'Album, The', + ALBUMARTISTSORT: 'Ensemble, The', + COMPOSERSORT: 'Bach, Johann Sebastian', + }, + }); + // ffmpeg's ID3v2.4 writer maps these three sort keys to TSOT/TSOP/TSOA. + generateMiniMp3(dir, { + filename: 'tags.mp3', + title: 'The Wall', + artist: 'The Band', + tags: { + ...common, + 'title-sort': 'Wall, The', + 'artist-sort': 'Band, The', + 'album-sort': 'Album, The', + }, + }); + generateMiniMp3(dir, { filename: 'plain.mp3' }); + }); + + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + async function track(name: string) { + const tracks = await new DirectoryAdapter({ path: dir }).getItems(); + return tracks.find((t) => t.filePath.endsWith(name))!; + } + + const extendedFields = (t: Awaited>) => ({ + composer: t.composer, + grouping: t.grouping, + totalTracks: t.totalTracks, + totalDiscs: t.totalDiscs, + sortTitle: t.sortTitle, + sortArtist: t.sortArtist, + sortAlbum: t.sortAlbum, + sortAlbumArtist: t.sortAlbumArtist, + sortComposer: t.sortComposer, + }); + + it('reads every extended field from FLAC Vorbis comments', async () => { + expect(extendedFields(await track('tags.flac'))).toEqual({ + composer: 'Johann Sebastian Bach', + grouping: 'Cantatas', + totalTracks: 12, + totalDiscs: 2, + sortTitle: 'Wall, The', + sortArtist: 'Band, The', + sortAlbum: 'Album, The', + sortAlbumArtist: 'Ensemble, The', + sortComposer: 'Bach, Johann Sebastian', + }); + }); + + it('reads composer, grouping, totals and title/artist/album sort from ID3v2.4', async () => { + expect(extendedFields(await track('tags.mp3'))).toEqual({ + composer: 'Johann Sebastian Bach', + grouping: 'Cantatas', + totalTracks: 12, + totalDiscs: 2, + sortTitle: 'Wall, The', + sortArtist: 'Band, The', + sortAlbum: 'Album, The', + sortAlbumArtist: undefined, + sortComposer: undefined, + }); + }); + + it('leaves every extended field unset on an untagged file', async () => { + const values = Object.values(extendedFields(await track('plain.mp3'))); + expect(values.every((v) => v === undefined)).toBe(true); + }); +}); + // The large-collection performance test moved to // `./directory.perf.test.ts` and runs via `bun run test:perf`. Default // `bun run test` / `test:integration` flows exclude `*.perf.test.ts` via diff --git a/packages/podkit-core/src/adapters/directory.test.ts b/packages/podkit-core/src/adapters/directory.test.ts index 82c0d83c..6f888334 100644 --- a/packages/podkit-core/src/adapters/directory.test.ts +++ b/packages/podkit-core/src/adapters/directory.test.ts @@ -103,6 +103,26 @@ describe('DirectoryAdapter', () => { }); describe('getItems', () => { + it('joins a multi-value composer tag and drops blank tag values', async () => { + mockGlob.mockImplementation(async () => ['/music/song.mp3']); + mockParseFile.mockImplementation(async () => ({ + common: { + title: 'Song', + composer: ['John Lennon', ' ', 'Paul McCartney'], + grouping: '', + titlesort: '', + track: { no: 1, of: null }, + }, + format: {}, + })); + + const [track] = await newAdapter({ path: '/music' }).getItems(); + expect(track!.composer).toBe('John Lennon, Paul McCartney'); + expect(track!.grouping).toBeUndefined(); + expect(track!.sortTitle).toBeUndefined(); + expect(track!.totalTracks).toBeUndefined(); + }); + it('returns empty array for empty directory', async () => { mockGlob.mockImplementation(async () => []); diff --git a/packages/podkit-core/src/adapters/directory.ts b/packages/podkit-core/src/adapters/directory.ts index 37bc3761..72e10860 100644 --- a/packages/podkit-core/src/adapters/directory.ts +++ b/packages/podkit-core/src/adapters/directory.ts @@ -112,6 +112,20 @@ const LOSSLESS_CODECS = new Set([ 'wav', ]); +/** + * Join a multi-value tag (e.g. two ID3 TCOM entries) into the single string + * the iPod stores. + */ +function joinTagValues(values: string[] | undefined): string | undefined { + const parts = values?.map((v) => v.trim()).filter((v) => v.length > 0) ?? []; + return parts.length > 0 ? parts.join(', ') : undefined; +} + +/** A text tag, unset when blank. */ +function tagText(value: string | undefined): string | undefined { + return value?.trim() ? value : undefined; +} + /** * Determine if a codec is lossless */ @@ -347,6 +361,15 @@ export class DirectoryAdapter implements CollectionAdapter Promise; size?: number }; +/** + * Tag fields beyond the core set that a source may or may not be able to + * report. The comment field is not among them: podkit keeps it for the sync tag. + */ +export const EXTENDED_TAG_FIELDS = [ + 'composer', + 'grouping', + 'totalTracks', + 'totalDiscs', + 'sortTitle', + 'sortArtist', + 'sortAlbum', + 'sortAlbumArtist', + 'sortComposer', +] as const; + +export type ExtendedTagField = (typeof EXTENDED_TAG_FIELDS)[number]; + /** * A track from a collection source */ @@ -42,6 +60,27 @@ export interface CollectionTrack { discNumber?: number; compilation?: boolean; duration?: number; // milliseconds + composer?: string; + grouping?: string; + totalTracks?: number; + totalDiscs?: number; + + // Sort overrides (e.g. "Beatles, The"); absent means sort by the plain field + sortTitle?: string; + sortArtist?: string; + sortAlbum?: string; + sortAlbumArtist?: string; + sortComposer?: string; + + /** + * Extended fields this source cannot report for this track (e.g. totals + * on Subsonic, or an OpenSubsonic field a plain Subsonic server omits). + * An unset listed field means "unknown", not "empty": sync neither + * compares nor clears it. Fields not listed are authoritative, so unset + * means the tag is absent and the device value is cleared. Omitted: all + * fields are authoritative (file tags). + */ + unreportedTagFields?: readonly ExtendedTagField[]; // File info filePath: string; diff --git a/packages/podkit-core/src/adapters/subsonic.ts b/packages/podkit-core/src/adapters/subsonic.ts index 0479847b..fe906f09 100644 --- a/packages/podkit-core/src/adapters/subsonic.ts +++ b/packages/podkit-core/src/adapters/subsonic.ts @@ -8,6 +8,7 @@ import SubsonicAPI from 'subsonic-api'; import type { Child, AlbumWithSongsID3 } from 'subsonic-api'; import type { CollectionAdapter, CollectionTrack, FileAccess } from './interface.js'; +import { EXTENDED_TAG_FIELDS } from './interface.js'; import type { TrackFilter, AudioFileType } from '../types.js'; import type { Warning } from '../sync/engine/types.js'; import type { AudioNormalization } from '../metadata/normalization.js'; @@ -776,6 +777,9 @@ export class SubsonicAdapter implements CollectionAdapter { expect(collisions[0]!.description).toBe('Artist - Song'); }); + test('a multi-disc add is checked at the disc folder the add will use', async () => { + createFakeAudioFile(mountPoint, 'Music/Artist/Album (disc 1)/01 - Song.flac'); + const adapter = await MassStorageAdapter.open(mountPoint, TEST_CAPABILITIES, { + metadataReader: createMockMetadataReader({ + '01 - Song.flac': { title: 'Song', artist: 'Artist', album: 'Album', trackNumber: 1 }, + }), + }); + const source = makeMockCollectionTrack({ + title: 'Song', + artist: 'Artist', + album: 'Album', + trackNumber: 1, + discNumber: 1, + totalDiscs: 2, + fileType: 'flac', + filePath: '/music/01 - Song.flac', + }); + const handler = createMusicHandler({ quality: 'high', transcoder: {} as never }); + const inputs = handler.getCollisionCheckInputs({ + operations: [{ type: 'add-direct-copy', source }], + estimatedTime: 0, + estimatedSize: 0, + warnings: [], + }); + + expect(adapter.checkAddCollisions(inputs).map((c) => c.path)).toEqual([ + 'Music/Artist/Album (disc 1)/01 - Song.flac', + ]); + }); + test('does not flag managed files as collisions', async () => { // Create file AND manifest so it's managed createFakeAudioFile(mountPoint, 'Music/Artist/Album/01 - Song.flac'); diff --git a/packages/podkit-core/src/ipod/track.ts b/packages/podkit-core/src/ipod/track.ts index b67e73b1..ef0c9b89 100644 --- a/packages/podkit-core/src/ipod/track.ts +++ b/packages/podkit-core/src/ipod/track.ts @@ -89,6 +89,11 @@ export class IpodTrackImpl implements IpodTrack { readonly composer?: string; readonly comment?: string; readonly grouping?: string; + readonly sortTitle?: string; + readonly sortArtist?: string; + readonly sortAlbum?: string; + readonly sortAlbumArtist?: string; + readonly sortComposer?: string; readonly trackNumber?: number; readonly totalTracks?: number; readonly discNumber?: number; @@ -134,6 +139,9 @@ export class IpodTrackImpl implements IpodTrack { */ readonly artworkSink = 'database' as const; + /** The iTunesDB stores composer, grouping, totals and sort fields. */ + readonly storesExtendedTags = true as const; + // Video-specific fields readonly tvShow?: string; readonly tvEpisode?: string; @@ -162,6 +170,11 @@ export class IpodTrackImpl implements IpodTrack { this.composer = data.composer ?? undefined; this.comment = data.comment ?? undefined; this.grouping = data.grouping ?? undefined; + this.sortTitle = data.sortTitle ?? undefined; + this.sortArtist = data.sortArtist ?? undefined; + this.sortAlbum = data.sortAlbum ?? undefined; + this.sortAlbumArtist = data.sortAlbumArtist ?? undefined; + this.sortComposer = data.sortComposer ?? undefined; // Track/disc info (0 values become undefined for optional fields) this.trackNumber = data.trackNumber || undefined; diff --git a/packages/podkit-core/src/ipod/types.ts b/packages/podkit-core/src/ipod/types.ts index 57da33fb..32257885 100644 --- a/packages/podkit-core/src/ipod/types.ts +++ b/packages/podkit-core/src/ipod/types.ts @@ -46,6 +46,16 @@ export interface TrackInput { comment?: string; /** Grouping (for organizing tracks) */ grouping?: string; + /** Title used for sorting (e.g. "Wall, The") */ + sortTitle?: string; + /** Artist used for sorting */ + sortArtist?: string; + /** Album used for sorting */ + sortAlbum?: string; + /** Album artist used for sorting */ + sortAlbumArtist?: string; + /** Composer used for sorting */ + sortComposer?: string; // Track/disc info /** Track number on disc */ @@ -139,6 +149,16 @@ export interface TrackFields { comment?: string; /** Grouping */ grouping?: string; + /** Title used for sorting (e.g. "Wall, The") */ + sortTitle?: string; + /** Artist used for sorting */ + sortArtist?: string; + /** Album used for sorting */ + sortAlbum?: string; + /** Album artist used for sorting */ + sortAlbumArtist?: string; + /** Composer used for sorting */ + sortComposer?: string; /** Track number on disc */ trackNumber?: number; /** Total number of tracks on disc */ @@ -239,6 +259,16 @@ export interface IpodTrack extends DeviceTrack { readonly comment?: string; /** Grouping */ readonly grouping?: string; + /** Title used for sorting */ + readonly sortTitle?: string; + /** Artist used for sorting */ + readonly sortArtist?: string; + /** Album used for sorting */ + readonly sortAlbum?: string; + /** Album artist used for sorting */ + readonly sortAlbumArtist?: string; + /** Composer used for sorting */ + readonly sortComposer?: string; // Track/disc info /** Track number on disc */ diff --git a/packages/podkit-core/src/sync/engine/types.ts b/packages/podkit-core/src/sync/engine/types.ts index 34095c5e..3781d91f 100644 --- a/packages/podkit-core/src/sync/engine/types.ts +++ b/packages/podkit-core/src/sync/engine/types.ts @@ -90,6 +90,15 @@ export interface MetadataChange { | 'trackNumber' | 'discNumber' | 'compilation' + | 'composer' + | 'grouping' + | 'totalTracks' + | 'totalDiscs' + | 'sortTitle' + | 'sortArtist' + | 'sortAlbum' + | 'sortAlbumArtist' + | 'sortComposer' | 'normalization' | 'bitrate' | 'fileType' diff --git a/packages/podkit-core/src/sync/engine/upgrades.ts b/packages/podkit-core/src/sync/engine/upgrades.ts index 7b0c8ea2..47f9a88e 100644 --- a/packages/podkit-core/src/sync/engine/upgrades.ts +++ b/packages/podkit-core/src/sync/engine/upgrades.ts @@ -10,7 +10,8 @@ * @module */ -import type { CollectionTrack } from '../../adapters/interface.js'; +import type { CollectionTrack, ExtendedTagField } from '../../adapters/interface.js'; +import { EXTENDED_TAG_FIELDS } from '../../adapters/interface.js'; import type { EncodingMode } from '../../transcode/types.js'; import { QUALITY_PRESETS } from '../../transcode/types.js'; import type { TranscodeTargetCodec } from '../../transcode/codecs.js'; @@ -37,6 +38,38 @@ const METADATA_CORRECTION_FIELDS = [ 'compilation', ] as const; +type MetadataCorrectionField = (typeof METADATA_CORRECTION_FIELDS)[number] | ExtendedTagField; + +/** + * Extended fields the source reports authoritatively for this track; only + * these are compared, written and cleared on a device that stores them. + */ +export function reportedExtendedFields(source: CollectionTrack): readonly ExtendedTagField[] { + const unreported = source.unreportedTagFields; + return unreported?.length + ? EXTENDED_TAG_FIELDS.filter((f) => !unreported.includes(f)) + : EXTENDED_TAG_FIELDS; +} + +/** The value that clears an extended field on the device: 0 for totals, '' otherwise. */ +export function clearedExtendedValue(field: ExtendedTagField): '' | 0 { + return field === 'totalTracks' || field === 'totalDiscs' ? 0 : ''; +} + +/** + * The metadata fields a source correction is detected and written for: + * extended fields only on devices that persist them + * (`DeviceTrack.storesExtendedTags`) and only where the source reports them. + */ +export function metadataCorrectionFields( + source: CollectionTrack, + device: DeviceTrack +): readonly MetadataCorrectionField[] { + return device.storesExtendedTags + ? [...METADATA_CORRECTION_FIELDS, ...reportedExtendedFields(source)] + : METADATA_CORRECTION_FIELDS; +} + /** * iPod filetype strings that indicate lossless formats. * @@ -800,7 +833,7 @@ export function detectUpgrades(source: CollectionTrack, ipod: DeviceTrack): Upgr } // Metadata correction: check non-matching metadata fields - for (const field of METADATA_CORRECTION_FIELDS) { + for (const field of metadataCorrectionFields(source, ipod)) { const sourceValue = source[field as keyof CollectionTrack]; const ipodValue = ipod[field as keyof DeviceTrack]; @@ -841,6 +874,12 @@ export function metadataValuesDiffer( return true; } + // The extended fields are written verbatim and nothing else rewrites them, + // so a case-only fix (e.g. a sort name) is a real change. + if ((EXTENDED_TAG_FIELDS as readonly string[]).includes(field)) { + return sourceValue !== ipodValue; + } + // For strings, case-insensitive comparison if (typeof sourceValue === 'string' && typeof ipodValue === 'string') { const sv = sourceValue.toLowerCase().trim(); diff --git a/packages/podkit-core/src/sync/music/handler.test.ts b/packages/podkit-core/src/sync/music/handler.test.ts index de49f542..39b208ab 100644 --- a/packages/podkit-core/src/sync/music/handler.test.ts +++ b/packages/podkit-core/src/sync/music/handler.test.ts @@ -11,6 +11,8 @@ import { parseSyncTag } from '../../metadata/sync-tags.js'; import type { FFmpegTranscoder } from '../../transcode/ffmpeg.js'; import type { DeviceCapabilities } from '@podkit/device-types'; import { makeMockCollectionTrack, makeMockDeviceTrack } from '../../test-utils/tracks.js'; +import { createSyncDiffer } from '../engine/differ.js'; +import { createSyncPlanner } from '../engine/planner.js'; // ============================================================================= // Test Fixtures @@ -2591,3 +2593,105 @@ describe('postProcessSyncTagsTranscode (--force-sync-tags-transcode)', () => { expect(diff.toUpdate[0]!.reasons).not.toContain('sync-tag-write'); }); }); + +describe('extended tag fields (composer, grouping, totals, sort)', () => { + // A copied MP3 already in sync on audio, so only tag fields can differ. + const source = (overrides: Partial = {}) => + makeCollectionTrack({ + fileType: 'mp3', + codec: 'mp3', + lossless: false, + bitrate: 256, + filePath: '/music/a.mp3', + ...overrides, + }); + const device = (overrides: Partial = {}) => + makeDeviceTrack({ + filetype: 'MPEG audio file', + bitrate: 256, + syncTag: parseSyncTag('[podkit:v1 quality=copy transferMode=fast codec=mp3 bitrate=256]'), + ...overrides, + }); + + function plan(src: CollectionTrack, dev: DeviceTrack) { + const handler = createMusicHandler(makeConfig()); + const diff = createSyncDiffer(handler).diff([src], [dev]); + return createSyncPlanner(handler).plan(diff).operations; + } + + test('a sort tag changed at the source alone plans a metadata update of that field', () => { + const ops = plan( + source({ sortArtist: 'Band, The' }), + device({ storesExtendedTags: true, sortArtist: 'The Band' }) + ); + expect(ops).toHaveLength(1); + expect(ops[0]).toMatchObject({ + type: 'update-metadata', + metadata: { sortArtist: 'Band, The' }, + }); + }); + + test('tags removed at the source clear them on the device', () => { + const ops = plan( + source(), + device({ storesExtendedTags: true, sortAlbum: 'Album, The', composer: 'X', totalTracks: 12 }) + ); + expect(ops).toHaveLength(1); + expect((ops[0] as Extract).metadata).toEqual({ + composer: '', + totalTracks: 0, + sortAlbum: '', + }); + }); + + test('a case-only fix of an extended field is an update; of genre it is not', () => { + const ops = plan( + source({ composer: 'Bach', genre: 'Classical' }), + device({ storesExtendedTags: true, composer: 'bach', genre: 'classical' }) + ); + expect(ops).toHaveLength(1); + expect((ops[0] as Extract).metadata).toEqual({ + composer: 'Bach', + }); + }); + + test('matching tags plan nothing', () => { + const tags = { composer: 'Bach', grouping: 'G', totalDiscs: 2, sortComposer: 'Bach, J. S.' }; + expect(plan(source(tags), device({ storesExtendedTags: true, ...tags }))).toEqual([]); + }); + + test('fields the source cannot report are neither compared nor cleared', () => { + const populated = { + storesExtendedTags: true, + composer: 'Bach', + grouping: 'Cantatas', + totalTracks: 12, + sortAlbum: 'Album, The', + } as const; + const subsonicLike = source({ + unreportedTagFields: ['grouping', 'totalTracks', 'totalDiscs', 'sortAlbum'], + composer: 'Bach', + }); + expect(plan(subsonicLike, device(populated))).toEqual([]); + }); + + test('a reported empty value clears the device, and the next sync converges', () => { + const src = source({ unreportedTagFields: ['grouping', 'totalTracks', 'totalDiscs'] }); + const dev = device({ storesExtendedTags: true, composer: 'Bach', grouping: 'Cantatas' }); + const ops = plan(src, dev); + expect(ops).toHaveLength(1); + const op = ops[0] as Extract; + expect(op.metadata).toEqual({ composer: '' }); + + const applied = device({ + storesExtendedTags: true, + grouping: 'Cantatas', + composer: op.metadata.composer || undefined, + }); + expect(plan(src, applied)).toEqual([]); + }); + + test('a device that does not store them is never flagged for them', () => { + expect(plan(source({ sortArtist: 'Band, The', composer: 'Bach' }), device())).toEqual([]); + }); +}); diff --git a/packages/podkit-core/src/sync/music/handler.ts b/packages/podkit-core/src/sync/music/handler.ts index 83df215f..5a71fcca 100644 --- a/packages/podkit-core/src/sync/music/handler.ts +++ b/packages/podkit-core/src/sync/music/handler.ts @@ -19,6 +19,7 @@ import { isFileReplacementUpgrade, isSourceLossless, metadataValuesDiffer, + metadataCorrectionFields, classifyDeviceBound, classifySourceBound, } from '../engine/upgrades.js'; @@ -165,6 +166,7 @@ function toCollisionInput(source: CollectionTrack): Omit { expect(trackInput.compilation).toBe(true); }); + it('passes composer, grouping, totals and sort fields to addTrack', async () => { + const tags = { + composer: 'Bach', + grouping: 'Cantatas', + totalTracks: 12, + totalDiscs: 2, + sortTitle: 'Wall, The', + sortArtist: 'Band, The', + sortAlbum: 'Album, The', + sortAlbumArtist: 'Ensemble, The', + sortComposer: 'Bach, J. S.', + }; + const executor = new MusicPipeline(deps); + const plan: SyncPlan = { + operations: [ + { + type: 'add-direct-copy', + source: createCollectionTrack('Artist', 'Song', 'Album', 'mp3', tags), + }, + ], + estimatedTime: 1, + estimatedSize: 5000000, + warnings: [], + }; + + for await (const _p of executor.execute(plan)) { + // iterate + } + + expect(mockAdapter.addTrack.mock.calls[0]![0]).toMatchObject(tags); + }); + // Initial-add bitrate baseline: when a lossy source carries a bitrate, // the copy executor MUST persist it on the iPod track. Without this, // `detectUpgrades`'s `source.bitrate && ipod.bitrate` quality-upgrade gate @@ -2061,6 +2093,41 @@ describe('MusicPipeline - update-metadata operations', () => { expect(mockAdapter.save.mock.calls.length).toBe(0); // ADR-019: engine owns save now }); + it('writes extended tag fields, including cleared ones, to the device', async () => { + let updateFields: Record | null = null; + const deviceTrack = createMockDeviceTrack('Artist', 'Song', 'Album', 'Music/EXT.m4a', { + update: (fields: Record) => { + updateFields = fields; + return deviceTrack; + }, + }); + mockAdapter = createMockDeviceAdapter([deviceTrack]); + const executor = new MusicPipeline(createDependencies(mockAdapter, mockTranscoder)); + const metadata = { + composer: 'Bach', + grouping: '', + totalTracks: 12, + totalDiscs: 0, + sortTitle: 'Wall, The', + sortArtist: 'Band, The', + sortAlbum: '', + sortAlbumArtist: 'Ensemble, The', + sortComposer: 'Bach, J. S.', + }; + const plan: SyncPlan = { + operations: [{ type: 'update-metadata', track: deviceTrack, metadata }], + estimatedTime: 0.01, + estimatedSize: 0, + warnings: [], + }; + + for await (const _p of executor.execute(plan)) { + // iterate + } + + expect(updateFields).toMatchObject(metadata); + }); + it('finds track by filePath for update', async () => { let foundByPath = false; const deviceTrack = createMockDeviceTrack( @@ -2510,6 +2577,48 @@ describe('upgrade operations - execution', () => { expect(upgradeProgress).toBeDefined(); }); + it('a file replacement clears extended fields the source reports but no longer has', async () => { + const existingTrack = Object.assign( + createDeviceTrack('Artist', 'Song', 'Album', { filePath: 'Music/EXISTING.m4a' }), + { + storesExtendedTags: true, + composer: 'Bach', + totalTracks: 12, + sortArtist: 'Artist, The', + grouping: 'Kept', + } + ); + db.getTracks.mockReturnValue([existingTrack]); + const plan: SyncPlan = { + ...createEmptyPlan(), + operations: [ + { + type: 'upgrade-transcode', + source: createCollectionTrack('Artist', 'Song', 'Album', 'flac', { + lossless: true, + sortArtist: 'Artist (The)', + unreportedTagFields: ['grouping'], + }), + target: existingTrack, + reason: 'quality-change', + preset: { name: 'high' }, + }, + ], + }; + + for await (const _p of new MusicPipeline(createDependencies(db, transcoder)).execute(plan, { + artwork: false, + })) { + // iterate + } + + const fields = db.updateTrack.mock.calls + .map((call) => call[1] as Record) + .find((f) => 'filetype' in f)!; + expect(fields).toMatchObject({ composer: '', totalTracks: 0, sortArtist: 'Artist (The)' }); + expect(fields).not.toHaveProperty('grouping'); + }); + it('executes upgrade without preset (copy-based quality-change)', async () => { const existingTrack = createDeviceTrack('Artist', 'Song', 'Album', { filePath: 'Music/EXISTING.m4a', diff --git a/packages/podkit-core/src/sync/music/pipeline.ts b/packages/podkit-core/src/sync/music/pipeline.ts index 69e859fa..d8522ee7 100644 --- a/packages/podkit-core/src/sync/music/pipeline.ts +++ b/packages/podkit-core/src/sync/music/pipeline.ts @@ -83,6 +83,7 @@ import type { ExecuteResult as ExecuteResultFromTypes, } from '../engine/types.js'; import type { DeviceAdapter, DeviceTrackMetadata } from '../../device/adapter.js'; +import { EXTENDED_TAG_FIELDS } from '../../adapters/interface.js'; import { isOggExtension } from '../../audio/containers.js'; import { MusicArtworkManager } from './artwork.js'; import { MusicTransferOps } from './transfer.js'; @@ -1312,6 +1313,10 @@ export class MusicPipeline implements SyncExecutor { if (metadata.compilation !== undefined) { updateFields.compilation = metadata.compilation; } + for (const field of EXTENDED_TAG_FIELDS) { + const value = metadata[field]; + if (value !== undefined) (updateFields as Record)[field] = value; + } if (metadata.normalization !== undefined) { updateFields.normalization = metadata.normalization; } diff --git a/packages/podkit-core/src/sync/music/planner.ts b/packages/podkit-core/src/sync/music/planner.ts index 3600c632..efa7dbeb 100644 --- a/packages/podkit-core/src/sync/music/planner.ts +++ b/packages/podkit-core/src/sync/music/planner.ts @@ -360,6 +360,21 @@ export function changesToMetadata(changes: MetadataChange[]): Partial)[field] = value; + } + } // Request ReplayGain tag writes for transcoded/optimized-copy upgrades. // Direct-copy upgrades preserve source file tags — no write needed. diff --git a/packages/podkit-core/src/types.ts b/packages/podkit-core/src/types.ts index 8b5cfac4..08960c7c 100644 --- a/packages/podkit-core/src/types.ts +++ b/packages/podkit-core/src/types.ts @@ -39,6 +39,15 @@ export interface TrackMetadata { trackNumber?: number; discNumber?: number; compilation?: boolean; + composer?: string; + grouping?: string; + totalTracks?: number; + totalDiscs?: number; + sortTitle?: string; + sortArtist?: string; + sortAlbum?: string; + sortAlbumArtist?: string; + sortComposer?: string; duration?: number; // milliseconds /** * Source bitrate in kbps. Carried on metadata-only updates so the diff --git a/test-packages/test-fixtures/src/mini-tracks.ts b/test-packages/test-fixtures/src/mini-tracks.ts index 29d16642..1ad37ffa 100644 --- a/test-packages/test-fixtures/src/mini-tracks.ts +++ b/test-packages/test-fixtures/src/mini-tracks.ts @@ -34,6 +34,11 @@ export interface MiniTrackOptions { * matters. */ comment?: string; + /** + * Further tags as ffmpeg `-metadata` key/value pairs, written as given + * (e.g. `composer`, `grouping`, `TITLESORT`). Container mapping is ffmpeg's. + */ + tags?: Record; /** Duration in seconds. Defaults to 1. */ durationSeconds?: number; /** Sine frequency in Hz. Defaults to 440. */ @@ -54,6 +59,9 @@ function metadataArgs(opts: MiniTrackOptions): string[] { if (opts.comment !== undefined) { args.push('-metadata', `comment=${opts.comment}`); } + for (const [key, value] of Object.entries(opts.tags ?? {})) { + args.push('-metadata', `${key}=${value}`); + } return args; } From 065f8bd173968892330478f3c5e5fb1e394bfdce Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 18:19:49 +0100 Subject: [PATCH 18/21] fix(sync): converge pending tag corrections with other updates in one sync A metadata correction or Sound Check update pending on a track that also gets an artwork-only operation was dropped until the next sync, and one that also needed a re-encode (a lowered cap, a custom bitrate) went to the in-place update and hid the re-encode. The metadata update now carries every pending correction and Sound Check change whichever reason leads, an artwork-only operation is followed by a metadata update for them, and a match headed to an in-place update is checked for a re-encode too. The replacement writes the source's metadata, so the correction rides along with it; a core field the source no longer has is still cleared by the next sync. --- .../src/sync/music/handler.test.ts | 53 +++++++ .../podkit-core/src/sync/music/handler.ts | 139 +++++++++++++----- 2 files changed, 159 insertions(+), 33 deletions(-) diff --git a/packages/podkit-core/src/sync/music/handler.test.ts b/packages/podkit-core/src/sync/music/handler.test.ts index 39b208ab..a2dd9aa6 100644 --- a/packages/podkit-core/src/sync/music/handler.test.ts +++ b/packages/podkit-core/src/sync/music/handler.test.ts @@ -2691,6 +2691,59 @@ describe('extended tag fields (composer, grouping, totals, sort)', () => { expect(plan(src, applied)).toEqual([]); }); + test('a metadata correction rides along with an artwork update in the same sync', () => { + const ops = plan( + source({ artworkHash: 'bbbbbbbb', hasArtwork: true, composer: 'Bach' }), + device({ + storesExtendedTags: true, + hasArtwork: true, + syncTag: parseSyncTag( + '[podkit:v1 quality=copy transferMode=fast codec=mp3 bitrate=256 art=aaaaaaaa]' + ), + }) + ); + expect(ops.map((op) => op.type).sort()).toEqual(['update-metadata', 'upgrade-artwork']); + expect(ops.find((op) => op.type === 'update-metadata')).toMatchObject({ + metadata: { composer: 'Bach' }, + }); + }); + + test('a metadata correction rides along with a Sound Check update', () => { + const ops = plan( + source({ + composer: 'Bach', + normalization: { source: 'replaygain-track', trackGain: -6, soundcheckValue: 2512 }, + }), + device({ storesExtendedTags: true }) + ); + expect(ops).toHaveLength(1); + const op = ops[0] as Extract; + expect(op.metadata.composer).toBe('Bach'); + expect(op.metadata.normalization).toBeDefined(); + }); + + test('a missing composer does not hide a re-encode to a custom bitrate; the next plan is empty', () => { + const handler = createMusicHandler( + makeConfig({ quality: 'low', customBitrate: 96, reduce: 'always' }) + ); + const planWith = (src: CollectionTrack, dev: DeviceTrack) => + createSyncPlanner(handler).plan(createSyncDiffer(handler).diff([src], [dev])).operations; + + const src = source({ composer: 'Bach' }); + const ops = planWith(src, device({ storesExtendedTags: true })); + expect(ops).toHaveLength(1); + expect(ops[0]).toMatchObject({ type: 'upgrade-transcode', reason: 'quality-change' }); + + const reencoded = device({ + storesExtendedTags: true, + composer: 'Bach', + filetype: 'AAC audio file', + bitrate: 96, + syncTag: parseSyncTag('[podkit:v1 quality=low encoding=vbr bitrate=96 transferMode=fast]'), + }); + expect(planWith(src, reencoded)).toEqual([]); + }); + test('a device that does not store them is never flagged for them', () => { expect(plan(source({ sortArtist: 'Band, The', composer: 'Bach' }), device())).toEqual([]); }); diff --git a/packages/podkit-core/src/sync/music/handler.ts b/packages/podkit-core/src/sync/music/handler.ts index 5a71fcca..c501b1be 100644 --- a/packages/podkit-core/src/sync/music/handler.ts +++ b/packages/podkit-core/src/sync/music/handler.ts @@ -23,6 +23,7 @@ import { classifyDeviceBound, classifySourceBound, } from '../engine/upgrades.js'; +import { EXTENDED_TAG_FIELDS } from '../../adapters/interface.js'; import type { QualityChange, QualityChangeDirection, QualityTarget } from '../engine/upgrades.js'; import { normalizationToDb } from '../../metadata/normalization.js'; import { @@ -61,6 +62,7 @@ import type { UnifiedSyncDiff, } from '../engine/content-type.js'; import { partitionExisting, sweepAllExisting, formatDryRunFromPlan } from '../engine/diff-utils.js'; +import type { PartitionUpdateEntry } from '../engine/diff-utils.js'; import type { MusicSyncConfig, ResolvedMusicConfig } from './config.js'; import { resolveMusicConfig } from './config.js'; import { MusicTrackClassifier, classifierFromConfig } from './classifier.js'; @@ -155,6 +157,24 @@ function buildMusicMetadataChanges( return changes; } +/** Metadata-only reasons a file replacement applies as well (it writes the source's metadata). */ +const REPLACEMENT_CARRIES: ReadonlySet = new Set([ + 'metadata-correction', + 'normalization-update', +]); + +/** Change fields a metadata update can carry alongside an artwork-only operation. */ +const RIDE_ALONG_FIELDS: ReadonlySet = new Set([ + 'genre', + 'year', + 'trackNumber', + 'discNumber', + 'albumArtist', + 'compilation', + 'normalization', + ...EXTENDED_TAG_FIELDS, +]); + /** * Convert a CollectionTrack to a CollisionCheckInput (shared fields only). */ @@ -404,19 +424,35 @@ export class MusicHandler implements ContentTypeHandler< } } - if (update.changes && update.changes.length > 0) continue; // already populated - - if (reason === 'transform-apply') { - // Build changes from device metadata → transformed source metadata - const transformedSource = this.transformSourceForAdd(update.source); - update.changes = buildMusicMetadataChanges(update.device, transformedSource); - } else if (reason === 'transform-remove') { - // Build changes from device metadata → original source metadata - update.changes = buildMusicMetadataChanges(update.device, update.source); - } else if (reason === 'metadata-correction') { - // Build changes for metadata fields that differ - const changes: MetadataChange[] = []; + if (!update.changes?.length) { + if (reason === 'transform-apply') { + // Build changes from device metadata → transformed source metadata + const transformedSource = this.transformSourceForAdd(update.source); + update.changes = buildMusicMetadataChanges(update.device, transformedSource); + } else if (reason === 'transform-remove') { + // Build changes from device metadata → original source metadata + update.changes = buildMusicMetadataChanges(update.device, update.source); + } + } + + // Sound Check and metadata corrections ride along with whichever reason + // leads, so a track with several pending updates converges in one sync. + const changes = [...(update.changes ?? [])]; + const has = (field: string) => changes.some((c) => c.field === field); + if (update.reasons.includes('normalization-update') && !has('normalization')) { + changes.push({ + field: 'normalization', + from: update.device.normalization + ? `${normalizationToDb(update.device.normalization)?.toFixed(1)} dB` + : 'absent', + to: update.source.normalization + ? `${normalizationToDb(update.source.normalization)?.toFixed(1)} dB` + : 'absent', + }); + } + if (update.reasons.includes('metadata-correction')) { for (const field of metadataCorrectionFields(update.source, update.device)) { + if (has(field)) continue; const sourceValue = update.source[field as keyof CollectionTrack]; const ipodValue = update.device[field as keyof DeviceTrack]; if (metadataValuesDiffer(field, sourceValue, ipodValue)) { @@ -427,20 +463,8 @@ export class MusicHandler implements ContentTypeHandler< }); } } - update.changes = changes; - } else if (reason === 'normalization-update') { - update.changes = [ - { - field: 'normalization', - from: update.device.normalization - ? `${normalizationToDb(update.device.normalization)?.toFixed(1)} dB` - : 'absent', - to: update.source.normalization - ? `${normalizationToDb(update.source.normalization)?.toFixed(1)} dB` - : 'absent', - }, - ]; } + update.changes = changes; } } @@ -470,7 +494,14 @@ export class MusicHandler implements ContentTypeHandler< const presetBitrate = this.config.presetBitrate ?? 0; const target = qualityTargetFromConfig(this.config); - partitionExisting(diff, (match) => { + const decide = ( + match: { source: CollectionTrack; device: DeviceTrack }, + report: (entry: { + source: CollectionTrack; + device: DeviceTrack; + qualityChange: QualityChange; + }) => void + ): PartitionUpdateEntry | null => { // Lossy sources take the down-only cap-reduction path. The classifier would // route a compatible/device-native lossy source to a COPY, but a convert // over-cap move must re-encode DOWN to the cap — so the classifier's routing @@ -490,7 +521,7 @@ export class MusicHandler implements ContentTypeHandler< this.config.reductionTolerance ); if (sourceChange && !sourceChange.reEncodes) { - (diff.reportOnlyQualityChanges ??= []).push({ + report({ source: match.source, device: match.device, qualityChange: sourceChange, @@ -547,7 +578,7 @@ export class MusicHandler implements ContentTypeHandler< }, }; } - (diff.reportOnlyQualityChanges ??= []).push({ + report({ source: match.source, device: match.device, qualityChange: change, @@ -559,7 +590,7 @@ export class MusicHandler implements ContentTypeHandler< // copy in `existing` and creates NO operation, so it never inflates // tracksToUpdate. if (!change.reEncodes) { - (diff.reportOnlyQualityChanges ??= []).push({ + report({ source: match.source, device: match.device, qualityChange: change, @@ -614,7 +645,7 @@ export class MusicHandler implements ContentTypeHandler< // moves and preconditions (encoding/lossless boundary) always carry // `reEncodes: true` and fall through to the re-encode path below. if (!change.reEncodes) { - (diff.reportOnlyQualityChanges ??= []).push({ + report({ source: match.source, device: match.device, qualityChange: change, @@ -653,7 +684,27 @@ export class MusicHandler implements ContentTypeHandler< ]; return { reasons: ['quality-change'], changes, qualityChange: change }; - }); + }; + + partitionExisting(diff, (match) => + decide(match, (r) => (diff.reportOnlyQualityChanges ??= []).push(r)) + ); + + // A match already headed to toUpdate for an in-place metadata or Sound + // Check change can also need a re-encode (a lowered cap or a custom + // bitrate). The replacement writes the source's metadata too, so the + // correction rides along with it instead of hiding the re-encode until the + // next sync (a core field the source no longer has is still cleared by the + // next sync's correction). Report-only outcomes for these tracks are + // surfaced by postProcessSourceDownReports. + for (const update of diff.toUpdate) { + if (!update.reasons.every((r) => REPLACEMENT_CARRIES.has(r))) continue; + const entry = decide(update, () => {}); + if (!entry) continue; + update.reasons = [...entry.reasons, ...update.reasons]; + update.qualityChange = entry.qualityChange; + update.changes = [...(entry.changes ?? []), ...(update.changes ?? [])]; + } } /** @@ -1275,12 +1326,12 @@ export class MusicHandler implements ContentTypeHandler< // or removal, but don't replace the audio file — route as upgrade-artwork if (primaryReason === 'artwork-updated' || primaryReason === 'artwork-removed') { ops.push(this.factory.createArtworkUpgrade(source, device, primaryReason as UpgradeReason)); - return ops; + return this.withRideAlongMetadata(ops, source, device, nonSyncTagReasons, changes); } if (this.plansArtworkOnlyAdd(source, device, nonSyncTagReasons)) { ops.push(this.factory.createArtworkUpgrade(source, device, 'artwork-added')); - return ops; + return this.withRideAlongMetadata(ops, source, device, nonSyncTagReasons, changes); } // File-replacement upgrades take priority over path-mismatch. If both are @@ -1316,6 +1367,28 @@ export class MusicHandler implements ContentTypeHandler< return ops; } + /** + * An artwork-only operation leaves the database entry alone, so Sound Check + * and metadata corrections pending on the same track follow it as a + * metadata update instead of waiting for the next sync. + */ + private withRideAlongMetadata( + ops: MusicOperation[], + source: CollectionTrack, + device: DeviceTrack, + reasons: UpdateReason[], + changes: MetadataChange[] | undefined + ): MusicOperation[] { + if (!reasons.includes('metadata-correction') && !reasons.includes('normalization-update')) { + return ops; + } + const metadataChanges = (changes ?? []).filter((c) => RIDE_ALONG_FIELDS.has(c.field)); + if (metadataChanges.length > 0) { + ops.push(this.factory.createMetadataUpdate(device, metadataChanges, source)); + } + return ops; + } + estimateSize(op: MusicOperation): number { return calculateMusicOperationSize(op); } From c9efcd9fdac09a83e3a334062b33d5d4a6087515 Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 18:31:30 +0100 Subject: [PATCH 19/21] fix(sync): converge tag corrections together with a transform apply or remove detectUpdates returned only the transform reason when a transform had to be applied or removed, so a composer or sort correction or a Sound Check update pending on the same track waited for the next sync. The transform reason now carries metadata-correction and normalization-update along when the source has them (Sound Check only when the device uses it), and change building appends them to the transform's changes (fields the transform itself changes keep the transform's value). The single metadata update applies all of them. --- .../src/sync/music/handler.test.ts | 45 +++++++++++++++++++ .../podkit-core/src/sync/music/handler.ts | 14 +++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/packages/podkit-core/src/sync/music/handler.test.ts b/packages/podkit-core/src/sync/music/handler.test.ts index a2dd9aa6..4b8ff3e1 100644 --- a/packages/podkit-core/src/sync/music/handler.test.ts +++ b/packages/podkit-core/src/sync/music/handler.test.ts @@ -2744,6 +2744,51 @@ describe('extended tag fields (composer, grouping, totals, sort)', () => { expect(planWith(src, reencoded)).toEqual([]); }); + test('a pending tag correction and Sound Check converge together with a transform apply', () => { + const handler = createMusicHandler( + makeConfig({ + transforms: { + cleanArtists: { enabled: true, drop: false, format: 'feat. {}', ignore: [] }, + }, + }) + ); + const planWith = (src: CollectionTrack, dev: DeviceTrack) => + createSyncPlanner(handler).plan(createSyncDiffer(handler).diff([src], [dev])).operations; + const normalization = { + source: 'replaygain-track' as const, + trackGain: -6, + soundcheckValue: 2512, + }; + const src = source({ + artist: 'Artist feat. Guest', + title: 'Song', + composer: 'Bach', + normalization, + }); + + const ops = planWith( + src, + device({ storesExtendedTags: true, artist: 'Artist feat. Guest', title: 'Song' }) + ); + expect(ops).toHaveLength(1); + const op = ops[0] as Extract; + expect(op.metadata).toMatchObject({ + artist: 'Artist', + title: 'Song (feat. Guest)', + composer: 'Bach', + }); + expect(op.metadata.normalization).toBeDefined(); + + const applied = device({ + storesExtendedTags: true, + artist: 'Artist', + title: 'Song (feat. Guest)', + composer: 'Bach', + normalization, + }); + expect(planWith(src, applied)).toEqual([]); + }); + test('a device that does not store them is never flagged for them', () => { expect(plan(source({ sortArtist: 'Band, The', composer: 'Bach' }), device())).toEqual([]); }); diff --git a/packages/podkit-core/src/sync/music/handler.ts b/packages/podkit-core/src/sync/music/handler.ts index c501b1be..d23c33e6 100644 --- a/packages/podkit-core/src/sync/music/handler.ts +++ b/packages/podkit-core/src/sync/music/handler.ts @@ -266,13 +266,23 @@ export class MusicHandler implements ContentTypeHandler< if (matchInfo) { const hasTransform = this.generateMatchKey(source) !== this.applyTransformKey(source); if (hasTransform) { + // A metadata correction or Sound Check update pending on the same + // track rides along with the transform's metadata update. + const withCorrection = (reason: UpdateReason): UpdateReason[] => [ + reason, + ...detectUpgrades(source, device).filter( + (r) => + r === 'metadata-correction' || + (r === 'normalization-update' && this.config.audioNormalization !== 'none') + ), + ]; if (!matchInfo.matchedByTransformKey && this.config.transformsEnabled) { // iPod has original metadata, transforms are enabled -> apply - return ['transform-apply']; + return withCorrection('transform-apply'); } if (matchInfo.matchedByTransformKey && !this.config.transformsEnabled) { // iPod has transformed metadata, transforms are disabled -> remove - return ['transform-remove']; + return withCorrection('transform-remove'); } } } From 123242f04d790f970f3aeadf74447a715c35419a Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 18:31:24 +0100 Subject: [PATCH 20/21] feat(sync): write the extended tag fields into file tags, including clears Portable mode mirrors metadata into the iPod's file tags through TagFields, but TagFields, buildTagFieldsFromInput, diffTagFields and TagLibTagWriter knew none of the new fields: an extended-only update diffed to {} and a file pulled off the device kept stale tags while the iTunesDB was current. They now carry composer, grouping, the totals and the five sort fields, with '' and 0 as clears (clearing an unset tag is not a change). Mass-storage change detection for these fields stays off. Several composer values are kept apart: CollectionTrack, DeviceTrackInput and TagFields gain composers, the separate values behind the joined composer, and the writer stores them as separate values. A file replacement on mass-storage still leaves the copied file's own tags alone. Adding a track in portable mode re-tags the copied file from the add input, which carried nothing for a field the source reports as empty, so a stale value in the copied file survived (and the database then matched the source, so no later sync fixed it). The add input now carries the cleared value for each reported field the source lacks; unreported fields stay untouched. iTunes writes TSOT, TSOP and TSOA into ID3v2.3 tags, which taglib only renders for v2.4, so any tag write dropped them; the writer now re-identifies them with v2.3 names before saving. iTunes keeps grouping in GRP1 and the work title in TIT1: when a tag has a GRP1 frame, grouping is written there and TIT1 is left alone, and a cleared grouping keeps an empty GRP1 so the next edit lands there again. The directory reader takes grouping from GRP1 when present. Fixtures cover ID3v2.3 written byte by byte the way iTunes writes it, ID3v2.4 and MP4. --- .changeset/file-tag-extended-fields.md | 10 + .../src/content/docs/reference/config-file.md | 5 +- .../adapters/directory.integration.test.ts | 149 +++++++++++++ .../src/adapters/directory.test.ts | 1 + .../podkit-core/src/adapters/directory.ts | 26 ++- .../podkit-core/src/adapters/interface.ts | 5 + packages/podkit-core/src/device/adapter.ts | 2 + .../device/ipod-adapter.integration.test.ts | 52 +++++ .../mass-storage-tag-writer-helpers.test.ts | 31 +++ ...ass-storage-tag-writer.integration.test.ts | 202 ++++++++++++++++++ .../src/device/mass-storage-tag-writer.ts | 131 +++++++++++- .../sync/music/pipeline.integration.test.ts | 72 ++++++- .../src/sync/music/pipeline.test.ts | 36 ++++ .../podkit-core/src/sync/music/pipeline.ts | 3 + .../podkit-core/src/sync/music/transfer.ts | 14 +- test-packages/test-fixtures/src/lib.ts | 2 + .../test-fixtures/src/mini-tracks.ts | 70 ++++++ 17 files changed, 806 insertions(+), 5 deletions(-) create mode 100644 .changeset/file-tag-extended-fields.md diff --git a/.changeset/file-tag-extended-fields.md b/.changeset/file-tag-extended-fields.md new file mode 100644 index 00000000..77725658 --- /dev/null +++ b/.changeset/file-tag-extended-fields.md @@ -0,0 +1,10 @@ +--- +"podkit": minor +"@podkit/core": minor +--- + +Write composer, grouping, track/disc totals and sort fields into file tags + +Under `transferMode = "portable"` composer, grouping, the totals and the sort fields, including clears, are now written into the iPod's file tags as well as its database, and podkit's tag writer (also used for mass-storage tag updates) writes them for MP3, M4A, FLAC, OGG and Opus. Several composers are kept as separate values. iTunes-style ID3v2.3 tags keep their title, artist and album sort frames on every tag write, and a grouping stored in iTunes' GRP1 frame is read and written there, leaving the work title in TIT1 alone. Mass-storage change detection is unchanged. + +With `transferMode = "portable"`, a track whose database entry a sync updates for these fields gets the same change in its file tags. No audio is copied or re-encoded for it. diff --git a/packages/docs-site/src/content/docs/reference/config-file.md b/packages/docs-site/src/content/docs/reference/config-file.md index 0c4375f8..ec4d2f5d 100644 --- a/packages/docs-site/src/content/docs/reference/config-file.md +++ b/packages/docs-site/src/content/docs/reference/config-file.md @@ -143,7 +143,10 @@ Why iPod portable is a separate case: the iPod's firmware plays from the iTunesDB and never looks at file tags during playback, but a user who copies files off the device (recovery, ripping back to a library) needs the file to be self-describing. `portable` opts into that extra -write at the cost of a small per-track tag-write. +write at the cost of a small per-track tag-write. The file tags carry +the same fields as the iTunesDB, including composer, grouping, +track/disc totals and the sort fields; a field cleared at the source is +removed from the file too. After upgrading from a podkit version that did not write file tags, the first sync on an existing mass-storage device will likely show a diff --git a/packages/podkit-core/src/adapters/directory.integration.test.ts b/packages/podkit-core/src/adapters/directory.integration.test.ts index 1a51ae91..de24cbf6 100644 --- a/packages/podkit-core/src/adapters/directory.integration.test.ts +++ b/packages/podkit-core/src/adapters/directory.integration.test.ts @@ -13,12 +13,15 @@ import { tmpdir } from 'node:os'; import { ensureFixturesExist, generateMiniFlac, + generateMiniM4a, generateMiniMp3, + generateMiniMp3WithId3v23, getMultiFormatFixturesDir, getMultiFormatEmbeddedFixturesDir, requireFFmpeg, SCENARIO_ARTISTS, } from '@podkit/test-fixtures'; +import { File as TagFile, Id3v2Settings } from 'node-taglib-sharp'; import { DirectoryAdapter } from './directory.js'; requireFFmpeg(); @@ -365,6 +368,152 @@ describe('DirectoryAdapter — composer, grouping, totals and sort tags', () => }); }); +// Synthetic ID3v2.3, ID3v2.4 and MP4 files, read back through the adapter's +// own reader. ID3v2.4 and MP4 are tagged with node-taglib-sharp (ffmpeg's +// muxers cannot write TSO2/TSOC or every MP4 sort atom). ID3v2.3 is written +// byte by byte the way iTunes does it, with the TSOT/TSOP/TSOA sort frames +// that ID3v2.3 itself lacks and taglib therefore drops. + +describe('DirectoryAdapter — extended tags in ID3v2.3, ID3v2.4 and MP4', () => { + let dir: string; + + const full = { + composers: ['Jóhann Sebastián Bäch', '久石 譲'], + grouping: 'Kantaten · 聖歌', + track: 3, + trackCount: 12, + disc: 1, + discCount: 2, + titleSort: 'Wäll, The', + performersSort: ['Bänd, The'], + albumSort: 'Älbum, The', + albumArtistsSort: ['Ensemble, The'], + composersSort: ['Bäch, Jóhann Sebastián'], + }; + const expectedFull = { + composer: 'Jóhann Sebastián Bäch, 久石 譲', + grouping: 'Kantaten · 聖歌', + totalTracks: 12, + totalDiscs: 2, + sortTitle: 'Wäll, The', + sortArtist: 'Bänd, The', + sortAlbum: 'Älbum, The', + sortAlbumArtist: 'Ensemble, The', + sortComposer: 'Bäch, Jóhann Sebastián', + }; + // Blank values a tagger can leave behind; they must read as unset. + const blank = { + composers: [' '], + grouping: ' ', + titleSort: ' ', + performersSort: [' '], + albumSort: ' ', + albumArtistsSort: [' '], + composersSort: [' '], + }; + + function tag(file: string, fields: Record, id3Version?: 3 | 4) { + const previous = [Id3v2Settings.defaultVersion, Id3v2Settings.forceDefaultVersion] as const; + if (id3Version) { + Id3v2Settings.defaultVersion = id3Version; + Id3v2Settings.forceDefaultVersion = true; + } + try { + const f = TagFile.createFromPath(file); + Object.assign(f.tag, { title: 'The Wäll', performers: ['The Bänd'], album: 'The Älbum' }); + Object.assign(f.tag, fields); + f.save(); + f.dispose(); + } finally { + [Id3v2Settings.defaultVersion, Id3v2Settings.forceDefaultVersion] = previous; + } + } + + beforeAll(async () => { + dir = join(tmpdir(), `podkit-tag-fixtures-${Date.now()}`); + await mkdir(dir, { recursive: true }); + for (const [name, fields] of [ + ['full', full], + ['blank', blank], + ] as const) { + tag(generateMiniMp3(dir, { filename: `${name}-v24.mp3` }), fields, 4); + tag(generateMiniM4a(dir, { filename: `${name}.m4a` }), fields); + } + const base: Array<[string, string]> = [ + ['TIT2', 'The Wäll'], + ['TPE1', 'The Bänd'], + ['TALB', 'The Älbum'], + ]; + generateMiniMp3WithId3v23(dir, 'full-v23.mp3', [ + ...base, + ['TCOM', full.composers.join('/')], + ['TIT1', full.grouping], + ['TRCK', '3/12'], + ['TPOS', '1/2'], + ['TSOT', full.titleSort], + ['TSOP', full.performersSort[0]!], + ['TSOA', full.albumSort], + ['TSO2', full.albumArtistsSort[0]!], + ['TSOC', full.composersSort[0]!], + ]); + generateMiniMp3WithId3v23(dir, 'blank-v23.mp3', [ + ...base, + ...(['TCOM', 'TIT1', 'TSOT', 'TSOP', 'TSOA', 'TSO2', 'TSOC'].map((id) => [id, ' ']) as Array< + [string, string] + >), + ]); + }); + + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + async function extended(name: string) { + const tracks = await new DirectoryAdapter({ path: dir }).getItems(); + const t = tracks.find((x) => x.filePath.endsWith(name))!; + return { + composer: t.composer, + grouping: t.grouping, + totalTracks: t.totalTracks, + totalDiscs: t.totalDiscs, + sortTitle: t.sortTitle, + sortArtist: t.sortArtist, + sortAlbum: t.sortAlbum, + sortAlbumArtist: t.sortAlbumArtist, + sortComposer: t.sortComposer, + }; + } + + for (const name of ['full-v23.mp3', 'full-v24.mp3']) { + it(`reads all nine fields, Unicode intact, from ${name}`, async () => { + expect(await extended(name)).toEqual(expectedFull); + }); + } + + it('reads all nine fields from MP4, where several composers share one ©wrt atom', async () => { + expect(await extended('full.m4a')).toEqual({ + ...expectedFull, + composer: 'Jóhann Sebastián Bäch; 久石 譲', + }); + }); + + it('reads grouping from an iTunes GRP1 frame, not the TIT1 work title', async () => { + generateMiniMp3WithId3v23(dir, 'grp1-v23.mp3', [ + ['TIT2', 'Grouped'], + ['TIT1', 'Mass in B minor'], + ['GRP1', 'Kyrie'], + ]); + expect((await extended('grp1-v23.mp3')).grouping).toBe('Kyrie'); + }); + + for (const name of ['blank-v23.mp3', 'blank-v24.mp3', 'blank.m4a']) { + it(`reads blank values as unset from ${name}`, async () => { + const values = await extended(name); + expect(Object.values(values).every((v) => v === undefined)).toBe(true); + }); + } +}); + // The large-collection performance test moved to // `./directory.perf.test.ts` and runs via `bun run test:perf`. Default // `bun run test` / `test:integration` flows exclude `*.perf.test.ts` via diff --git a/packages/podkit-core/src/adapters/directory.test.ts b/packages/podkit-core/src/adapters/directory.test.ts index 6f888334..0c49b57c 100644 --- a/packages/podkit-core/src/adapters/directory.test.ts +++ b/packages/podkit-core/src/adapters/directory.test.ts @@ -118,6 +118,7 @@ describe('DirectoryAdapter', () => { const [track] = await newAdapter({ path: '/music' }).getItems(); expect(track!.composer).toBe('John Lennon, Paul McCartney'); + expect(track!.composers).toEqual(['John Lennon', 'Paul McCartney']); expect(track!.grouping).toBeUndefined(); expect(track!.sortTitle).toBeUndefined(); expect(track!.totalTracks).toBeUndefined(); diff --git a/packages/podkit-core/src/adapters/directory.ts b/packages/podkit-core/src/adapters/directory.ts index 72e10860..4751c735 100644 --- a/packages/podkit-core/src/adapters/directory.ts +++ b/packages/podkit-core/src/adapters/directory.ts @@ -121,6 +121,29 @@ function joinTagValues(values: string[] | undefined): string | undefined { return parts.length > 0 ? parts.join(', ') : undefined; } +/** + * iTunes stores grouping in an ID3 GRP1 frame and the work title in TIT1; + * music-metadata maps both to `grouping`. A GRP1 frame, when present, is the + * grouping, even empty (a cleared grouping keeps one, so TIT1 never takes + * its place). + */ +function itunesGrouping( + native: Record> | undefined +): string | undefined { + for (const [format, frames] of Object.entries(native ?? {})) { + if (!format.startsWith('ID3v2')) continue; + const grp1 = frames.find((f) => f.id === 'GRP1'); + if (grp1 && typeof grp1.value === 'string') return grp1.value; + } + return undefined; +} + +/** The non-blank values of a multi-value tag, when there are several. */ +function multipleTagValues(values: string[] | undefined): string[] | undefined { + const parts = values?.map((v) => v.trim()).filter((v) => v.length > 0) ?? []; + return parts.length > 1 ? parts : undefined; +} + /** A text tag, unset when blank. */ function tagText(value: string | undefined): string | undefined { return value?.trim() ? value : undefined; @@ -362,7 +385,8 @@ export class DirectoryAdapter implements CollectionAdapter { } }); + it('iPod portable: extended tag changes and clears reach the file on disk', async () => { + const testIpod = await createTestIpod(); + try { + const db = await IpodDatabase.open(testIpod.path); + try { + const adapter = new IpodDeviceAdapter(db, capsForGeneration('classic_7g')); + const added = adapter.addTrack({ + title: 'Portable Tags', + artist: 'The Band', + album: 'Album', + composer: 'Clara Schumann', + grouping: 'Lieder', + totalTracks: 12, + sortArtist: 'Band, The', + sortComposer: 'Schumann, Clara', + transferMode: 'portable', + }); + added.copyFile(mp3Path); + await adapter.save(); + + const track = adapter.getTracks()[0]!; + adapter.updateTrack(track, { + grouping: '', + totalTracks: 0, + sortArtist: 'Band (The)', + transferMode: 'portable', + }); + await adapter.save(); + + const filePath = join(testIpod.path, track.filePath.replace(/^:/, '').replaceAll(':', '/')); + const { common } = await mm.parseFile(filePath, { skipCovers: true }); + expect({ + composer: common.composer, + grouping: common.grouping || undefined, + totalTracks: common.track.of || undefined, + artistsort: common.artistsort, + composersort: common.composersort, + }).toEqual({ + composer: ['Clara Schumann'], + grouping: undefined, + totalTracks: undefined, + artistsort: 'Band (The)', + composersort: 'Schumann, Clara', + }); + } finally { + db.close(); + } + } finally { + await testIpod.cleanup(); + } + }); + it('iPod portable: multiple updates to one track coalesce into a single writeTags call', async () => { const testIpod = await createTestIpod(); try { diff --git a/packages/podkit-core/src/device/mass-storage-tag-writer-helpers.test.ts b/packages/podkit-core/src/device/mass-storage-tag-writer-helpers.test.ts index 48fd958a..f9d09878 100644 --- a/packages/podkit-core/src/device/mass-storage-tag-writer-helpers.test.ts +++ b/packages/podkit-core/src/device/mass-storage-tag-writer-helpers.test.ts @@ -17,6 +17,28 @@ describe('buildTagFieldsFromInput', () => { expect(buildTagFieldsFromInput({})).toEqual({}); }); + test('copies the extended fields', () => { + const extended = { + composer: 'C', + grouping: 'G', + totalTracks: 12, + totalDiscs: 2, + sortTitle: 'ST', + sortArtist: 'SA', + sortAlbum: 'SB', + sortAlbumArtist: 'SAA', + sortComposer: 'SC', + }; + expect(buildTagFieldsFromInput({ title: 'T', ...extended })).toEqual({ + title: 'T', + ...extended, + }); + expect(buildTagFieldsFromInput({ composer: 'A, B', composers: ['A', 'B'] })).toEqual({ + composer: 'A, B', + composers: ['A', 'B'], + }); + }); + test('copies every supplied textual field', () => { expect( buildTagFieldsFromInput({ @@ -72,6 +94,15 @@ describe('diffTagFields', () => { }); }); + test('carries changed and cleared extended fields, and skips clears of unset tags', () => { + expect( + diffTagFields( + { composer: 'Bach', sortArtist: 'Band, The', totalTracks: 12 }, + { composer: 'Bach', sortArtist: '', totalTracks: 10, grouping: '', totalDiscs: 0 } + ) + ).toEqual({ sortArtist: '', totalTracks: 10 }); + }); + test('handles boolean compilation toggle off explicitly', () => { expect(diffTagFields({ compilation: true }, { compilation: false })).toEqual({ compilation: false, diff --git a/packages/podkit-core/src/device/mass-storage-tag-writer.integration.test.ts b/packages/podkit-core/src/device/mass-storage-tag-writer.integration.test.ts index a8628c18..82ed342c 100644 --- a/packages/podkit-core/src/device/mass-storage-tag-writer.integration.test.ts +++ b/packages/podkit-core/src/device/mass-storage-tag-writer.integration.test.ts @@ -16,12 +16,15 @@ import { generateMiniFlac, generateMiniM4a, generateMiniMp3, + generateMiniMp3WithId3Frames, + generateMiniMp3WithId3v23, generateMiniOggOpus, generateMiniOggVorbis, requireFFmpeg, } from '@podkit/test-fixtures'; import { PODKIT_TEMP_SUFFIX } from '../utils/atomic-fs.js'; import { TagLibTagWriter, type TagFields } from './mass-storage-tag-writer.js'; +import { DirectoryAdapter } from '../adapters/directory.js'; requireFFmpeg(); @@ -102,6 +105,23 @@ function generateTestImage(width = 100, height = 100): Buffer { return Buffer.from(result); } +/** The nine extended fields as music-metadata reads them; empty values as undefined. */ +function extendedOf(metadata: mm.IAudioMetadata) { + const c = metadata.common; + const text = (v: string | undefined) => v || undefined; + return { + composer: c.composer?.length ? c.composer : undefined, + grouping: text(c.grouping), + totalTracks: c.track.of || undefined, + totalDiscs: c.disk.of || undefined, + titlesort: text(c.titlesort), + artistsort: text(c.artistsort), + albumsort: text(c.albumsort), + albumartistsort: text(c.albumartistsort), + composersort: text(c.composersort), + }; +} + /** Read the comment tag from an audio file using music-metadata */ async function readComment(filePath: string): Promise { const metadata = await mm.parseFile(filePath, { skipCovers: true }); @@ -221,6 +241,15 @@ describe('TagLibTagWriter', () => { discNumber: 2, compilation: true, comment: '[podkit:v1 quality=high]', + composer: 'Clara Schumann', + grouping: 'Lieder', + totalTracks: 12, + totalDiscs: 3, + sortTitle: 'Title, New', + sortArtist: 'Artist, New', + sortAlbum: 'Album, New', + sortAlbumArtist: 'Album Artist, New', + sortComposer: 'Schumann, Clara', }; function assertAllFields(metadata: mm.IAudioMetadata): void { @@ -233,6 +262,17 @@ describe('TagLibTagWriter', () => { expect(metadata.common.track.no).toBe(7); expect(metadata.common.disk.no).toBe(2); expect(metadata.common.compilation).toBe(true); + expect(extendedOf(metadata)).toEqual({ + composer: ['Clara Schumann'], + grouping: 'Lieder', + totalTracks: 12, + totalDiscs: 3, + titlesort: 'Title, New', + artistsort: 'Artist, New', + albumsort: 'Album, New', + albumartistsort: 'Album Artist, New', + composersort: 'Schumann, Clara', + }); } test('FLAC: round-trips every supported field', async () => { @@ -281,6 +321,168 @@ describe('TagLibTagWriter', () => { }); }); + describe('writeTags — clearing extended fields', () => { + const set: TagFields = { + composer: 'Clara Schumann', + grouping: 'Lieder', + totalTracks: 12, + totalDiscs: 3, + sortTitle: 'T, x', + sortArtist: 'A, x', + sortAlbum: 'B, x', + sortAlbumArtist: 'C, x', + sortComposer: 'D, x', + }; + const cleared: TagFields = { + composer: '', + grouping: '', + totalTracks: 0, + totalDiscs: 0, + sortTitle: '', + sortArtist: '', + sortAlbum: '', + sortAlbumArtist: '', + sortComposer: '', + }; + const generators: Array<[string, (dir: string, name: string) => string]> = [ + ['FLAC', (d, n) => generateFlac(d, `${n}.flac`)], + ['MP3', (d, n) => generateMp3(d, `${n}.mp3`)], + ['M4A', (d, n) => generateM4a(d, `${n}.m4a`)], + ['OGG', (d, n) => generateOgg(d, `${n}.ogg`)], + ]; + for (const [format, generate] of generators) { + test(`${format}: '' and 0 remove the tags`, async () => { + const filePath = generate(tempDir, 'clear'); + await writer.writeTags(filePath, set); + await writer.writeTags(filePath, cleared); + const metadata = await mm.parseFile(filePath, { skipCovers: true }); + expect(extendedOf(metadata)).toEqual({ + composer: undefined, + grouping: undefined, + totalTracks: undefined, + totalDiscs: undefined, + titlesort: undefined, + artistsort: undefined, + albumsort: undefined, + albumartistsort: undefined, + composersort: undefined, + }); + }); + } + }); + + describe('writeTags — multi-value composers', () => { + for (const [format, generate] of [ + ['FLAC', (d: string) => generateFlac(d, 'composers.flac')], + ['MP3', (d: string) => generateMp3(d, 'composers.mp3')], + ] as const) { + test(`${format}: separate composer values stay separate`, async () => { + const filePath = generate(tempDir); + await writer.writeTags(filePath, { + composer: 'John Lennon, Paul McCartney', + composers: ['John Lennon', 'Paul McCartney'], + }); + const metadata = await mm.parseFile(filePath, { skipCovers: true }); + expect(metadata.common.composer).toEqual(['John Lennon', 'Paul McCartney']); + }); + } + }); + + describe('writeTags — iTunes ID3v2.3 frames', () => { + const itunesV23: Array<[string, string]> = [ + ['TIT2', 'Title'], + ['TPE1', 'Artist'], + ['TALB', 'Album'], + ['TIT1', 'Mass in B minor'], + ['GRP1', 'Kyrie'], + ['TSOT', 'Title, x'], + ['TSOP', 'Artist, x'], + ['TSOA', 'Album, x'], + ['TSO2', 'Album Artist, x'], + ['TSOC', 'Composer, x'], + ]; + const native = async (filePath: string) => { + const metadata = await mm.parseFile(filePath, { skipCovers: true }); + const frames = metadata.native['ID3v2.3'] ?? []; + return Object.fromEntries(frames.map((f) => [f.id, f.value])); + }; + + test('an unrelated change keeps all five sort frames and both grouping frames', async () => { + const filePath = generateMiniMp3WithId3v23(tempDir, 'v23.mp3', itunesV23); + await writer.writeTags(filePath, { album: 'Changed', comment: '[podkit:v1 quality=copy]' }); + await writer.writePicture(filePath, generateTestImage(8, 8)); + + expect(await native(filePath)).toMatchObject({ + TALB: 'Changed', + TIT1: 'Mass in B minor', + GRP1: 'Kyrie', + TSOT: 'Title, x', + TSOP: 'Artist, x', + TSOA: 'Album, x', + TSO2: 'Album Artist, x', + TSOC: 'Composer, x', + }); + }); + + test('sort and grouping updates stay in the v2.3 frames; the work title is untouched', async () => { + const filePath = generateMiniMp3WithId3v23(tempDir, 'v23-update.mp3', itunesV23); + await writer.writeTags(filePath, { sortTitle: 'Title, y', grouping: 'Gloria' }); + + const frames = await native(filePath); + expect(frames).toMatchObject({ TSOT: 'Title, y', GRP1: 'Gloria', TIT1: 'Mass in B minor' }); + expect(frames.TSOP).toBe('Artist, x'); + }); + + test('clearing removes the sort frames, empties GRP1 and keeps the work title', async () => { + const filePath = generateMiniMp3WithId3v23(tempDir, 'v23-clear.mp3', itunesV23); + await writer.writeTags(filePath, { + grouping: '', + sortTitle: '', + sortArtist: '', + sortAlbum: '', + sortAlbumArtist: '', + sortComposer: '', + }); + + const frames = await native(filePath); + for (const id of ['TSOT', 'TSOP', 'TSOA', 'TSO2', 'TSOC']) { + expect([id, frames[id]]).toEqual([id, undefined]); + } + expect(frames.GRP1).toBe(''); + expect(frames.TIT1).toBe('Mass in B minor'); + }); + + for (const version of [3, 4] as const) { + test(`ID3v2.${version}: clear, reread, regroup — the work title is never grouping and never overwritten`, async () => { + const filePath = generateMiniMp3WithId3Frames( + tempDir, + `work-v2${version}.mp3`, + [ + ['TIT2', 'Kyrie eleison'], + ['TIT1', 'Mass in B minor'], + ['GRP1', 'Kyrie'], + ], + version + ); + const groupingOf = async () => + (await new DirectoryAdapter({ path: tempDir }).getItems()).find((t) => + t.filePath.endsWith(`work-v2${version}.mp3`) + )?.grouping; + const workTitle = async () => + (await mm.parseFile(filePath)).native[`ID3v2.${version}`]?.find((f) => f.id === 'TIT1') + ?.value; + + await writer.writeTags(filePath, { grouping: '' }); + expect(await groupingOf()).toBeUndefined(); + expect(await workTitle()).toBe('Mass in B minor'); + + await writer.writeTags(filePath, { grouping: 'Gloria' }); + expect(await groupingOf()).toBe('Gloria'); + expect(await workTitle()).toBe('Mass in B minor'); + }); + } + }); + describe('writeTags — partial updates and edge cases', () => { test('undefined fields leave existing values untouched', async () => { const filePath = generateFlac(tempDir, 'partial.flac'); diff --git a/packages/podkit-core/src/device/mass-storage-tag-writer.ts b/packages/podkit-core/src/device/mass-storage-tag-writer.ts index 3afa9e0f..41ff6367 100644 --- a/packages/podkit-core/src/device/mass-storage-tag-writer.ts +++ b/packages/podkit-core/src/device/mass-storage-tag-writer.ts @@ -11,9 +11,21 @@ import * as fs from 'node:fs'; -import { ByteVector, File as TagFile, Picture, PictureType, SeekOrigin } from 'node-taglib-sharp'; +import { + ByteVector, + File as TagFile, + Id3v2FrameIdentifier, + Id3v2Tag, + Id3v2TextInformationFrame, + Picture, + PictureType, + SeekOrigin, + StringType, + TagTypes, +} from 'node-taglib-sharp'; import type { IFileAbstraction, IStream } from 'node-taglib-sharp'; +import { EXTENDED_TAG_FIELDS } from '../adapters/interface.js'; import { atomicWriteFileWithSync } from '../utils/atomic-fs.js'; import { CategorizedSyncError, errnoOf } from '../sync/engine/errors.js'; import type { ErrorCause } from '../sync/engine/types.js'; @@ -384,6 +396,17 @@ export interface TagFields { discNumber?: number; compilation?: boolean; comment?: string; + composer?: string; + /** Separate composer values to write instead of the joined `composer`. */ + composers?: readonly string[]; + grouping?: string; + totalTracks?: number; + totalDiscs?: number; + sortTitle?: string; + sortArtist?: string; + sortAlbum?: string; + sortAlbumArtist?: string; + sortComposer?: string; replayGain?: ReplayGainFields; } @@ -399,8 +422,22 @@ interface TagFieldsSource { discNumber?: number; compilation?: boolean; comment?: string; + composer?: string; + /** Separate composer values to write instead of the joined `composer`. */ + composers?: readonly string[]; + grouping?: string; + totalTracks?: number; + totalDiscs?: number; + sortTitle?: string; + sortArtist?: string; + sortAlbum?: string; + sortAlbumArtist?: string; + sortComposer?: string; } +/** '' and 0 clear an extended tag; unset on the current side reads the same. */ +const isClearedTag = (v: unknown): boolean => v === undefined || v === '' || v === 0; + /** * Project a TagFields-compatible source object (e.g. a `DeviceTrackInput`) * into a `TagFields` containing only the fields the caller actually @@ -419,6 +456,10 @@ export function buildTagFieldsFromInput(source: TagFieldsSource): TagFields { if (source.discNumber !== undefined) out.discNumber = source.discNumber; if (source.compilation !== undefined) out.compilation = source.compilation; if (source.comment !== undefined) out.comment = source.comment; + for (const field of EXTENDED_TAG_FIELDS) { + if (source[field] !== undefined) (out as Record)[field] = source[field]; + } + if (out.composer && source.composers) out.composers = source.composers; return out; } @@ -460,9 +501,94 @@ export function diffTagFields(current: TagFieldsSource, fields: TagFieldsSource) if (fields.comment !== undefined && fields.comment !== current.comment) { out.comment = fields.comment; } + for (const field of EXTENDED_TAG_FIELDS) { + const next = fields[field]; + if (next === undefined) continue; + const prev = current[field]; + const unchanged = isClearedTag(next) ? isClearedTag(prev) : next === prev; + if (!unchanged) (out as Record)[field] = next; + } + if (out.composer && fields.composers) out.composers = fields.composers; return out; } +/** Write the extended fields; '' and 0 remove the tag. */ +function applyExtendedTags(file: TagFile, fields: TagFields): void { + const tag = file.tag; + const list = (v: string) => (v ? [v] : []); + if (fields.composer !== undefined) { + tag.composers = + fields.composer && fields.composers ? [...fields.composers] : list(fields.composer); + } + if (fields.grouping !== undefined && !writeGrp1Grouping(file, fields.grouping)) { + tag.grouping = fields.grouping; + } + if (fields.totalTracks !== undefined) tag.trackCount = fields.totalTracks; + if (fields.totalDiscs !== undefined) tag.discCount = fields.totalDiscs; + if (fields.sortTitle !== undefined) tag.titleSort = fields.sortTitle; + if (fields.sortArtist !== undefined) tag.performersSort = list(fields.sortArtist); + if (fields.sortAlbum !== undefined) tag.albumSort = fields.sortAlbum; + if (fields.sortAlbumArtist !== undefined) tag.albumArtistsSort = list(fields.sortAlbumArtist); + if (fields.sortComposer !== undefined) tag.composersSort = list(fields.sortComposer); +} + +/** + * Frame ids taglib knows only for ID3v2.4 but iTunes also writes into v2.3 + * tags (TSO2/TSOC have v2.3 names in taglib already), plus iTunes' GRP1 + * grouping frame, which taglib does not know at all. Identifiers with a + * v2.3 name let these frames survive a v2.3 render. + */ +const ITUNES_V23_IDS = new Map( + ['TSOT', 'TSOP', 'TSOA', 'GRP1'].map((id) => [ + id, + new Id3v2FrameIdentifier(id, id, undefined as unknown as string), + ]) +); + +function id3v2Of(file: TagFile): Id3v2Tag | undefined { + return (file.getTag(TagTypes.Id3v2, false) as Id3v2Tag | undefined) ?? undefined; +} + +function textFrame(id: string, text: string): Id3v2TextInformationFrame { + const frame = Id3v2TextInformationFrame.fromIdentifier(ITUNES_V23_IDS.get(id)!, StringType.UTF16); + frame.text = [text]; + return frame; +} + +/** + * iTunes keeps grouping in GRP1 (and the work title in TIT1). When the tag + * already has a GRP1 frame, grouping is written there and TIT1 is left + * alone. A cleared grouping keeps an empty GRP1 frame: without it the + * reader would fall back to TIT1 and a later grouping edit would land in + * TIT1, overwriting the work title. Returns false when there is no GRP1 + * frame to write. + */ +function writeGrp1Grouping(file: TagFile, grouping: string): boolean { + const id3 = id3v2Of(file); + const existing = id3?.frames.filter((f) => f.frameId.toString() === 'GRP1') ?? []; + if (!id3 || existing.length === 0) return false; + for (const frame of existing) id3.removeFrame(frame); + id3.addFrame(textFrame('GRP1', grouping)); + return true; +} + +/** + * taglib renders TSOT/TSOP/TSOA only for ID3v2.4, so saving a v2.3 tag + * would drop the title, artist and album sort frames iTunes writes there, + * whatever field was changed. Re-identify them with v2.3 names first. + */ +function keepItunesV23Frames(file: TagFile): void { + const id3 = id3v2Of(file); + if (!id3 || id3.version !== 3) return; + // replaceFrame mutates the live frame list, so iterate a copy. + for (const frame of id3.frames.slice()) { + const id = frame.frameId.toString(); + if (id === 'GRP1' || !ITUNES_V23_IDS.has(id)) continue; + const text = (frame as Id3v2TextInformationFrame).text ?? []; + id3.replaceFrame(frame, textFrame(id, text.join('/'))); + } +} + /** * Interface for writing metadata tags to audio files. * Injectable for testing — tests can provide a mock implementation. @@ -517,6 +643,7 @@ export class TagLibTagWriter implements TagWriter { if (fields.discNumber !== undefined) tag.disc = fields.discNumber; if (fields.compilation !== undefined) tag.isCompilation = fields.compilation; if (fields.comment !== undefined) tag.comment = fields.comment; + applyExtendedTags(file, fields); if (fields.replayGain !== undefined) { const rg = fields.replayGain; tag.replayGainTrackGain = rg.trackGain; @@ -524,6 +651,7 @@ export class TagLibTagWriter implements TagWriter { if (rg.albumGain !== undefined) tag.replayGainAlbumGain = rg.albumGain; if (rg.albumPeak !== undefined) tag.replayGainAlbumPeak = rg.albumPeak; } + keepItunesV23Frames(file); file.save(); } finally { file.dispose(); @@ -539,6 +667,7 @@ export class TagLibTagWriter implements TagWriter { const picture = Picture.fromData(ByteVector.fromByteArray(imageData)); picture.type = PictureType.FrontCover; file.tag.pictures = [picture]; + keepItunesV23Frames(file); file.save(); } finally { file.dispose(); diff --git a/packages/podkit-core/src/sync/music/pipeline.integration.test.ts b/packages/podkit-core/src/sync/music/pipeline.integration.test.ts index eb1b397f..818887ee 100644 --- a/packages/podkit-core/src/sync/music/pipeline.integration.test.ts +++ b/packages/podkit-core/src/sync/music/pipeline.integration.test.ts @@ -19,7 +19,8 @@ import { describe, expect, it, beforeAll, afterAll } from 'bun:test'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { spawn } from 'node:child_process'; +import { execFileSync, spawn } from 'node:child_process'; +import * as mm from 'music-metadata'; import { requireFFmpeg, requireGpodTool } from '@podkit/test-fixtures'; import { requireLibgpodNode } from '@podkit/libgpod-node'; import { @@ -299,6 +300,75 @@ describe('SyncExecutor integration', () => { }); }); + describe('portable add', () => { + it('clears a field the source reports as empty and keeps an unreported one', async () => { + const testIpod = await createTestIpod(); + try { + // An MP3 carrying a composer and a grouping tag of its own. + const tagged = join(testDir, 'portable-stale.mp3'); + execFileSync('ffmpeg', [ + '-v', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:duration=1', + '-c:a', + 'libmp3lame', + '-b:a', + '128k', + '-metadata', + 'composer=Stale Composer', + '-metadata', + 'grouping=Kept Grouping', + tagged, + ]); + + const db = await IpodDatabase.open(testIpod.path); + try { + const device = new IpodDeviceAdapter(db, capsForGeneration('classic_7g')); + // The source reports composer (empty) but not grouping. + const source: CollectionTrack = { + ...createCollectionTrack('Test Artist', 'Stale', 'Test Album', tagged, 'mp3'), + unreportedTagFields: ['grouping'], + }; + const plan: SyncPlan = { + operations: [{ type: 'add-direct-copy', source }], + estimatedTime: 1, + estimatedSize: 50000, + warnings: [], + }; + + const result = await runMusicPlan( + plan, + { device, transcoder }, + { + transferMode: 'portable', + } + ); + expect(result.failed).toBe(0); + await device.save(); + + const track = db.getTracks()[0]!; + const filePath = join( + testIpod.path, + track.filePath.replace(/^:/, '').replaceAll(':', '/') + ); + const { common } = await mm.parseFile(filePath, { skipCovers: true }); + expect({ composer: common.composer, grouping: common.grouping }).toEqual({ + composer: undefined, + grouping: 'Kept Grouping', + }); + } finally { + db.close(); + } + } finally { + await testIpod.cleanup(); + } + }); + }); + describe('transcode operation', () => { it('transcodes a WAV file and adds to iPod', async () => { const testIpod = await createTestIpod(); diff --git a/packages/podkit-core/src/sync/music/pipeline.test.ts b/packages/podkit-core/src/sync/music/pipeline.test.ts index 7047ea57..b7d54139 100644 --- a/packages/podkit-core/src/sync/music/pipeline.test.ts +++ b/packages/podkit-core/src/sync/music/pipeline.test.ts @@ -2619,6 +2619,42 @@ describe('upgrade operations - execution', () => { expect(fields).not.toHaveProperty('grouping'); }); + it("a mass-storage replacement leaves the copied file's extended tags alone", async () => { + const existingTrack = createDeviceTrack('Artist', 'Song', 'Album', { + filePath: 'Music/EXISTING.mp3', + }); + db.getTracks.mockReturnValue([existingTrack]); + const plan: SyncPlan = { + ...createEmptyPlan(), + operations: [ + { + type: 'upgrade-direct-copy', + source: createCollectionTrack('Artist', 'Song', 'Album', 'mp3', { + lossless: false, + composer: 'A, B', + composers: ['A', 'B'], + sortArtist: 'Artist, The', + }), + target: existingTrack, + reason: 'quality-change', + }, + ], + }; + + for await (const _p of new MusicPipeline(createDependencies(db, transcoder)).execute(plan, { + artwork: false, + })) { + // iterate + } + + const fields = db.updateTrack.mock.calls + .map((call) => call[1] as Record) + .find((f) => 'filetype' in f)!; + for (const field of ['composer', 'composers', 'sortArtist']) { + expect([field, fields[field]]).toEqual([field, undefined]); + } + }); + it('executes upgrade without preset (copy-based quality-change)', async () => { const existingTrack = createDeviceTrack('Artist', 'Song', 'Album', { filePath: 'Music/EXISTING.m4a', diff --git a/packages/podkit-core/src/sync/music/pipeline.ts b/packages/podkit-core/src/sync/music/pipeline.ts index d8522ee7..e3bef565 100644 --- a/packages/podkit-core/src/sync/music/pipeline.ts +++ b/packages/podkit-core/src/sync/music/pipeline.ts @@ -1317,6 +1317,9 @@ export class MusicPipeline implements SyncExecutor { const value = metadata[field]; if (value !== undefined) (updateFields as Record)[field] = value; } + if (updateFields.composer && operation.source?.composers) { + updateFields.composers = operation.source.composers; + } if (metadata.normalization !== undefined) { updateFields.normalization = metadata.normalization; } diff --git a/packages/podkit-core/src/sync/music/transfer.ts b/packages/podkit-core/src/sync/music/transfer.ts index 2c803cdc..4e1a0211 100644 --- a/packages/podkit-core/src/sync/music/transfer.ts +++ b/packages/podkit-core/src/sync/music/transfer.ts @@ -49,6 +49,7 @@ function toDeviceTrackInput(track: CollectionTrack): DeviceTrackInput { discNumber: track.discNumber, compilation: track.compilation, composer: track.composer, + composers: track.composers, grouping: track.grouping, totalTracks: track.totalTracks, totalDiscs: track.totalDiscs, @@ -107,6 +108,16 @@ export class MusicTransferOps { transferMode: ctx.transferMode, }; + // A portable file is re-tagged from this input, and the copied file may + // still carry a value the source reports as empty: clear it explicitly. + if (ctx.transferMode === 'portable') { + for (const field of reportedExtendedFields(source)) { + if (trackInput[field] === undefined) { + Object.assign(trackInput, { [field]: clearedExtendedValue(field) }); + } + } + } + // Write sync tag for transcode operations if (operation.type === 'add-transcode' && operation.preset) { const syncTag = this.buildSyncTagForPreset( @@ -331,7 +342,7 @@ export class MusicTransferOps { // Only devices that store the extended fields get them, and fields the // source reports but no longer has are cleared, so the replacement leaves // nothing stale. Elsewhere (mass-storage) the replacement file carries - // the source's own tags. + // the source's own tags; rewriting them would only merge multi-value ones. if (foundTrack.storesExtendedTags) { const reported = reportedExtendedFields(source); for (const field of EXTENDED_TAG_FIELDS) { @@ -339,6 +350,7 @@ export class MusicTransferOps { source[field] ?? (reported.includes(field) ? clearedExtendedValue(field) : undefined); if (value !== undefined) (updateFields as Record)[field] = value; } + if (source.composers && updateFields.composer) updateFields.composers = source.composers; } // Request ReplayGain tag writes for transcoded/optimized-copy upgrades. diff --git a/test-packages/test-fixtures/src/lib.ts b/test-packages/test-fixtures/src/lib.ts index 1038ee96..74154c3f 100644 --- a/test-packages/test-fixtures/src/lib.ts +++ b/test-packages/test-fixtures/src/lib.ts @@ -25,6 +25,8 @@ export { generateMiniFlac, generateMiniM4a, generateMiniMp3, + generateMiniMp3WithId3Frames, + generateMiniMp3WithId3v23, generateMiniOggOpus, generateMiniOggVorbis, type MiniTrackOptions, diff --git a/test-packages/test-fixtures/src/mini-tracks.ts b/test-packages/test-fixtures/src/mini-tracks.ts index 1ad37ffa..3b508f71 100644 --- a/test-packages/test-fixtures/src/mini-tracks.ts +++ b/test-packages/test-fixtures/src/mini-tracks.ts @@ -15,6 +15,7 @@ */ import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { requireEncoder } from './encoder-guard.js'; @@ -162,3 +163,72 @@ export function generateMiniOggOpus(dir: string, opts: MiniTrackOptions): string ]); return outPath; } + +/** + * Generate an MP3 carrying a hand-built ID3v2 tag (version 3 or 4) with + * exactly the given text frames (UTF-16), in order. Taggers cannot produce + * some real-world tags: iTunes writes the v2.4-only TSOT/TSOP/TSOA sort + * frames and its GRP1 grouping frame into v2.3, which taglib and ffmpeg drop. + */ +export function generateMiniMp3WithId3Frames( + dir: string, + filename: string, + frames: Array<[id: string, text: string]>, + version: 3 | 4 = 3 +): string { + requireEncoder('libmp3lame'); + const outPath = join(dir, filename); + runFfmpeg([ + ...sineInputArgs({ filename }, 44100), + '-c:a', + 'libmp3lame', + '-write_id3v2', + '0', + '-write_xing', + '0', + outPath, + ]); + // v2.4 frame sizes are synchsafe; v2.3 ones plain big-endian. + const size = (n: number) => + version === 4 + ? Buffer.from([(n >> 21) & 0x7f, (n >> 14) & 0x7f, (n >> 7) & 0x7f, n & 0x7f]) + : Buffer.from([(n >>> 24) & 0xff, (n >> 16) & 0xff, (n >> 8) & 0xff, n & 0xff]); + const body = Buffer.concat( + frames.map(([id, text]) => { + const content = Buffer.concat([ + Buffer.from([0x01, 0xff, 0xfe]), + Buffer.from(text, 'utf16le'), + ]); + return Buffer.concat([ + Buffer.from(id, 'latin1'), + size(content.length), + Buffer.alloc(2), + content, + ]); + }) + ); + const n = body.length; + const header = Buffer.from([ + 0x49, + 0x44, + 0x33, + version, + 0x00, + 0x00, + (n >> 21) & 0x7f, + (n >> 14) & 0x7f, + (n >> 7) & 0x7f, + n & 0x7f, + ]); + writeFileSync(outPath, Buffer.concat([header, body, readFileSync(outPath)])); + return outPath; +} + +/** {@link generateMiniMp3WithId3Frames} with an ID3v2.3 tag. */ +export function generateMiniMp3WithId3v23( + dir: string, + filename: string, + frames: Array<[id: string, text: string]> +): string { + return generateMiniMp3WithId3Frames(dir, filename, frames, 3); +} From 6ca57c93f876f5e48c7b877a7dfbf2ddc97a786c Mon Sep 17 00:00:00 2001 From: Thomas Gumpinger Date: Tue, 27 Oct 2026 18:30:57 +0100 Subject: [PATCH 21/21] feat(collections): map OpenSubsonic composer and grouping, and sort names on opt-in The Subsonic adapter reported none of the extended tag fields. It now maps OpenSubsonic displayComposer and Child.groupings (joined with ", ", typed locally because subsonic-api predates it). Following OpenSubsonic's unsupported-versus-empty rule, an absent field stays in unreportedTagFields and is neither compared nor cleared, a present empty one clears; track/disc totals are not in the API and stay unreported. When the song lists several composer contributors, their names become composers, so portable file tags keep them as separate values. Sort names are used only with the new subsonic-only collection key serverSortNames = true. A server may report a derived ordering key as sortName instead of the file's sort tag, and no rule tells such a key from a real sort tag, so by default sort fields from a Subsonic source are neither written nor cleared. With the opt-in each sortName is used as given; artist, album-artist and composer sorts come from the single credited artist or composer, several credits leave the field unreported, and playlist entries without their album leave the album sort unreported. The key is optional with a default, so no config migration; it is rejected on directory collections and for non-boolean values. --- .changeset/extended-tag-fidelity.md | 2 +- .changeset/subsonic-extended-tags.md | 10 ++ docs/agents/config-migrations.md | 1 + .../src/content/docs/reference/config-file.md | 1 + .../docs/user-guide/collections/subsonic.md | 23 +++ .../content/docs/user-guide/syncing/music.md | 2 +- packages/podkit-cli/src/config/loader.test.ts | 42 +++++ packages/podkit-cli/src/config/loader.ts | 15 ++ packages/podkit-cli/src/config/types.ts | 7 + .../podkit-cli/src/utils/source-adapter.ts | 1 + .../podkit-core/src/adapters/interface.ts | 2 + .../podkit-core/src/adapters/subsonic.test.ts | 162 ++++++++++++++++++ packages/podkit-core/src/adapters/subsonic.ts | 118 ++++++++++++- 13 files changed, 379 insertions(+), 7 deletions(-) create mode 100644 .changeset/subsonic-extended-tags.md diff --git a/.changeset/extended-tag-fidelity.md b/.changeset/extended-tag-fidelity.md index 197fd2d1..860e63cf 100644 --- a/.changeset/extended-tag-fidelity.md +++ b/.changeset/extended-tag-fidelity.md @@ -6,7 +6,7 @@ Sync composer, grouping, track/disc totals and sort fields to iPods -The directory adapter now reads composer, grouping, total tracks, total discs and the sort fields (sort title, artist, album, album artist, composer) from file tags. They are written to the iPod database when a track is added, and a change to any of them at the source is detected as a metadata correction and updated in place without copying the file again. A tag removed at the source is cleared on the device. A field the source cannot report is left untouched rather than cleared; Subsonic sources do not report these fields yet. +The directory adapter now reads composer, grouping, total tracks, total discs and the sort fields (sort title, artist, album, album artist, composer) from file tags. They are written to the iPod database when a track is added, and a change to any of them at the source is detected as a metadata correction and updated in place without copying the file again. A tag removed at the source is cleared on the device. A field the source cannot report is left untouched rather than cleared. Mass-storage change detection is unchanged: podkit does not compare these fields there. A multi-disc track added to a mass-storage device now gets the `Album (disc N)` folder its path template asks for, since the total disc count reaches it. The comment field still carries podkit's sync tag; file comments are not copied. diff --git a/.changeset/subsonic-extended-tags.md b/.changeset/subsonic-extended-tags.md new file mode 100644 index 00000000..1b63d5e4 --- /dev/null +++ b/.changeset/subsonic-extended-tags.md @@ -0,0 +1,10 @@ +--- +"podkit": minor +"@podkit/core": minor +--- + +Map composer and grouping from Subsonic, and sort names with `serverSortNames` + +The Subsonic adapter now maps composer and grouping where the server provides the OpenSubsonic fields, so they reach iPods like the same tags from a folder collection; composers credited separately stay separate values in portable file tags. A field the server does not report (track/disc totals, or the OpenSubsonic fields on a plain Subsonic server) is left untouched on the iPod; a field it reports as empty clears the iPod value. + +Sort names are used only with the new per-collection `serverSortNames = true`. A server may report a derived ordering key as the sort name, so by default sort fields from a Subsonic source are neither written nor cleared. diff --git a/docs/agents/config-migrations.md b/docs/agents/config-migrations.md index 73d85288..857efa56 100644 --- a/docs/agents/config-migrations.md +++ b/docs/agents/config-migrations.md @@ -24,6 +24,7 @@ continue to work; no migration or version bump was needed. | Field | Scope | Type | Default | Purpose | |-------|-------|------|---------|---------| | `playlist` | `[music.]` block (subsonic only) | `string` | absent (= full-library) | Constrains a subsonic collection to a single named server playlist. Parse-time error if set on a directory collection. | +| `serverSortNames` | `[music.]` block (subsonic only) | `boolean` | `false` | Use the server's `sortName` values as sort tags. Parse-time error on a directory collection or a non-boolean value. | | `allowEmptyPlaylist` | global `[defaults]` / top-level | `boolean` | `false` | When `true`, a headless sync against a playlist-scoped collection that resolves to zero tracks proceeds instead of aborting non-zero. Env override: `PODKIT_ALLOW_EMPTY_PLAYLIST`. | **IMPORTANT — three-place rule.** When adding any new config field, three diff --git a/packages/docs-site/src/content/docs/reference/config-file.md b/packages/docs-site/src/content/docs/reference/config-file.md index ec4d2f5d..d2215db9 100644 --- a/packages/docs-site/src/content/docs/reference/config-file.md +++ b/packages/docs-site/src/content/docs/reference/config-file.md @@ -299,6 +299,7 @@ path = "/cache/path" | `password` | string | no | - | Subsonic password (can also use env var) | | `path` | string | yes | - | Local cache path for downloaded files | | `playlist` | string | no | - | Sync only this named server playlist instead of the whole library. The name must match exactly one playlist on the server — the sync aborts before transferring anything if the name is not found or is ambiguous. Only valid on `subsonic` collections; setting it on a directory collection is a config error. See [Playlist-Scoped Collections](/user-guide/collections/subsonic#playlist-scoped-collections). | +| `serverSortNames` | boolean | no | `false` | Use the server's sort names as the iPod's sort fields. Only for servers that report your real sort tags; by default podkit neither writes nor clears sort fields from a Subsonic source. Only valid on `subsonic` collections. See [Composer, Grouping and Sort Names](/user-guide/collections/subsonic#composer-grouping-and-sort-names). | The password can be provided via the config file, or through environment variables (see [Environment Variables](/reference/environment-variables)). diff --git a/packages/docs-site/src/content/docs/user-guide/collections/subsonic.md b/packages/docs-site/src/content/docs/user-guide/collections/subsonic.md index 5539be7d..df01a077 100644 --- a/packages/docs-site/src/content/docs/user-guide/collections/subsonic.md +++ b/packages/docs-site/src/content/docs/user-guide/collections/subsonic.md @@ -199,6 +199,29 @@ Classic Subsonic and Airsonic servers do not expose ReplayGain data. Tracks sync See [Sound Check](/user-guide/syncing/sound-check) for more details. +## Composer, Grouping and Sort Names + +OpenSubsonic servers report composer and grouping, which podkit writes to iPods (see [Music Syncing](/user-guide/syncing/music#metadata)). Composers the server credits separately stay separate values in portable file tags. A field the server does not report is left as it is on the iPod; a field it reports as empty clears the iPod value. + +Sort names are different: a server can report its own ordering key as the sort name instead of your sort tags. Such a key (for example `cafe` for "The Café") cannot be told apart from a real sort tag, so by default podkit neither writes nor clears the iPod's sort fields from a Subsonic source; sort fields synced from a folder collection stay as they are. + +If your server reports your real sort tags (check its settings; some servers make this an option), opt in per collection: + +```toml +[music.navidrome] +type = "subsonic" +url = "https://your-server.example.com" +username = "your-username" +serverSortNames = true +``` + +With `serverSortNames = true` podkit uses each sort name exactly as the server reports it, even when it equals the plain name: + +- A **blank** sort name clears the iPod's sort field. +- An **absent** sort name (the server omits the field) leaves the iPod's sort field as it is. +- The artist, album artist and composer sorts come from the credited artist or composer. An **empty** list of credits clears the iPod's sort field; **several** credits leave it as it is, since there is no single sort name to use. +- Tracks from a playlist come without their album, so the album sort is left as it is for them. + ## Artwork Album artwork embedded in audio files is automatically transferred to your device during sync. diff --git a/packages/docs-site/src/content/docs/user-guide/syncing/music.md b/packages/docs-site/src/content/docs/user-guide/syncing/music.md index 4ee2edfa..59ffd034 100644 --- a/packages/docs-site/src/content/docs/user-guide/syncing/music.md +++ b/packages/docs-site/src/content/docs/user-guide/syncing/music.md @@ -37,7 +37,7 @@ Composer, grouping, the totals and the sort fields are written to iPods, which k Limitations: - **The comment field is podkit's.** podkit stores its sync tag in the iPod comment field and does not copy your files' comments onto the iPod. -- **Subsonic sources** do not provide these fields; podkit leaves them as they are on the iPod. +- **Subsonic sources** provide composer and grouping only when the server implements the OpenSubsonic extensions; track/disc totals are not part of the API. A field the server does not report is left as it is on the iPod. Sort names from a Subsonic server are used only with the collection's `serverSortNames` opt-in; see [Composer, Grouping and Sort Names](/user-guide/collections/subsonic#composer-grouping-and-sort-names). ### Track Matching diff --git a/packages/podkit-cli/src/config/loader.test.ts b/packages/podkit-cli/src/config/loader.test.ts index 18252b67..f4341df1 100644 --- a/packages/podkit-cli/src/config/loader.test.ts +++ b/packages/podkit-cli/src/config/loader.test.ts @@ -795,6 +795,48 @@ playlist = "Workout" }); }); + it('parses serverSortNames on a subsonic collection and keeps it through a merge', () => { + const configPath = path.join(tempDir, 'config.toml'); + fs.writeFileSync( + configPath, + v(` +[music.nav] +type = "subsonic" +url = "https://music.example.com" +username = "james" +serverSortNames = true +`) + ); + + const result = loadConfigFile(configPath); + expect(result?.music?.nav?.serverSortNames).toBe(true); + const merged = mergeConfigs(result!, { quality: 'low' }); + expect(merged.music?.nav?.serverSortNames).toBe(true); + }); + + it('rejects serverSortNames on a directory collection and non-boolean values', () => { + const configPath = path.join(tempDir, 'config.toml'); + fs.writeFileSync( + configPath, + v(`\n[music.local]\npath = "/music"\nserverSortNames = true\n`) + ); + expect(() => loadConfigFile(configPath)).toThrow( + /"serverSortNames" is only valid for subsonic/ + ); + + fs.writeFileSync( + configPath, + v(` +[music.nav] +type = "subsonic" +url = "https://music.example.com" +username = "james" +serverSortNames = "yes" +`) + ); + expect(() => loadConfigFile(configPath)).toThrow(/must be true or false/); + }); + it('rejects playlist on a directory collection', () => { const configPath = path.join(tempDir, 'config.toml'); fs.writeFileSync( diff --git a/packages/podkit-cli/src/config/loader.ts b/packages/podkit-cli/src/config/loader.ts index 714c729c..1fac229c 100644 --- a/packages/podkit-cli/src/config/loader.ts +++ b/packages/podkit-cli/src/config/loader.ts @@ -805,6 +805,18 @@ function parseMusicCollections( ); } + if (rawCollection.serverSortNames !== undefined) { + if (collectionType !== 'subsonic') { + throw new Error( + `"serverSortNames" is only valid for subsonic collections, but [music.${name}] is a directory collection. ` + + `Remove "serverSortNames" or set type = "subsonic".` + ); + } + if (typeof rawCollection.serverSortNames !== 'boolean') { + throw new Error(`Invalid "serverSortNames" in [music.${name}]: must be true or false.`); + } + } + if (collectionType === 'directory') { if (typeof rawCollection.path !== 'string') { throw new Error( @@ -842,6 +854,9 @@ function parseMusicCollections( username: rawCollection.username, password: rawCollection.password, // Optional - can also use env var playlist: rawCollection.playlist, // Optional - subsonic-only scope + ...(rawCollection.serverSortNames !== undefined && { + serverSortNames: rawCollection.serverSortNames as boolean, + }), }; } hasAnyCollection = true; diff --git a/packages/podkit-cli/src/config/types.ts b/packages/podkit-cli/src/config/types.ts index 4c9339a3..c09eb830 100644 --- a/packages/podkit-cli/src/config/types.ts +++ b/packages/podkit-cli/src/config/types.ts @@ -145,6 +145,12 @@ export interface MusicCollectionConfig { * library. Rejected at parse time on a directory collection. */ playlist?: string; + /** + * Use the server's sort names as sort tags (optional, subsonic-only, + * default false). Only for servers that report real sort tags; otherwise + * sort fields are left alone. + */ + serverSortNames?: boolean; } /** @@ -580,6 +586,7 @@ export interface ConfigFileMusicCollection { username?: string; password?: string; playlist?: string; + serverSortNames?: unknown; } /** diff --git a/packages/podkit-cli/src/utils/source-adapter.ts b/packages/podkit-cli/src/utils/source-adapter.ts index 9d939d68..23c143fe 100644 --- a/packages/podkit-cli/src/utils/source-adapter.ts +++ b/packages/podkit-cli/src/utils/source-adapter.ts @@ -145,6 +145,7 @@ function createSubsonicAdapterFromConfig( // Playlist-scope: when set, the adapter resolves+validates this named // server playlist at connect() and syncs only its tracks. playlist: config.playlist, + serverSortNames: config.serverSortNames, }); } diff --git a/packages/podkit-core/src/adapters/interface.ts b/packages/podkit-core/src/adapters/interface.ts index f9f3bb76..fbcd6862 100644 --- a/packages/podkit-core/src/adapters/interface.ts +++ b/packages/podkit-core/src/adapters/interface.ts @@ -270,6 +270,8 @@ export interface SubsonicAdapterConfig { password: string; /** When set, sync only this named server playlist instead of the whole library */ playlist?: string; + /** Use the server's sort names as sort tags (see the Subsonic adapter) */ + serverSortNames?: boolean; } /** diff --git a/packages/podkit-core/src/adapters/subsonic.test.ts b/packages/podkit-core/src/adapters/subsonic.test.ts index cfa6cd03..e40d5cc2 100644 --- a/packages/podkit-core/src/adapters/subsonic.test.ts +++ b/packages/podkit-core/src/adapters/subsonic.test.ts @@ -9,6 +9,8 @@ import { SubsonicAdapter, SubsonicConnectionError } from './subsonic.js'; import type { SubsonicAdapterConfig } from './subsonic.js'; import { PlaylistNotFoundError, AmbiguousPlaylistError } from './subsonic/playlist.js'; import type { CollectionTrack } from './interface.js'; +import { EXTENDED_TAG_FIELDS } from './interface.js'; +import { buildTagFieldsFromInput } from '../device/mass-storage-tag-writer.js'; import type { Child, AlbumWithSongsID3 } from 'subsonic-api'; import { replayGainToSoundcheck } from '../metadata/normalization.js'; import { hashArtwork } from '../artwork/hash.js'; @@ -307,6 +309,166 @@ describe('SubsonicAdapter Sound Check (ReplayGain)', () => { }); }); +describe('SubsonicAdapter OpenSubsonic composer, grouping and sort names', () => { + async function mapSong( + song: Partial & { groupings?: string[] }, + album?: Partial | null, + config: Partial = {} + ): Promise { + const fullSong: Child = { + id: 's', + isDir: false, + title: 'The Wall', + artist: 'The Band', + ...song, + }; + const fullAlbum: AlbumWithSongsID3 | undefined = + album === null + ? undefined + : { + id: 'a', + name: 'The Album', + artist: 'The Ensemble', + songCount: 1, + duration: 1, + created: new Date('2024-01-01T00:00:00Z'), + ...album, + }; + return await (createTestAdapter(config) as any)['mapSongToTrack'](fullSong, fullAlbum); + } + const artist = (name: string, sortName?: string) => ({ id: name, name, albumCount: 1, sortName }); + const SORTS = [ + 'sortTitle', + 'sortArtist', + 'sortAlbum', + 'sortAlbumArtist', + 'sortComposer', + ] as const; + const optIn = { serverSortNames: true }; + + it('maps displayComposer and groupings', async () => { + const track = await mapSong({ + displayComposer: 'Johann Sebastian Bach', + groupings: ['Cantatas', ' ', 'Sacred'], + }); + expect([track.composer, track.grouping]).toEqual(['Johann Sebastian Bach', 'Cantatas, Sacred']); + expect(track.unreportedTagFields).not.toContain('composer'); + expect(track.unreportedTagFields).not.toContain('grouping'); + }); + + it('keeps separate composer credits apart in file tags', async () => { + const track = await mapSong({ + displayComposer: 'John Lennon, Paul McCartney', + contributors: [ + { role: 'composer', artist: artist('John Lennon') }, + { role: 'composer', artist: artist('Paul McCartney') }, + { role: 'producer', artist: artist('George Martin') }, + ], + }); + expect(track.composer).toBe('John Lennon, Paul McCartney'); + expect(buildTagFieldsFromInput(track)).toMatchObject({ + composer: 'John Lennon, Paul McCartney', + composers: ['John Lennon', 'Paul McCartney'], + }); + }); + + it('reports a supported empty composer or grouping, so the device field is cleared', async () => { + const track = await mapSong({ displayComposer: '', groupings: [] }); + expect([track.composer, track.grouping]).toEqual([undefined, undefined]); + expect(track.unreportedTagFields).not.toContain('composer'); + expect(track.unreportedTagFields).not.toContain('grouping'); + }); + + it('leaves every extended field unreported on a plain Subsonic server', async () => { + const track = await mapSong({}); + expect(track.unreportedTagFields).toEqual([...EXTENDED_TAG_FIELDS]); + }); + + // Derived ordering keys, as a server may report them instead of sort tags. + const derivedKeys = { + title: 'Aria', + sortName: 'aria', + artists: [artist('Johann Sebastian Bach', 'bach')], + albumArtists: [artist('Røyksopp', 'royksopp')], + contributors: [{ role: 'composer', artist: artist('Los Lobos', 'lobos') }], + }; + + it('leaves server sort names unreported by default, whatever they look like', async () => { + const track = await mapSong(derivedKeys, { name: 'The Café', sortName: 'cafe' }); + for (const field of SORTS) expect([field, track[field]]).toEqual([field, undefined]); + expect(track.unreportedTagFields).toEqual(expect.arrayContaining([...SORTS])); + }); + + it('by default neither writes nor clears folder-synced sorts, on two syncs', async () => { + const track = await mapSong(derivedKeys, { name: 'The Café', sortName: 'cafe' }); + // The device as an earlier folder sync left it: same core fields, plus sorts. + const folderSynced = { + ...track, + storesExtendedTags: true, + sortTitle: 'Aria', + sortArtist: 'Bach, Johann Sebastian', + sortAlbum: 'Café, The', + sortAlbumArtist: 'Röyksopp', + sortComposer: 'Lobos, Los', + } as unknown as DeviceTrack; + expect(detectUpgrades(track, folderSynced)).toEqual([]); + const again = await mapSong(derivedKeys, { name: 'The Café', sortName: 'cafe' }); + expect(detectUpgrades(again, folderSynced)).toEqual([]); + }); + + it('with serverSortNames, uses sort names as given, the plain name included', async () => { + const track = await mapSong( + { + sortName: 'Wall, The', + artists: [artist('The Band', 'Band, The')], + albumArtists: [artist('Bach', 'Bach')], + contributors: [{ role: 'composer', artist: artist('Hildegard', 'Bingen, Hildegard von') }], + }, + { sortName: 'Album, The' }, + optIn + ); + expect(SORTS.map((f) => track[f])).toEqual([ + 'Wall, The', + 'Band, The', + 'Album, The', + 'Bach', + 'Bingen, Hildegard von', + ]); + expect(track.unreportedTagFields).toEqual([ + 'composer', + 'grouping', + 'totalTracks', + 'totalDiscs', + ]); + }); + + it('with serverSortNames, an absent sort name is unreported and a blank one clears', async () => { + const track = await mapSong({ sortName: '', artists: [] }, null, optIn); + expect(track.unreportedTagFields).not.toContain('sortTitle'); + expect(track.unreportedTagFields).not.toContain('sortArtist'); + // No album enrichment (playlist entries): the album sort is unknown. + expect(track.unreportedTagFields).toContain('sortAlbum'); + expect([track.sortTitle, track.sortArtist]).toEqual([undefined, undefined]); + }); + + it('with serverSortNames, several credited artists or composers leave the sort unreported', async () => { + const track = await mapSong( + { + artists: [artist('A', 'A, x'), artist('B', 'B, x')], + contributors: [ + { role: 'composer', artist: artist('C', 'C, x') }, + { role: 'composer', artist: artist('D', 'D, x') }, + ], + }, + {}, + optIn + ); + expect(track.unreportedTagFields).toEqual( + expect.arrayContaining(['sortArtist', 'sortComposer']) + ); + }); +}); + // ============================================================================= // Artwork Presence Detection Tests // ============================================================================= diff --git a/packages/podkit-core/src/adapters/subsonic.ts b/packages/podkit-core/src/adapters/subsonic.ts index fe906f09..a21d289f 100644 --- a/packages/podkit-core/src/adapters/subsonic.ts +++ b/packages/podkit-core/src/adapters/subsonic.ts @@ -6,8 +6,13 @@ */ import SubsonicAPI from 'subsonic-api'; -import type { Child, AlbumWithSongsID3 } from 'subsonic-api'; -import type { CollectionAdapter, CollectionTrack, FileAccess } from './interface.js'; +import type { Child, AlbumWithSongsID3, ArtistID3 } from 'subsonic-api'; +import type { + CollectionAdapter, + CollectionTrack, + ExtendedTagField, + FileAccess, +} from './interface.js'; import { EXTENDED_TAG_FIELDS } from './interface.js'; import type { TrackFilter, AudioFileType } from '../types.js'; import type { Warning } from '../sync/engine/types.js'; @@ -35,6 +40,12 @@ export interface SubsonicAdapterConfig { * a missing or ambiguous name aborts before any transfer. */ playlist?: string; + /** + * Use the server's `sortName` values as sort tags. Off by default: a server + * may report a derived ordering key there, so without the opt-in sort + * fields are neither written nor cleared. + */ + serverSortNames?: boolean; } /** @@ -252,6 +263,101 @@ function getCodec(suffix: string | undefined, contentType: string | undefined): return undefined; } +/** A field's value and whether the server reported it at all. */ +interface Reported { + value: T | undefined; + reported: boolean; +} + +/** + * A server sort name as a sort override, used only when the collection opts + * in (`serverSortNames`): servers may report a derived ordering key there + * instead of the file's sort tag, which cannot be told apart from a real + * tag. `undefined` is unreported; a blank value is a reported empty; + * anything else, even the plain name, is the tag. + */ +function sortOverride(sortName: string | undefined): Reported { + if (sortName === undefined) return { value: undefined, reported: false }; + return { value: sortName.trim() ? sortName : undefined, reported: true }; +} + +/** + * Sort override of the single credited artist. No credited artists is a + * reported empty value; several artists have no one sort key, so the field + * is left unreported rather than cleared. + */ +function soleArtistSortOverride(artists: ArtistID3[] | undefined): Reported { + if (artists === undefined) return { value: undefined, reported: false }; + if (artists.length === 0) return { value: undefined, reported: true }; + if (artists.length > 1) return { value: undefined, reported: false }; + return sortOverride(artists[0]!.sortName); +} + +/** OpenSubsonic `Child.groupings`, newer than the subsonic-api types. */ +type OpenSubsonicChild = Child & { groupings?: string[] }; + +/** + * Sort override of the single credited composer. Same rules as for artists: + * none is a reported empty value, several leave the field unreported. + */ +function soleComposerSortOverride(song: Child): Reported { + if (song.contributors === undefined) return { value: undefined, reported: false }; + const composers = song.contributors.filter((c) => c.role === 'composer' && c.artist); + return soleArtistSortOverride(composers.map((c) => c.artist!)); +} + +/** + * Extended tag fields from the OpenSubsonic additions to `Child`. A plain + * Subsonic server omits them, and the API has no track/disc totals, so + * those fields are listed in `unreportedTagFields` and never cleared on + * the device. + */ +function openSubsonicTags( + song: OpenSubsonicChild, + album: AlbumWithSongsID3 | undefined, + serverSortNames: boolean +): Pick { + const fields: Partial>> = { + composer: { + value: song.displayComposer?.trim() || undefined, + reported: song.displayComposer !== undefined, + }, + grouping: { + value: + song.groupings + ?.map((g) => g.trim()) + .filter(Boolean) + .join(', ') || undefined, + reported: song.groupings !== undefined, + }, + ...(serverSortNames && { + sortTitle: sortOverride(song.sortName), + sortArtist: soleArtistSortOverride(song.artists), + // Without album enrichment (playlist entries) the album fields are unknown. + sortAlbum: album ? sortOverride(album.sortName) : { value: undefined, reported: false }, + sortAlbumArtist: soleArtistSortOverride(song.albumArtists ?? album?.artists), + sortComposer: soleComposerSortOverride(song), + }), + }; + const out: Pick = {}; + const unreported: ExtendedTagField[] = []; + for (const field of EXTENDED_TAG_FIELDS) { + const f = fields[field]; + if (!f?.reported) unreported.push(field); + else if (f.value !== undefined) (out as Record)[field] = f.value; + } + out.unreportedTagFields = unreported; + // Separate composer credits, so file tags keep them apart. + const composers = new Set( + song.contributors + ?.filter((c) => c.role === 'composer') + .map((c) => c.artist?.name?.trim()) + .filter((name): name is string => Boolean(name)) + ); + if (out.composer && composers.size > 1) out.composers = [...composers]; + return out; +} + /** * Adapter for reading tracks from Subsonic-compatible servers * @@ -295,6 +401,9 @@ export class SubsonicAdapter implements CollectionAdapter