From 3de7efa89f7bc4da5562bf55b5ec865bf84c40de Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Sun, 11 Oct 2026 00:46:25 +0000 Subject: [PATCH 1/3] Update duroxide to 0.1.31 (race fixes) duroxide 0.1.31 fixes dropped queue reads and signal waits, the continue-as-new carry of unread queue messages, inputs that arrive between two executions, duplicate starts, and the KV key order used by prune. The SDK compiles unchanged: it does not use TurnResult or any other changed public item. - Require duroxide 0.1.31. - Temporarily patch duroxide and duroxide-pg to their unreleased fix branches (microsoft/duroxide#67, microsoft/duroxide-pg#35). Remove the patch and pin the released crates before release. - Document the new contract: a read that loses a race takes no message, continueAsNew carries exactly the unread messages, getKvAllKeys returns keys in key order. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- .github/copilot-instructions.md | 2 +- CHANGELOG.md | 41 +++++++++++++++++++++++++++++++++ Cargo.toml | 11 ++++++++- README.md | 2 +- docs/user-guide.md | 3 +++ 5 files changed, 56 insertions(+), 3 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index f4cc0c5..926d75f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -171,7 +171,7 @@ Activities have no restrictions — they run once, result is cached. Release manifests must resolve `duroxide` and `duroxide-pg` from crates.io: ```toml -duroxide = { version = "0.1.30", features = ["sqlite"] } +duroxide = { version = "0.1.31", features = ["sqlite"] } duroxide-pg = "0.1.35" ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index aeff2be..93d47f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Bumped `duroxide` dependency** — `0.1.30` → `0.1.31` (race fixes, below). Until + `duroxide` and `duroxide-pg` publish these fixes, `Cargo.toml` patches both crates to their + fix branches; that patch is removed before release. +- **A signal sent to a dropped `waitForEvent` is discarded.** When a `waitForEvent` loses a + `ctx.race()`, a signal already given to it is no longer handed to a later wait for the same + event. The later wait receives the next signal. Before, it could receive the stale value, fail + replay, or wait forever. +- **A duplicate start does not change a running instance.** `client.startOrchestration()` for + an instance that already has history is ignored; it can no longer replace the instance's + orchestration name, version or input. A sub-orchestration whose instance id collides with a + running instance now fails its parent, as it already did for a finished one. + +### Fixed + +- **A queue read that loses a race takes no message.** A drain loop that races + `ctx.dequeueEvent()` against a short timer could fail with + `nondeterministic: history schedule but no emitted action`, skip a message, or read one twice + when the timer won and a message arrived in the same batch. A dropped read is now cancelled + where it is dropped, so replay picks the same winners as the first run. +- **`continueAsNew` carries exactly the unread messages.** A dropped or still-pending read + counted as a consumer, so a message could be lost at continue-as-new. The runtime now carries + every message that no read returned (at most 100, as before). +- **Inputs that arrive while an instance continues as new are kept.** Queue messages, signals + and cancellation requests that arrived between two executions could be deleted. They now go to + the next execution. +- **`ctx.getKvAllKeys()` and `ctx.pruneKvValuesUpdatedBefore()` use key order.** Replay could + fail with `nondeterministic: kv clear mismatch` after a prune. + +The queue, signal and continue-as-new changes apply to executions started by `duroxide` 0.1.31 +or later. An execution started by an older runtime keeps the old behaviour until it completes or +continues as new. See the `duroxide` migration guide for details and rolling upgrades. + +### Added + +- `__tests__/races.test.js` — drain loops that race `ctx.dequeueEvent()` against a timer: every + message arrives once and in order, also across `continueAsNew`. +- `__tests__/kv_store.test.js` — KV keys are listed and pruned in key order, and a replayed prune + matches. + ## [0.2.0] - 2026-10-04 ### Changed diff --git a/Cargo.toml b/Cargo.toml index 636f3b6..358e4f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -duroxide = { version = "0.1.30", features = ["sqlite"] } +duroxide = { version = "0.1.31", features = ["sqlite"] } duroxide-pg = "0.1.35" napi = { version = "2", features = ["async", "napi6", "serde-json"] } napi-derive = "2" @@ -22,6 +22,15 @@ tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter", "json"] [build-dependencies] napi-build = "2" +# Temporary: build against the unreleased duroxide 0.1.31 queue-race fixes +# (microsoft/duroxide, branch waldemort/queue-race-fixes) and the matching +# duroxide-pg fetch-lease fix (microsoft/duroxide-pg, branch +# waldemort/fetch-lease-and-race-validations). Remove this section and pin the +# released crates once both are published. +[patch.crates-io] +duroxide = { git = "https://github.com/microsoft/duroxide", branch = "waldemort/queue-race-fixes" } +duroxide-pg = { git = "https://github.com/microsoft/duroxide-pg", branch = "waldemort/fetch-lease-and-race-validations" } + [profile.release] lto = true diff --git a/README.md b/README.md index 8dec748..a23acaa 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ All scheduling methods return descriptors that must be **yielded**: | `ctx.clearValue(key)` | Remove a single KV entry (no yield) | | `ctx.clearAllValues()` | Remove all KV entries (no yield) | | `ctx.getKvAllValues()` | Snapshot all KV entries for the current instance | -| `ctx.getKvAllKeys()` | List all KV keys for the current instance | +| `ctx.getKvAllKeys()` | List all KV keys for the current instance, in key order | | `ctx.getKvLength()` | Count KV entries for the current instance | | `ctx.pruneKvValuesUpdatedBefore(cutoffMs)` | Remove persisted KV entries older than a cutoff | | `yield ctx.getValueFromInstance(instanceId, key)` | Read another instance's KV entry | diff --git a/docs/user-guide.md b/docs/user-guide.md index d1f1cbc..a04f51a 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -315,6 +315,9 @@ Key properties: - **FIFO ordering** — messages delivered in enqueue order - **Survives continue-as-new** — queue state persists across restarts - **Blocks until available** — orchestration suspends until a message arrives +- **Safe to race** — `yield ctx.race(ctx.dequeueEvent('tasks'), ctx.scheduleTimer(ms))` drains + with a timeout. A read that loses the race takes no message, and `continueAsNew` carries every + message that no read returned (at most 100; the runtime drops the rest with a warning) ### Retry with Session Affinity From f9dfbc88d563b32d87cb656a9d2eb1e2dcee62e0 Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Sun, 11 Oct 2026 00:46:25 +0000 Subject: [PATCH 2/3] test: race dequeueEvent against a timer; KV key order Add regression tests for the duroxide 0.1.31 fixes, ported from the Node reproduction of the queue/timer race: - A drain loop races ctx.dequeueEvent() against a 20 ms timer while 300 numbered messages arrive with 0-30 ms gaps. Every message must arrive once and in order, and the timer must win at least once. - The same race with a 10 ms timer, where every timer win continues as new. Every message must still arrive once and in order, so a dropped read must not count as a consumer of the carried messages. - Eight KV keys set out of order are listed in key order and pruned, and the next turn replays the writes and the prune. Before microsoft/duroxide#57 was fixed, both used hash order, which differs between replays, so the replay failed with "nondeterministic: kv clear mismatch". Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- __tests__/kv_store.test.js | 30 +++++++++++ __tests__/races.test.js | 103 +++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/__tests__/kv_store.test.js b/__tests__/kv_store.test.js index a6cfd9e..c9e2b8b 100644 --- a/__tests__/kv_store.test.js +++ b/__tests__/kv_store.test.js @@ -94,6 +94,36 @@ describe('kv store', () => { }); }); + it('kv keys are listed and pruned in key order, and the prune replays', async () => { + // duroxide 0.1.31 lists and prunes keys in key order (microsoft/duroxide#57). Before, both + // used hash order, which differs between replays, so the replay of a prune that cleared + // several keys failed with "nondeterministic: kv clear mismatch". + const keys = ['k7', 'k3', 'k5', 'k1', 'k8', 'k2', 'k6', 'k4']; + await withRuntime((runtime) => { + runtime.registerOrchestration('KvKeyOrder', function* (ctx) { + for (const key of keys) ctx.setValue(key, key); + const listed = ctx.getKvAllKeys(); + // A cutoff after every write prunes all eight keys. + const pruned = ctx.pruneKvValuesUpdatedBefore(Number.MAX_SAFE_INTEGER); + // The next turn replays the writes and the prune, and must clear the keys in the + // order this turn recorded. + yield ctx.scheduleTimer(5); + return { listed, pruned, after: ctx.getKvAllKeys() }; + }); + }, async (client) => { + const instanceId = uid('kv-key-order'); + await client.startOrchestration(instanceId, 'KvKeyOrder', null); + + const status = await client.waitForOrchestration(instanceId, 5000); + assert.strictEqual(status.status, 'Completed', `orchestration failed: ${status.error}`); + assert.deepStrictEqual(status.output, { + listed: ['k1', 'k2', 'k3', 'k4', 'k5', 'k6', 'k7', 'k8'], + pruned: 8, + after: [], + }); + }); + }); + it('kv bulk snapshot and prune', async () => { await withRuntime((runtime) => { runtime.registerOrchestration('KvBulk', function* (ctx, input) { diff --git a/__tests__/races.test.js b/__tests__/races.test.js index 78d1e82..3ae4d04 100644 --- a/__tests__/races.test.js +++ b/__tests__/races.test.js @@ -596,3 +596,106 @@ describe('type preservation', () => { assert.strictEqual(typeof r3.output.value, 'object'); }); }); + +// ─── dequeueEvent raced against a timer ────────────────────────── +// +// A drain loop races a queue read against a short timer. When the timer wins, the read is +// dropped. Since duroxide 0.1.31 a dropped read is cancelled at the drop: it never takes a +// message, replay picks the same winners as the live run, and continueAsNew carries exactly +// the messages that no read returned. Before, a timer win followed by a message in the same +// batch could fail the instance with "nondeterministic: history schedule but no emitted +// action", skip a message, or lose one at continueAsNew. + +function decodeMessage(value) { + return typeof value === 'string' ? JSON.parse(value) : value; +} + +// Numbered messages with deterministic 0-30 ms gaps, a 100 ms pause after every 50th +// message so the timer also wins, and a final { done: true }. +async function produceNumbered(client, instanceId, count) { + for (let i = 0; i < count; i++) { + await client.enqueueEvent(instanceId, 'q', { i }); + await new Promise((r) => setTimeout(r, i % 50 === 49 ? 100 : (i * 17 + 7) % 31)); + } + await client.enqueueEvent(instanceId, 'q', { done: true }); +} + +describe('dequeueEvent raced against a timer', () => { + it('a read that loses to the timer takes no message (FIFO, exactly once)', async () => { + const count = 300; + const instanceId = uid('race-drain-loop'); + const client = new Client(provider); + const runtime = new Runtime(provider, { + orchestrationConcurrency: 2, + workerConcurrency: 2, + dispatcherPollIntervalMs: 10, + }); + + runtime.registerOrchestration('RaceDrainLoop', function* (ctx) { + const seq = []; + let timers = 0; + while (true) { + const winner = yield ctx.race(ctx.dequeueEvent('q'), ctx.scheduleTimer(20)); + if (winner.index === 1) { + timers++; + continue; + } + const message = decodeMessage(winner.value); + if (message.done) return { seq, timers }; + seq.push(message.i); + } + }); + + await runtime.start(); + try { + await client.startOrchestration(instanceId, 'RaceDrainLoop', null); + await produceNumbered(client, instanceId, count); + const result = await client.waitForOrchestration(instanceId, 60000); + + assert.strictEqual(result.status, 'Completed', `orchestration failed: ${result.error}`); + assert.deepStrictEqual(result.output.seq, Array.from({ length: count }, (_, i) => i)); + assert.ok(result.output.timers > 0, 'the timer never won, so no read was dropped'); + } finally { + await runtime.shutdown(100); + } + }); + + it('continueAsNew after a lost read carries every unread message', async () => { + const count = 200; + const instanceId = uid('race-drain-can'); + const client = new Client(provider); + const runtime = new Runtime(provider, { + orchestrationConcurrency: 2, + workerConcurrency: 2, + dispatcherPollIntervalMs: 10, + }); + + runtime.registerOrchestration('RaceDrainCan', function* (ctx, state) { + let next = state.next; + while (true) { + const winner = yield ctx.race(ctx.dequeueEvent('q'), ctx.scheduleTimer(10)); + if (winner.index === 1) { + // The read lost and is dropped. Messages it did not take move to the next execution. + return yield ctx.continueAsNew({ next, cans: state.cans + 1 }); + } + const message = decodeMessage(winner.value); + if (message.done) return { next, cans: state.cans }; + if (message.i !== next) throw new Error(`expected message ${next}, got ${message.i}`); + next++; + } + }); + + await runtime.start(); + try { + await client.startOrchestration(instanceId, 'RaceDrainCan', { next: 0, cans: 0 }); + await produceNumbered(client, instanceId, count); + const result = await client.waitForOrchestration(instanceId, 60000); + + assert.strictEqual(result.status, 'Completed', `orchestration failed: ${result.error}`); + assert.strictEqual(result.output.next, count); + assert.ok(result.output.cans > 0, 'the timer never won, so the instance never continued as new'); + } finally { + await runtime.shutdown(100); + } + }); +}); From b4a8e33658f40d3e9a7e11e2eda98c9216246883 Mon Sep 17 00:00:00 2001 From: Affan Dar Date: Mon, 12 Oct 2026 06:32:14 -0700 Subject: [PATCH 3/3] Update duroxide to 0.1.32 and duroxide-pg to 0.1.36 Both are on crates.io, so remove the temporary [patch.crates-io] block that built against their fix branches. 0.1.31 was reserved and never published. Changelog: add the duroxide-pg lock fix and the shutdown change. Docs: Runtime.shutdown now returns as soon as the runtime stops; the timeout is a grace period, not a fixed delay. --- .github/copilot-instructions.md | 4 ++-- CHANGELOG.md | 16 ++++++++++++---- Cargo.toml | 13 ++----------- __tests__/kv_store.test.js | 2 +- __tests__/races.test.js | 2 +- docs/architecture.md | 6 +++--- 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 926d75f..4d2f622 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -171,8 +171,8 @@ Activities have no restrictions — they run once, result is cached. Release manifests must resolve `duroxide` and `duroxide-pg` from crates.io: ```toml -duroxide = { version = "0.1.31", features = ["sqlite"] } -duroxide-pg = "0.1.35" +duroxide = { version = "0.1.32", features = ["sqlite"] } +duroxide-pg = "0.1.36" ``` Temporary local `[patch.crates-io]` sections or inline `path =` overrides are diff --git a/CHANGELOG.md b/CHANGELOG.md index 93d47f2..63af22e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- **Bumped `duroxide` dependency** — `0.1.30` → `0.1.31` (race fixes, below). Until - `duroxide` and `duroxide-pg` publish these fixes, `Cargo.toml` patches both crates to their - fix branches; that patch is removed before release. +- **Bumped `duroxide` dependency** — `0.1.30` → `0.1.32` (race fixes and shutdown, below). + `0.1.31` was reserved and never published. +- **Bumped `duroxide-pg` dependency** — `0.1.35` → `0.1.36`. Under heavy load, an orchestration + fetched from PostgreSQL could come with a lock that had already expired. Every ack then failed + with `Invalid lock token`, and the message was retried for as long as the load lasted. The lock + now starts when the instance is claimed (migration `0025`). An SDK version built with + `duroxide-pg` 0.1.35 refuses a database that has this migration. +- **`runtime.shutdown(timeoutMs)` returns as soon as the runtime stops.** `timeoutMs` (default + 1000) is now the longest it waits for in-flight work before cancelling the rest, not a fixed + delay. Shutdown also stops every task the runtime started, so no poller keeps a database + connection open after it returns. - **A signal sent to a dropped `waitForEvent` is discarded.** When a `waitForEvent` loses a `ctx.race()`, a signal already given to it is no longer handed to a later wait for the same event. The later wait receives the next signal. Before, it could receive the stale value, fail @@ -37,7 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`ctx.getKvAllKeys()` and `ctx.pruneKvValuesUpdatedBefore()` use key order.** Replay could fail with `nondeterministic: kv clear mismatch` after a prune. -The queue, signal and continue-as-new changes apply to executions started by `duroxide` 0.1.31 +The queue, signal and continue-as-new changes apply to executions started by `duroxide` 0.1.32 or later. An execution started by an older runtime keeps the old behaviour until it completes or continues as new. See the `duroxide` migration guide for details and rolling upgrades. diff --git a/Cargo.toml b/Cargo.toml index 358e4f0..4d5329d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -duroxide = { version = "0.1.31", features = ["sqlite"] } -duroxide-pg = "0.1.35" +duroxide = { version = "0.1.32", features = ["sqlite"] } +duroxide-pg = "0.1.36" napi = { version = "2", features = ["async", "napi6", "serde-json"] } napi-derive = "2" async-trait = "0.1" @@ -22,15 +22,6 @@ tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter", "json"] [build-dependencies] napi-build = "2" -# Temporary: build against the unreleased duroxide 0.1.31 queue-race fixes -# (microsoft/duroxide, branch waldemort/queue-race-fixes) and the matching -# duroxide-pg fetch-lease fix (microsoft/duroxide-pg, branch -# waldemort/fetch-lease-and-race-validations). Remove this section and pin the -# released crates once both are published. -[patch.crates-io] -duroxide = { git = "https://github.com/microsoft/duroxide", branch = "waldemort/queue-race-fixes" } -duroxide-pg = { git = "https://github.com/microsoft/duroxide-pg", branch = "waldemort/fetch-lease-and-race-validations" } - [profile.release] lto = true diff --git a/__tests__/kv_store.test.js b/__tests__/kv_store.test.js index c9e2b8b..fa7817c 100644 --- a/__tests__/kv_store.test.js +++ b/__tests__/kv_store.test.js @@ -95,7 +95,7 @@ describe('kv store', () => { }); it('kv keys are listed and pruned in key order, and the prune replays', async () => { - // duroxide 0.1.31 lists and prunes keys in key order (microsoft/duroxide#57). Before, both + // duroxide 0.1.32 lists and prunes keys in key order (microsoft/duroxide#57). Before, both // used hash order, which differs between replays, so the replay of a prune that cleared // several keys failed with "nondeterministic: kv clear mismatch". const keys = ['k7', 'k3', 'k5', 'k1', 'k8', 'k2', 'k6', 'k4']; diff --git a/__tests__/races.test.js b/__tests__/races.test.js index 3ae4d04..a5d01ed 100644 --- a/__tests__/races.test.js +++ b/__tests__/races.test.js @@ -600,7 +600,7 @@ describe('type preservation', () => { // ─── dequeueEvent raced against a timer ────────────────────────── // // A drain loop races a queue read against a short timer. When the timer wins, the read is -// dropped. Since duroxide 0.1.31 a dropped read is cancelled at the drop: it never takes a +// dropped. Since duroxide 0.1.32 a dropped read is cancelled at the drop: it never takes a // message, replay picks the same winners as the live run, and continueAsNew carries exactly // the messages that no read returned. Before, a timer win followed by a message in the same // batch could fail the instance with "nondeterministic: history schedule but no emitted diff --git a/docs/architecture.md b/docs/architecture.md index 749da0e..dafa032 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -249,8 +249,8 @@ Release manifests resolve both Rust dependencies from crates.io. Keep not ship `[patch.crates-io]` sections or inline `path =` overrides: ```toml -duroxide = { version = "0.1.30", features = ["sqlite"] } -duroxide-pg = "0.1.35" +duroxide = { version = "0.1.32", features = ["sqlite"] } +duroxide-pg = "0.1.36" ``` Temporary local path overrides may be useful during cross-repository @@ -389,7 +389,7 @@ File-based SQLite can hit "database is locked" errors under concurrent orchestra ### Runtime Shutdown -`Runtime.shutdown(timeoutMs)` waits for the full timeout duration unconditionally (the Rust runtime doesn't short-circuit when dispatchers drain). Use a small timeout (e.g., 100ms) in tests. +`Runtime.shutdown(timeoutMs)` returns as soon as all runtime work has stopped. `timeoutMs` (default 1000) is the longest it waits for in-flight work before cancelling the rest; it is not a fixed delay. ### Platform-Specific Binary