diff --git a/changelog.d/contract-projection-charge.md b/changelog.d/contract-projection-charge.md new file mode 100644 index 000000000..24a38b585 --- /dev/null +++ b/changelog.d/contract-projection-charge.md @@ -0,0 +1,3 @@ +## Fixed + +- Count only the values a contract maps against a contract query's 1 MiB page budget. Each record's whole frontmatter was counted once per mapped field, so a page of 128 ordinary records (about 1.5 KB of frontmatter, 14 mapped fields) was refused with `too_large` / `contract_projection_budget`, and so was any single record over about 80 KB. Apps such as Writer could not load a Reader library of that shape. diff --git a/docs/sdk-publication.md b/docs/sdk-publication.md index 32b9d4307..a5a3a6e05 100644 --- a/docs/sdk-publication.md +++ b/docs/sdk-publication.md @@ -83,7 +83,11 @@ updated; CI must not auto-bless its output. When updating pins, match the workflow's Rust path remapping as well as its pinned tools and optimizer flags: map the compiler host's checkout root to `/source` and Cargo home to `/cargo`. Source paths can survive WASM stripping, so builds without these mappings do not -produce publication-compatible bytes. Verify the corrected pins with the actual +produce publication-compatible bytes. A toolchain with the `rust-src` +component embeds its own source path for the standard library where the +workflow's minimal toolchain keeps `/rustc/`: map +`~/.rustup/toolchains//lib/rustlib/src/rust` to the `/rustc/` +of `rustc -vV` as well, and check that the built WASM contains no `/home/` path. Verify the corrected pins with the actual `scripts/sdk-publication.mjs pack-all` cold-consumer and archive checks. After committing runtime changes and rebuilding `cargo xtask wasm-app` with those mappings, run `node scripts/sdk-publication.mjs pin-runtime` to regenerate diff --git a/runtime/crates/replica/src/replica/client/contract_query.rs b/runtime/crates/replica/src/replica/client/contract_query.rs index ef84d0bec..4b360a9a9 100644 --- a/runtime/crates/replica/src/replica/client/contract_query.rs +++ b/runtime/crates/replica/src/replica/client/contract_query.rs @@ -10,6 +10,7 @@ use crate::{Store, convert}; use mdbn_core::state::StateView; use mdbn_wire::client::{Include, Issue, QueryResult, Severity}; use mdbn_wire::common::Value; +use std::collections::BTreeMap; const PROJECTION_BYTES: usize = 1024 * 1024; @@ -180,25 +181,29 @@ impl Replica { unreachable!("frontmatter is a map") }; let effective = catalog.effective_values(&record.path, &record.types, &raw); - let bytes = mdbn_core::value::Value::Map(effective.clone()) - .to_json() - .len(); + // Each record field's size is measured once, however many contract + // fields map it. + let mut sizes: BTreeMap<&str, usize> = BTreeMap::new(); let mut projections = Vec::new(); for implementation in implementations .iter() .filter(|i| record.types.contains(&i.type_name)) { - // Charge a conservative clone bound before Core copies mapped values. - let charge = bytes - .checked_mul(implementation.fields.len().max(1)) - .and_then(|n| { - n.checked_add( - implementation - .fields - .iter() - .map(|(a, b)| a.len() + b.len()) - .sum::(), - ) + // Charge what Core is about to copy, before it copies: every + // mapping clones the one value it selects. Unmapped fields are + // never copied, so they cost nothing here. + let charge = implementation + .fields + .iter() + .try_fold(0usize, |total, (a, b)| { + let value = *sizes.entry(b.as_str()).or_insert_with(|| { + mdbn_core::types::select_one(&effective, b) + .map_or(0, |v| v.to_json().len()) + }); + total + .checked_add(value)? + .checked_add(a.len())? + .checked_add(b.len()) }); remaining = charge .and_then(|n| remaining.checked_sub(n)) diff --git a/runtime/crates/replica/src/tests/describe.rs b/runtime/crates/replica/src/tests/describe.rs index 8d50771ef..f65db6d4f 100644 --- a/runtime/crates/replica/src/tests/describe.rs +++ b/runtime/crates/replica/src/tests/describe.rs @@ -385,16 +385,17 @@ fn contract_query_refuses_unindexed_full_windows_and_checks_session_and_selectio ); } -#[test] -fn contract_query_projection_budget_is_charged_before_wide_mapping_clones() { - let properties = (0..128) - .map(|i| format!(" value{i}: {{type: boolean}}\n")) +/// A type mapping `mappings` contract fields, each from its `source` field of +/// JSON Schema type `kind`. +fn wide(mappings: usize, source: &str, kind: &str) -> Vec<(String, String)> { + let properties = (0..mappings) + .map(|i| format!(" value{i}: {{type: {kind}}}\n")) .collect::(); - let fields = (0..128) - .map(|i| format!("'/value{i}': '/completed'")) + let fields = (0..mappings) + .map(|i| format!("'/value{i}': '/{source}'")) .collect::>() .join(", "); - let (_, mut a) = configured(vec![ + vec![ ("mdbase.yaml".into(), "spec_version: '0.3.0'\n".into()), ( "_contracts/wide.md".into(), @@ -405,16 +406,22 @@ fn contract_query_projection_budget_is_charged_before_wide_mapping_clones() { ( "_types/wide.md".into(), format!( - "---\nkind: mdbase.type\nname: wide\nschema:\n dialect: json-schema-2020-12\n value:\n type: object\n properties:\n completed: {{type: boolean}}\nimplements:\n - contract: acme.task\n version: 1.0.0\n fields: {{{fields}}}\n---\n" + "---\nkind: mdbase.type\nname: wide\nschema:\n dialect: json-schema-2020-12\n value:\n type: object\n properties:\n completed: {{type: boolean}}\n large: {{type: string}}\nimplements:\n - contract: acme.task\n version: 1.0.0\n fields: {{{fields}}}\n---\n" ), ), - ]); + ] +} + +#[test] +fn contract_query_projection_budget_is_charged_before_wide_mapping_clones() { + // 128 mappings of one 12 KB value would copy 1.5 MB: refused before the copy. + let (_, mut a) = configured(wide(128, "large", "string")); assert!(a.r.catalog.is_valid(), "{:?}", a.r.catalog.issues()); a.create( 1, "rows/wide.md", &format!( - "---\ntype: wide\ncompleted: true\nunmapped: {}\n---\n", + "---\ntype: wide\ncompleted: true\nlarge: {}\n---\n", "x".repeat(12000) ), ); @@ -427,6 +434,37 @@ fn contract_query_projection_budget_is_charged_before_wide_mapping_clones() { ); } +#[test] +fn contract_query_projection_budget_charges_mapped_values_not_the_whole_record() { + // Each record carries 4 KB the contract never maps. Charging the whole + // record once per mapping (14 x 4 KB x 60 records = 3 MB) refused every + // page of an ordinary library; the projection itself is 14 booleans a record. + let (_, mut a) = configured(wide(14, "completed", "boolean")); + assert!(a.r.catalog.is_valid(), "{:?}", a.r.catalog.issues()); + for i in 0..60u8 { + a.create( + i + 1, + &format!("rows/{i:03}.md"), + &format!( + "---\ntype: wide\ncompleted: true\nunmapped: {}\n---\n", + "x".repeat(4000) + ), + ); + } + settle(&mut [&mut a]); + let result = contract_page(&mut a, "1.0.0", None).unwrap(); + assert_eq!(result.records.len(), 60); + assert!(result.records.iter().all(|r| { + r.implementations.len() == 1 + && r.implementations[0].fields.0.len() == 14 + && r.implementations[0] + .fields + .0 + .iter() + .all(|(_, v)| *v == Value::Bool(true)) + })); +} + #[test] fn contract_query_empty_implementation_set_is_complete_without_raw_record_inference() { let (_, mut a) = configured(resources()); diff --git a/runtime/packages/sdk/scripts/runtime-methods.json b/runtime/packages/sdk/scripts/runtime-methods.json index 9e19199f5..7f8a2478c 100644 --- a/runtime/packages/sdk/scripts/runtime-methods.json +++ b/runtime/packages/sdk/scripts/runtime-methods.json @@ -1,6 +1,6 @@ { "source": { - "commit": "aae332572834fbfa065c0f696017c0de3540649d", + "commit": "7cbf82a4f5b1ef39947ddee6424b436bc8d3daa7", "path": "runtime/crates/replica/src/frames.rs" }, "methods": [ diff --git a/runtime/packages/sdk/src/application/asset-pins.json b/runtime/packages/sdk/src/application/asset-pins.json index 52386b4a5..d877bc31f 100644 --- a/runtime/packages/sdk/src/application/asset-pins.json +++ b/runtime/packages/sdk/src/application/asset-pins.json @@ -1,11 +1,11 @@ { "runtime": { - "sha256": "7004351583c3a79411e644fafe9f5786816d32ea058605d1905fd50666012f40", - "bytes": 3875815, - "maxBytes": 3875815, + "sha256": "247410a98e3a31adcd9c453cae4b607ba0528cce617d6a288bfec8745352d0b1", + "bytes": 3877699, + "maxBytes": 3877699, "source": { "repository": "mdbase-dev/mdbase-connect", - "commit": "aae332572834fbfa065c0f696017c0de3540649d" + "commit": "7cbf82a4f5b1ef39947ddee6424b436bc8d3daa7" } }, "sqlite": {