From 5bd2edd1a478fb682aaf07318f17db30bfe7e88a Mon Sep 17 00:00:00 2001 From: Sami Fouad Date: Sat, 12 Sep 2026 14:44:05 -0600 Subject: [PATCH] Prepare deka-modules 0.3.0 with dsc#167 shared gate rulings -codex --- Cargo.lock | 74 +++++- Cargo.toml | 3 +- README.md | 57 +++- src/ds_imports.rs | 46 +++- src/integrity.rs | 70 +++++ src/lib.rs | 5 + src/module_spec.rs | 23 ++ src/project_gate.rs | 217 ++++++++++------ src/reconcile_tests.rs | 288 +++++++++++++++++++++ tests/fixtures/import_scan/declarations.ds | 14 + tests/fixtures/import_scan/noise.ds | 9 + 11 files changed, 713 insertions(+), 93 deletions(-) create mode 100644 src/integrity.rs create mode 100644 src/reconcile_tests.rs create mode 100644 tests/fixtures/import_scan/declarations.ds create mode 100644 tests/fixtures/import_scan/noise.ds diff --git a/Cargo.lock b/Cargo.lock index 62d4425..0633f61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,21 +8,60 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "deka-modules" -version = "0.2.0" +version = "0.3.0" dependencies = [ "serde", "serde_json", + "sha2", "tempfile", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "errno" version = "0.3.14" @@ -39,6 +78,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -160,6 +209,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "syn" version = "3.0.5" @@ -184,12 +244,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 0aff686..46fcbdd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deka-modules" -version = "0.2.0" +version = "0.3.0" edition = "2024" license = "Apache-2.0" description = "deka's module-resolution contracts: specifier vocabulary, project gate, DS import scanning, ds_modules resolution" @@ -10,6 +10,7 @@ publish = ["crates-io"] [dependencies] serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +sha2 = "0.10" [dev-dependencies] tempfile = "3.23.0" diff --git a/README.md b/README.md index e9c9c69..328f338 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,50 @@ scanned, and resolved on disk: - **`module_spec`** — the module specifier vocabulary: bare vs. relative vs. `@deka/`-scoped specs, summoned JavaScript (`@js/`), the closed stdlib module list, DekaScript source extensions (`.ds`, `.dsx`) and the file-resolution candidates they produce. -- **`project_gate`** — the project-boundary gate: given a set of imports and a - project's declared dependencies plus `ds_modules/` contents, decides whether - every import is either stdlib, declared-and-installed, or satisfied by a - `deka link`. -- **`ds_imports`** — a compiler-free scanner that extracts `import`/`from` - specifiers out of raw DekaScript source text (comment- and string-aware, no - parser dependency). +- **`project_gate`** — the shared lock + declared + installed + fsGraph + integrity gate for package imports, with explicit toolchain and local-link rules. +- **`ds_imports`** — the single compiler-free static import scanner for both + consumers' gates (comments, strings, multiline declarations, and re-exports). +- **`integrity`** — dsc-compatible SHA-256 hashing of installed package trees. - **`modules`** — `ds_modules/` resolution: locating the module root for a project, installing/linking packages into it, and reading it back for the gate and the runtime loader. +## 0.3.0 reconciliation (dsc#167) + +`module_spec::STDLIB_MODULE_NAMES` is the closed name vocabulary, extended only +by `STDLIB_SPEC_PREFIXES`: `component/`, `deka/`, `encoding/`, and `db/`. +`is_stdlib_module_spec` strips one `@deka/` alias before matching the same +vocabulary. Unknown `@deka/*`, `ui`, and `ui/*` are not stdlib. The old +`project_gate::is_stdlib_module_spec` path re-exports that same function. +`CLOSED_STDLIB_MODULES` remains a separate compiler-provided export contract: +`math` / `@deka/math` needs no package declaration or installation. + +Use `project_gate::validate_project_source(root, source, options)` for one +source. For a module graph, scan each source with `ds_imports::paths`, combine +those results, and call `validate_project`. This scanner is the gate contract; +consumers keep their real parser for compilation. The scanner collects static +quoted imports and re-exports, skips comments/string/template text, and does +not collect dynamic `import(...)` expressions. It is not a syntax validator; +quoted paths retain their source spelling (including escapes). Fixtures under +`tests/fixtures/import_scan` pin the common gate inputs without compiler deps. + +All bare package imports (including foreign scopes and unknown `@deka/*`) +require declaration and installation; relative, project-root `@/`, URL, and +compiler-provided math imports are excluded. Local links still require a +declaration and a valid target, but waive installed-package lock/hash checks. +An external module root supplies only recognized stdlib, never arbitrary +packages. `require_lockfile: false` permits an absent lock; it does not disable +checks against a lock that exists. + +Installed packages require a tuple entry in `deka.lock` `packages` (or legacy +`php.packages`). If tuple metadata records `fsGraph.hash` (or `fs_graph.hash`), +the resolved package tree must match it. Hashing uses dsc's sorted relative +paths + NUL + file bytes + newline, with the same dependency/build/cache and +macOS metadata exclusions. Missing fsGraph is accepted for older locks; +malformed recorded hashes and integrity mismatches fail with an install hint. +This is installed-tree integrity, distinct from package archive integrity. + ## Summoned JavaScript (`@js/`) `@js/three-js` is a reserved routing class for foreign JavaScript, distinct @@ -41,8 +74,8 @@ no extension probing or exports-map interpretation. The project gate requires the exact `@js/three-js` identity in `deka.json` `dependencies` or `devDependencies`, the matching `deka.lock` `packages` entry (using pm's existing `[version, source, metadata, integrity]` tuple), and a -resolvable vendored entry. Lock presence is checked here; integrity verification -remains the package manager's responsibility. Another `@js/` dependency, an +resolvable vendored entry. Summoned archive integrity verification remains the package manager's +responsibility; the fsGraph rule above applies to installed DekaScript packages. Another `@js/` dependency, an unprefixed name, `ds_modules/` copy, or local link does not satisfy the import. External stdlib roots and `require_lockfile: false` do not waive these checks: summoned JavaScript remains a project-owned, declared-and-vendored dependency. @@ -50,9 +83,9 @@ summoned JavaScript remains a project-owned, declared-and-vendored dependency. ## Who uses this `dekaruntime/deka` depends on this crate for its module resolver and project -gate. `dsc` (the standalone compiler) is expected to pick it up next, so both -tools agree on the same specifier and resolution rules without importing the -whole runtime. +gate. `dsc` (the standalone compiler) already consumes matching resolution APIs. +Both consumers will converge on these 0.3.0 gate contracts in follow-up changes +after publication. ## Versioning diff --git a/src/ds_imports.rs b/src/ds_imports.rs index 106c4d2..87d29ba 100644 --- a/src/ds_imports.rs +++ b/src/ds_imports.rs @@ -7,6 +7,7 @@ pub fn paths(source: &str) -> Vec { let mut out = Vec::new(); let bytes = source.as_bytes(); let mut i = 0; + let mut declaration = false; while i < bytes.len() { match bytes[i] { b'/' if bytes.get(i + 1) == Some(&b'/') => { @@ -21,26 +22,40 @@ pub fn paths(source: &str) -> Vec { } i = i.saturating_add(2); } - b'"' | b'\'' => i = skip_string(bytes, i), + b'"' | b'\'' | b'`' => { + i = skip_string(bytes, i); + declaration = false; + } b'i' if is_word(bytes, i, b"import") => { i += 6; i = skip_ws(bytes, i); + declaration = bytes.get(i) != Some(&b'('); if i < bytes.len() && (bytes[i] == b'"' || bytes[i] == b'\'') && let Some((path, next)) = take_string(bytes, i) { + declaration = false; out.push(path); i = next; continue; } } - b'f' if is_word(bytes, i, b"from") => { + b'e' if is_word(bytes, i, b"export") => { + i = skip_ws(bytes, i + 6); + declaration = matches!(bytes.get(i), Some(b'{' | b'*')); + } + b';' => { + declaration = false; + i += 1; + } + b'f' if declaration && is_word(bytes, i, b"from") => { i += 4; i = skip_ws(bytes, i); if i < bytes.len() && (bytes[i] == b'"' || bytes[i] == b'\'') && let Some((path, next)) = take_string(bytes, i) { + declaration = false; out.push(path); i = next; continue; @@ -63,23 +78,40 @@ fn is_word(bytes: &[u8], i: usize, word: &[u8]) -> bool { } fn is_ident(b: u8) -> bool { - b.is_ascii_alphanumeric() || b == b'_' + b.is_ascii_alphanumeric() || b == b'_' || b == b'$' || !b.is_ascii() } fn skip_ws(bytes: &[u8], mut i: usize) -> usize { - while i < bytes.len() && bytes[i].is_ascii_whitespace() { - i += 1; + loop { + match bytes.get(i) { + Some(b) if b.is_ascii_whitespace() => i += 1, + Some(b'/') if bytes.get(i + 1) == Some(&b'/') => { + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + } + Some(b'/') if bytes.get(i + 1) == Some(&b'*') => { + i += 2; + while i + 1 < bytes.len() && &bytes[i..i + 2] != b"*/" { + i += 1; + } + i = (i + 2).min(bytes.len()); + } + _ => break, + } } i } fn skip_string(bytes: &[u8], i: usize) -> usize { - take_string(bytes, i).map(|(_, next)| next).unwrap_or(i + 1) + take_string(bytes, i) + .map(|(_, next)| next) + .unwrap_or(bytes.len()) } fn take_string(bytes: &[u8], i: usize) -> Option<(String, usize)> { let quote = *bytes.get(i)?; - if quote != b'"' && quote != b'\'' { + if quote != b'"' && quote != b'\'' && quote != b'`' { return None; } let mut j = i + 1; diff --git a/src/integrity.rs b/src/integrity.rs new file mode 100644 index 0000000..3bf3947 --- /dev/null +++ b/src/integrity.rs @@ -0,0 +1,70 @@ +//! dsc-compatible installed-package fsGraph hashing. +use sha2::{Digest, Sha256}; +use std::io::Read; +use std::path::{Path, PathBuf}; + +/// SHA-256 of sorted relative paths, NUL, file bytes, and newline per file. +/// Excludes dependency/build/cache trees and macOS metadata, as in dsc#167. +pub fn compute_fs_graph_hash(root: &Path) -> Result { + let mut files = Vec::new(); + collect_integrity_files(root, &mut files)?; + files.sort(); + let mut hasher = Sha256::new(); + for path in files { + let rel = path + .strip_prefix(root) + .map_err(|_| "failed to normalize integrity path".to_string())?; + let rel_str = rel.to_string_lossy().replace('\\', "/"); + hasher.update(rel_str.as_bytes()); + hasher.update(b"\0"); + let mut file = std::fs::File::open(&path) + .map_err(|err| format!("failed to open {}: {err}", path.display()))?; + let mut buf = [0u8; 8192]; + loop { + let read = file + .read(&mut buf) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buf[..read]); + } + hasher.update(b"\n"); + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn collect_integrity_files(current: &Path, out: &mut Vec) -> Result<(), String> { + let mut entries = std::fs::read_dir(current) + .map_err(|err| format!("failed to read {}: {err}", current.display()))? + .collect::, _>>() + .map_err(|err| format!("failed to read {}: {err}", current.display()))?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_string_lossy(); + if path.is_dir() { + if matches!( + name.as_ref(), + "ds_modules" + | "php_modules" + | ".git" + | "target" + | "node_modules" + | "dist" + | ".deka" + | ".cache" + ) { + continue; + } + collect_integrity_files(&path, out)?; + } else if path.is_file() { + if name == ".DS_Store" || name.starts_with("._") { + continue; + } + out.push(path); + } + } + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 9ca4180..ddf08cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,3 +9,8 @@ pub mod ds_imports; pub mod module_spec; pub mod modules; pub mod project_gate; + +pub mod integrity; + +#[cfg(test)] +mod reconcile_tests; diff --git a/src/module_spec.rs b/src/module_spec.rs index 56e246e..e71a183 100644 --- a/src/module_spec.rs +++ b/src/module_spec.rs @@ -10,6 +10,7 @@ pub fn is_bare_module_specifier(spec: &str) -> bool { && !spec.starts_with("file://") } +/// Authoritative dsc prefix vocabulary (dsc#167), owned here for both consumers. /// Bare-specifier prefixes that name stdlib module families. Both the project /// gate (`is_stdlib_module_spec`) and the browser import map emitted by /// `deka build` derive their prefix lists from here so the two cannot drift: @@ -17,6 +18,28 @@ pub fn is_bare_module_specifier(spec: &str) -> bool { /// browser loads them from. pub const STDLIB_SPEC_PREFIXES: &[&str] = &["component/", "deka/", "encoding/", "db/"]; +/// Exhaustive stdlib names, apart from the families in [`STDLIB_SPEC_PREFIXES`]. +/// dsc#167: `@deka/` is an alias, never a wildcard; `ui` is no longer stdlib. +/// This vocabulary is distinct from the compiler-provided export contracts in +/// [`CLOSED_STDLIB_MODULES`]. +pub const STDLIB_MODULE_NAMES: &[&str] = &[ + "json", "postgres", "mysql", "sqlite", "bytes", "buffer", "http", "tcp", "tls", "fs", "crypto", + "jwt", "test", "cookies", "auth", "db", "time", "io", "math", +]; + +/// Shared compiler/gate stdlib membership (dsc#167). +pub fn is_stdlib_module_spec(spec: &str) -> bool { + let spec = spec.trim(); + if !is_bare_module_specifier(spec) { + return false; + } + let bare = spec.strip_prefix("@deka/").unwrap_or(spec); + STDLIB_MODULE_NAMES.contains(&bare) + || STDLIB_SPEC_PREFIXES + .iter() + .any(|prefix| bare.starts_with(prefix)) +} + /// Closed, toolchain-provided stdlib modules and the exact named exports each /// one guarantees. Unlike every other stdlib module, these have no package /// under `ds_modules/`: the dsc compiler lowers their imports to local diff --git a/src/project_gate.rs b/src/project_gate.rs index 5e28c8e..ca459ac 100644 --- a/src/project_gate.rs +++ b/src/project_gate.rs @@ -17,9 +17,9 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use crate::module_spec::{ - STDLIB_SPEC_PREFIXES, ds_source_candidates, is_bare_module_specifier, - is_closed_stdlib_module_spec, is_summoned_js_module_spec, module_spec_aliases, - resolve_summoned_js_module_file, summoned_js_package_name, + ds_source_candidates, is_bare_module_specifier, is_closed_stdlib_module_spec, + is_summoned_js_module_spec, module_spec_aliases, resolve_summoned_js_module_file, + summoned_js_package_name, }; use crate::modules::resolve_modules_dir; @@ -48,52 +48,19 @@ impl Default for GateOptions { } } -/// Whether a specifier names a stdlib module, and so is subject to the gate. -/// -/// This is the union of the three copies it replaces. Two of them diverged: -/// `js_pipeline` listed `http` and the other two did not, and `esm_loader` -/// accepted any `@deka/*` while the others resolved the tail against the bare -/// list. The union is the strictest reading of the three, and it closes the -/// hole where `deka build` skipped `@deka/http` entirely because neither rule -/// matched it. -/// -/// Closed, toolchain-provided modules (`math`, see -/// [`is_closed_stdlib_module_spec`]) are stdlib by membership but are exempted -/// from the gate's declaration/installation rules by `validate_project`: the -/// compiler provides them, so there is no package to declare or install. -pub fn is_stdlib_module_spec(spec: &str) -> bool { - let spec = spec.trim(); - if !is_bare_module_specifier(spec) || spec.starts_with("@user/") { - return false; - } +// Retain the 0.2 public path while keeping vocabulary in module_spec. +pub use crate::module_spec::is_stdlib_module_spec; - // Any `@deka/*` specifier is stdlib by construction — the scope is ours. - spec.starts_with("@deka/") - || STDLIB_SPEC_PREFIXES - .iter() - .any(|prefix| spec.starts_with(prefix)) - || matches!( - spec, - "json" - | "postgres" - | "mysql" - | "sqlite" - | "bytes" - | "buffer" - | "http" - | "tcp" - | "tls" - | "fs" - | "crypto" - | "jwt" - | "test" - | "cookies" - | "auth" - | "db" - | "time" - | "io" - | "math" - ) +/// Scan one source with the shared compiler-free scanner and apply the gate. +/// Consumers walking a graph should collect `ds_imports::paths` for each source +/// and pass the combined specifiers to [`validate_project`]. Compilation may +/// use a parser, but must not substitute a parser walk for this gate scan. +pub fn validate_project_source( + project_root: &Path, + source: &str, + opts: &GateOptions, +) -> Result<(), String> { + validate_project(project_root, &crate::ds_imports::paths(source), opts) } /// Locate the file a stdlib specifier resolves to under a modules directory. @@ -127,8 +94,13 @@ pub fn resolve_module_file(modules_dir: &Path, spec: &str) -> Option { /// Rules, in order: /// 1. summoned JS imports are declared, locked, and vendored (never waived) /// 2. `deka.lock` exists, unless the caller waived it -/// 3. every stdlib import is **declared** in `deka.json` dependencies -/// 4. every stdlib import resolves to a file under the modules directory +/// 3. every package import is **declared** in `deka.json` dependencies +/// 4. every package import resolves under ds_modules or a declared local link +/// 5. installed packages have lock entries and match any recorded fsGraph hash +/// +/// Compiler-provided math is exempt from package checks. External module roots +/// supply only recognized stdlib. A waived absent lock permits unlocked packages; +/// a present lock is always checked. Links waive installed-package lock/hash checks. /// /// The stdlib declaration rule did not originally exist. Resolution was /// satisfied by a directory happening to be present, so a package could import something it never @@ -140,19 +112,14 @@ pub fn validate_project( ) -> Result<(), String> { validate_summoned_js_imports(project_root, imports, opts.context)?; - // An external module root means the runtime supplies the stdlib; the local - // tree is not expected to contain it. - if opts + let external_stdlib = opts .module_root .as_deref() - .is_some_and(|root| root != project_root) - { - return Ok(()); - } + .is_some_and(|root| root != project_root); let who = opts.context; - if opts.require_lockfile { + if opts.require_lockfile && !external_stdlib { let lock_path = project_root.join("deka.lock"); if !lock_path.is_file() { return Err(format!( @@ -162,23 +129,26 @@ pub fn validate_project( } } - let stdlib_imports: BTreeSet = imports + let package_imports: BTreeSet = imports .iter() .map(|s| s.trim().to_string()) - .filter(|s| is_stdlib_module_spec(s)) + .filter(|s| { + is_bare_module_specifier(s) && !s.starts_with("@/") && !is_summoned_js_module_spec(s) + }) + .filter(|s| !(external_stdlib && is_stdlib_module_spec(s))) // Closed, toolchain-provided modules (dsc#142's `math`) ship inside // the compiler: there is intentionally no package to declare or // install, so stdlib declaration and installation do not apply to them. .filter(|s| !is_closed_stdlib_module_spec(s)) .collect(); - if stdlib_imports.is_empty() { + if package_imports.is_empty() { return Ok(()); } - // Stdlib declaration — declared in deka.json. + // Package declaration — declared in deka.json. let declared = declared_dependencies(project_root); - let undeclared: Vec<&String> = stdlib_imports + let undeclared: Vec<&String> = package_imports .iter() .filter(|spec| !is_declared(spec, &declared)) .collect(); @@ -204,22 +174,22 @@ pub fn validate_project( // like it worked. let linked = crate::modules::read_linked_modules(project_root) .map_err(|error| format!("{who}: local package link is unusable: {error}"))?; - let stdlib_imports: BTreeSet = stdlib_imports + let package_imports: BTreeSet = package_imports .into_iter() .filter(|spec| !is_satisfied_by_link(spec, &linked)) .collect(); - if stdlib_imports.is_empty() { + if package_imports.is_empty() { return Ok(()); } - // Stdlib installation — present on disk. + // Package installation — present on disk. let modules_dir = resolve_modules_dir(project_root); if !modules_dir.is_dir() { return Err(format!( - "{who} requires {} at project root when using stdlib imports ({}). Run `deka install`.", + "{who} requires {} at project root when using package imports ({}). Run `deka install`.", modules_dir.display(), - stdlib_imports + package_imports .iter() .cloned() .collect::>() @@ -227,14 +197,14 @@ pub fn validate_project( )); } - let missing: Vec = stdlib_imports + let missing: Vec = package_imports .iter() .filter(|spec| resolve_module_file(&modules_dir, spec).is_none()) .cloned() .collect(); if missing.is_empty() { - Ok(()) + validate_package_integrity(project_root, &modules_dir, &package_imports, opts) } else { Err(format!( "{who}: declared but not installed under {}: {}. Run `deka install`.", @@ -244,6 +214,102 @@ pub fn validate_project( } } +/// Lock package identity preserves foreign scopes and drops import subpaths. +fn lock_package_name(spec: &str) -> String { + if spec.starts_with('@') { + spec.split('/').take(2).collect::>().join("/") + } else { + format!("@deka/{}", spec.split('/').next().unwrap_or(spec)) + } +} + +fn validate_package_integrity( + project_root: &Path, + modules_dir: &Path, + imports: &BTreeSet, + opts: &GateOptions, +) -> Result<(), String> { + let who = opts.context; + let lock_path = project_root.join("deka.lock"); + let raw = match std::fs::read_to_string(&lock_path) { + Ok(raw) => raw, + Err(error) if error.kind() == std::io::ErrorKind::NotFound && !opts.require_lockfile => { + return Ok(()); + } + Err(error) => { + return Err(format!( + "{who}: cannot read {}: {error}", + lock_path.display() + )); + } + }; + let lock: serde_json::Value = serde_json::from_str(&raw) + .map_err(|error| format!("{who}: invalid {}: {error}", lock_path.display()))?; + let packages = lock + .get("packages") + .and_then(|value| value.as_object()) + .or_else(|| lock.get("php")?.get("packages")?.as_object()); + let identities: BTreeSet = imports.iter().map(|spec| lock_package_name(spec)).collect(); + for package in identities { + let aliases = module_spec_aliases(&package); + let entry = packages + .and_then(|packages| aliases.iter().find_map(|alias| packages.get(alias))) + .ok_or_else(|| { + format!("{who}: module '{package}' has no deka.lock entry. Run `deka install`.") + })?; + let (_, _, metadata, _): (String, String, serde_json::Value, String) = + serde_json::from_value(entry.clone()).map_err(|error| { + format!("{who}: invalid deka.lock entry for '{package}': {error}") + })?; + // Older locks can omit fsGraph. If present, malformed metadata must not + // silently disable integrity verification. + let Some(graph) = metadata.get("fsGraph").or_else(|| metadata.get("fs_graph")) else { + continue; + }; + let expected = graph + .get("hash") + .and_then(|hash| hash.as_str()) + .map(str::trim) + .filter(|hash| hash.len() == 64 && hash.bytes().all(|ch| ch.is_ascii_hexdigit())) + .ok_or_else(|| format!("{who}: invalid fsGraph hash for '{package}' in deka.lock"))?; + // Hash the same alias tree that resolution selected, even if both bare + // and scoped package layouts exist. + for spec in imports + .iter() + .filter(|spec| lock_package_name(spec) == package) + { + let resolved = resolve_module_file(modules_dir, spec).ok_or_else(|| { + format!( + "{who}: module '{spec}' is no longer installed under {}", + modules_dir.display() + ) + })?; + let package_dir = module_spec_aliases(spec) + .into_iter() + .map(|alias| modules_dir.join(bare_package_path(&alias))) + .find(|dir| dir.is_dir() && resolved.starts_with(dir)) + .ok_or_else(|| { + format!("{who}: module '{package}' has no installed package directory") + })?; + let actual = crate::integrity::compute_fs_graph_hash(&package_dir) + .map_err(|error| format!("{who}: integrity mismatch for '{package}': {error}"))?; + if !actual.eq_ignore_ascii_case(expected) { + return Err(format!( + "{who}: integrity mismatch: module '{package}' in ds_modules/ does not match deka.lock. Run `deka install`." + )); + } + } + } + Ok(()) +} + +fn bare_package_path(spec: &str) -> String { + spec.split('/') + .take(if spec.starts_with('@') { 2 } else { 1 }) + .collect::>() + .join("/") +} + /// Summoned dependencies are project-owned, even when a runtime supplies the /// stdlib or waives its lockfile. Exact @js identities prevent scope aliases /// and local links from satisfying a different trust class. @@ -334,6 +400,13 @@ fn bare_name(spec: &str) -> &str { if let Some(rest) = spec.strip_prefix("@deka/") { return rest.split('/').next().unwrap_or(rest); } + if spec.starts_with('@') { + return spec + .match_indices('/') + .nth(1) + .map(|(end, _)| &spec[..end]) + .unwrap_or(spec); + } spec.split('/').next().unwrap_or(spec) } @@ -369,7 +442,7 @@ mod tests { write( tmp.path(), "deka.lock", - r#"{"lockfileVersion":1,"packages":{}}"#, + r#"{"lockfileVersion":1,"packages":{"@deka/crypto":["0.2.0","registry",{},"tarball"]}}"#, ); let crypto = tmp.path().join("ds_modules/@deka/crypto"); std::fs::create_dir_all(&crypto).unwrap(); @@ -463,7 +536,7 @@ mod tests { fn converged_specifiers_are_stdlib() { assert!(is_stdlib_module_spec("http"), "js_pipeline listed it"); assert!(is_stdlib_module_spec("@deka/http"), "build.rs skipped it"); - assert!(is_stdlib_module_spec("@deka/anything")); + assert!(!is_stdlib_module_spec("@deka/anything")); assert!(!is_stdlib_module_spec("@user/thing")); assert!(!is_stdlib_module_spec("./local")); } diff --git a/src/reconcile_tests.rs b/src/reconcile_tests.rs new file mode 100644 index 0000000..714b9dd --- /dev/null +++ b/src/reconcile_tests.rs @@ -0,0 +1,288 @@ +use crate::{ + ds_imports::paths, + integrity::compute_fs_graph_hash, + module_spec::{STDLIB_MODULE_NAMES, STDLIB_SPEC_PREFIXES, is_stdlib_module_spec}, + project_gate::{GateOptions, validate_project, validate_project_source}, +}; +use std::path::Path; + +fn write(root: &Path, name: &str, body: &str) { + let path = root.join(name); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, body).unwrap(); +} + +fn project() -> tempfile::TempDir { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "deka.json", + r#"{"dependencies":{"@deka/crypto":"1"}}"#, + ); + write( + tmp.path(), + "ds_modules/@deka/crypto/index.ds", + "export fn noop() {}\n", + ); + lock( + tmp.path(), + serde_json::json!({"fsGraph": {"hash": compute_fs_graph_hash(&tmp.path().join("ds_modules/@deka/crypto")).unwrap()}}), + ); + tmp +} + +fn lock(root: &Path, metadata: serde_json::Value) { + write( + root, + "deka.lock", + &serde_json::json!({"packages":{"@deka/crypto":["1","registry",metadata,"tarball"]}}) + .to_string(), + ); +} + +fn gate(root: &Path) -> Result<(), String> { + validate_project_source( + root, + "import { noop } from 'crypto'", + &GateOptions::default(), + ) +} + +#[test] +fn dsc_vocabulary_is_closed_and_alias_symmetric() { + assert_eq!( + STDLIB_SPEC_PREFIXES, + &["component/", "deka/", "encoding/", "db/"] + ); + for name in STDLIB_MODULE_NAMES.iter().copied().chain([ + "component/router", + "deka/core", + "encoding/json", + "db/postgres", + ]) { + assert!(is_stdlib_module_spec(name), "{name}"); + assert!(is_stdlib_module_spec(&format!("@deka/{name}")), "{name}"); + } + for name in [ + "anything", + "ui", + "ui/button", + "json/extra", + "http/extra", + "", + ] { + assert!(!is_stdlib_module_spec(name), "{name}"); + assert!(!is_stdlib_module_spec(&format!("@deka/{name}")), "{name}"); + } + assert!(!is_stdlib_module_spec("@user/json")); + assert!(!is_stdlib_module_spec("./json")); +} + +#[test] +fn wildcard_and_ui_imports_do_not_bypass_package_gate() { + let tmp = project(); + for spec in ["@deka/anything", "ui", "ui/button", "@deka/ui/button"] { + let err = + validate_project(tmp.path(), &[spec.into()], &GateOptions::default()).unwrap_err(); + assert!(err.contains("not declared"), "{err}"); + } +} + +#[test] +fn gate_rejects_modified_added_and_removed_package_files() { + for action in ["modify", "add", "remove"] { + let tmp = project(); + write(tmp.path(), "ds_modules/@deka/crypto/extra.ds", "original"); + lock( + tmp.path(), + serde_json::json!({"fsGraph":{"hash":compute_fs_graph_hash(&tmp.path().join("ds_modules/@deka/crypto")).unwrap()}}), + ); + gate(tmp.path()).unwrap(); + match action { + "modify" => write(tmp.path(), "ds_modules/@deka/crypto/extra.ds", "modified"), + "add" => write(tmp.path(), "ds_modules/@deka/crypto/new.ds", "new"), + _ => std::fs::remove_file(tmp.path().join("ds_modules/@deka/crypto/extra.ds")).unwrap(), + } + assert!(gate(tmp.path()).unwrap_err().contains("integrity mismatch")); + } +} + +#[test] +fn lock_entry_required_and_recorded_hash_cannot_be_waived() { + let tmp = project(); + write(tmp.path(), "deka.lock", r#"{"packages":{}}"#); + assert!(gate(tmp.path()).unwrap_err().contains("no deka.lock entry")); + for graph in [ + serde_json::json!({}), + serde_json::json!({"hash":"invalid"}), + serde_json::Value::Null, + ] { + lock(tmp.path(), serde_json::json!({"fsGraph":graph})); + assert!(gate(tmp.path()).unwrap_err().contains("invalid fsGraph")); + } + lock( + tmp.path(), + serde_json::json!({"fsGraph":{"hash":"0".repeat(64)}}), + ); + assert!( + validate_project_source( + tmp.path(), + "import 'crypto'", + &GateOptions { + require_lockfile: false, + ..GateOptions::default() + } + ) + .unwrap_err() + .contains("integrity mismatch") + ); + lock(tmp.path(), serde_json::json!({})); + gate(tmp.path()).unwrap(); // Historical lock without fsGraph. +} + +#[test] +fn fs_graph_matches_dsc_byte_contract_and_exclusions() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "z.ds", "z"); + write(tmp.path(), "a/index.ds", "a"); + // Independently computed SHA-256 of b"a/index.ds\0a\nz.ds\0z\n". + assert_eq!( + compute_fs_graph_hash(tmp.path()).unwrap(), + "7807374a91ed0ff97e105b0016949811bd4555ba5e4c38d2cc34decf286f64a6" + ); + for name in [ + "ds_modules/x.ds", + "php_modules/x.ds", + "node_modules/x.js", + "target/x", + "dist/x", + ".git/x", + ".cache/x", + ".deka/x", + ".DS_Store", + "._x", + ] { + write(tmp.path(), name, "ignored"); + } + assert_eq!( + compute_fs_graph_hash(tmp.path()).unwrap(), + "7807374a91ed0ff97e105b0016949811bd4555ba5e4c38d2cc34decf286f64a6" + ); +} + +#[test] +fn scanner_parity_fixtures_are_shared_gate_inputs() { + let cases: &[(&str, &[&str])] = &[ + ( + include_str!("../tests/fixtures/import_scan/declarations.ds"), + &[ + "./side.ds", + "io", + "json", + "@deka/crypto", + "@vendor/package/subpath", + "./point.ds", + "./all.ds", + "./commented.ds", + "bytes", + "buffer", + ], + ), + ( + include_str!("../tests/fixtures/import_scan/noise.ds"), + &["crypto"], + ), + ]; + let tmp = project(); + for (source, expected) in cases { + assert_eq!(paths(source), *expected); + assert_eq!( + validate_project_source(tmp.path(), source, &GateOptions::default()), + validate_project( + tmp.path(), + &expected.iter().map(|s| s.to_string()).collect::>(), + &GateOptions::default() + ) + ); + } +} + +#[test] +fn third_party_subpaths_preserve_scope_and_check_integrity() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "ds_modules/@vendor/tool/sub.ds", + "export const x = 1", + ); + let hash = compute_fs_graph_hash(&tmp.path().join("ds_modules/@vendor/tool")).unwrap(); + write(tmp.path(), "deka.lock", &serde_json::json!({"php":{"packages":{"@vendor/tool":["1","registry",{"fs_graph":{"hash":hash}},"tarball"]}}}).to_string()); + write( + tmp.path(), + "deka.json", + r#"{"dependencies":{"@other/tool":"1"}}"#, + ); + let source = "import { x } from '@vendor/tool/sub'"; + assert!( + validate_project_source(tmp.path(), source, &GateOptions::default()) + .unwrap_err() + .contains("not declared") + ); + write( + tmp.path(), + "deka.json", + r#"{"dependencies":{"@vendor/tool":"1"}}"#, + ); + validate_project_source(tmp.path(), source, &GateOptions::default()).unwrap(); + write(tmp.path(), "ds_modules/@vendor/tool/sub.ds", "changed"); + assert!( + validate_project_source(tmp.path(), source, &GateOptions::default()) + .unwrap_err() + .contains("integrity mismatch") + ); +} + +#[test] +fn declared_local_link_overrides_installed_hash_but_stale_link_fails() { + use crate::modules::{LinkEntry, LinkManifest, write_links_at}; + let tmp = project(); + let linked = tempfile::tempdir().unwrap(); + write(linked.path(), "deka.json", r#"{"name":"@deka/crypto"}"#); + write(linked.path(), "index.ds", "different working copy"); + write_links_at( + tmp.path(), + &LinkManifest { + packages: [( + "@deka/crypto".into(), + LinkEntry { + path: linked.path().to_path_buf(), + }, + )] + .into(), + ..LinkManifest::default() + }, + ) + .unwrap(); + lock( + tmp.path(), + serde_json::json!({"fsGraph":{"hash":"0".repeat(64)}}), + ); + gate(tmp.path()).unwrap(); + write(tmp.path(), "deka.json", "{}"); + assert!(gate(tmp.path()).unwrap_err().contains("not declared")); + write( + tmp.path(), + "deka.json", + r#"{"dependencies":{"crypto":"1"}}"#, + ); + linked.close().unwrap(); + assert!(gate(tmp.path()).unwrap_err().contains("link is unusable")); +} + +#[test] +fn integrity_checks_the_resolved_alias_not_a_shadow_copy() { + let tmp = project(); + write(tmp.path(), "ds_modules/crypto/index.ds", "tampered shadow"); + assert!(gate(tmp.path()).unwrap_err().contains("integrity mismatch")); + validate_project_source(tmp.path(), "import '@deka/crypto'", &GateOptions::default()).unwrap(); +} diff --git a/tests/fixtures/import_scan/declarations.ds b/tests/fixtures/import_scan/declarations.ds new file mode 100644 index 0000000..59fe0cd --- /dev/null +++ b/tests/fixtures/import_scan/declarations.ds @@ -0,0 +1,14 @@ +import "./side.ds" +import { echo } from "io" +import defaultValue from 'json' +import * as crypto from "@deka/crypto" +import { + thing, + other +} from "@vendor/package/subpath" +export { Point } from "./point.ds" +export * from "./all.ds" +import /* gate trivia */ "./commented.ds" +import { x } from /* intervening comment */ "bytes" +import { y } from // line comment +"buffer" diff --git a/tests/fixtures/import_scan/noise.ds b/tests/fixtures/import_scan/noise.ds new file mode 100644 index 0000000..9b02878 --- /dev/null +++ b/tests/fixtures/import_scan/noise.ds @@ -0,0 +1,9 @@ +// import { bad } from "comment" +/* export * from "block" */ +const text = "import { bad } from \"string\"" +const single = 'import "single"' +const template = `from "template"` +const from = "variable" +const important = "identifier" +import("dynamic") +import { real } from "crypto"