Prepare deka-modules 0.3.0: reconcile the four dsc#167 rulings

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deka-modules"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2024"
 license = "Apache-2.0"
 description = "deka's module-resolution contracts: specifier vocabulary, project gate, DS import scanning, ds_modules resolution"
@@ -10,6 +10,7 @@ publish = ["crates-io"]
 [dependencies]
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+sha2 = "0.10"
 
 [dev-dependencies]
 tempfile = "3.23.0"

README.md

@@ -9,17 +9,50 @@ scanned, and resolved on disk:
 - **`module_spec`** — the module specifier vocabulary: bare vs. relative vs.
   `@deka/`-scoped specs, summoned JavaScript (`@js/`), the closed stdlib module list, DekaScript source
   extensions (`.ds`, `.dsx`) and the file-resolution candidates they produce.
-- **`project_gate`** — the project-boundary gate: given a set of imports and a
-  project's declared dependencies plus `ds_modules/` contents, decides whether
-  every import is either stdlib, declared-and-installed, or satisfied by a
-  `deka link`.
-- **`ds_imports`** — a compiler-free scanner that extracts `import`/`from`
-  specifiers out of raw DekaScript source text (comment- and string-aware, no
-  parser dependency).
+- **`project_gate`** — the shared lock + declared + installed + fsGraph
+  integrity gate for package imports, with explicit toolchain and local-link rules.
+- **`ds_imports`** — the single compiler-free static import scanner for both
+  consumers' gates (comments, strings, multiline declarations, and re-exports).
+- **`integrity`** — dsc-compatible SHA-256 hashing of installed package trees.
 - **`modules`** — `ds_modules/` resolution: locating the module root for a
   project, installing/linking packages into it, and reading it back for the
   gate and the runtime loader.
 
+## 0.3.0 reconciliation (dsc#167)
+
+`module_spec::STDLIB_MODULE_NAMES` is the closed name vocabulary, extended only
+by `STDLIB_SPEC_PREFIXES`: `component/`, `deka/`, `encoding/`, and `db/`.
+`is_stdlib_module_spec` strips one `@deka/` alias before matching the same
+vocabulary. Unknown `@deka/*`, `ui`, and `ui/*` are not stdlib. The old
+`project_gate::is_stdlib_module_spec` path re-exports that same function.
+`CLOSED_STDLIB_MODULES` remains a separate compiler-provided export contract:
+`math` / `@deka/math` needs no package declaration or installation.
+
+Use `project_gate::validate_project_source(root, source, options)` for one
+source. For a module graph, scan each source with `ds_imports::paths`, combine
+those results, and call `validate_project`. This scanner is the gate contract;
+consumers keep their real parser for compilation. The scanner collects static
+quoted imports and re-exports, skips comments/string/template text, and does
+not collect dynamic `import(...)` expressions. It is not a syntax validator;
+quoted paths retain their source spelling (including escapes). Fixtures under
+`tests/fixtures/import_scan` pin the common gate inputs without compiler deps.
+
+All bare package imports (including foreign scopes and unknown `@deka/*`)
+require declaration and installation; relative, project-root `@/`, URL, and
+compiler-provided math imports are excluded. Local links still require a
+declaration and a valid target, but waive installed-package lock/hash checks.
+An external module root supplies only recognized stdlib, never arbitrary
+packages. `require_lockfile: false` permits an absent lock; it does not disable
+checks against a lock that exists.
+
+Installed packages require a tuple entry in `deka.lock` `packages` (or legacy
+`php.packages`). If tuple metadata records `fsGraph.hash` (or `fs_graph.hash`),
+the resolved package tree must match it. Hashing uses dsc's sorted relative
+paths + NUL + file bytes + newline, with the same dependency/build/cache and
+macOS metadata exclusions. Missing fsGraph is accepted for older locks;
+malformed recorded hashes and integrity mismatches fail with an install hint.
+This is installed-tree integrity, distinct from package archive integrity.
+
 ## Summoned JavaScript (`@js/`)
 
 `@js/three-js` is a reserved routing class for foreign JavaScript, distinct
@@ -41,18 +74,18 @@ no extension probing or exports-map interpretation.
 The project gate requires the exact `@js/three-js` identity in `deka.json`
 `dependencies` or `devDependencies`, the matching `deka.lock` `packages` entry
 (using pm's existing `[version, source, metadata, integrity]` tuple), and a
-resolvable vendored entry. Lock presence is checked here; integrity verification
-remains the package manager's responsibility. Another `@js/` dependency, an
+resolvable vendored entry. Summoned archive integrity verification remains the package manager's
+responsibility; the fsGraph rule above applies to installed DekaScript packages. Another `@js/` dependency, an
 unprefixed name, `ds_modules/` copy, or local link does not satisfy the import.
 External stdlib roots and `require_lockfile: false` do not waive these checks:
 summoned JavaScript remains a project-owned, declared-and-vendored dependency.
 
 ## Who uses this
 
 `dekaruntime/deka` depends on this crate for its module resolver and project
-gate. `dsc` (the standalone compiler) is expected to pick it up next, so both
-tools agree on the same specifier and resolution rules without importing the
-whole runtime.
+gate. `dsc` (the standalone compiler) already consumes matching resolution APIs.
+Both consumers will converge on these 0.3.0 gate contracts in follow-up changes
+after publication.
 
 ## Versioning
 

src/ds_imports.rs

@@ -7,6 +7,7 @@ pub fn paths(source: &str) -> Vec<String> {
     let mut out = Vec::new();
     let bytes = source.as_bytes();
     let mut i = 0;
+    let mut declaration = false;
     while i < bytes.len() {
         match bytes[i] {
             b'/' if bytes.get(i + 1) == Some(&b'/') => {
@@ -21,26 +22,40 @@ pub fn paths(source: &str) -> Vec<String> {
                 }
                 i = i.saturating_add(2);
             }
-            b'"' | b'\'' => i = skip_string(bytes, i),
+            b'"' | b'\'' | b'`' => {
+                i = skip_string(bytes, i);
+                declaration = false;
+            }
             b'i' if is_word(bytes, i, b"import") => {
                 i += 6;
                 i = skip_ws(bytes, i);
+                declaration = bytes.get(i) != Some(&b'(');
                 if i < bytes.len()
                     && (bytes[i] == b'"' || bytes[i] == b'\'')
                     && let Some((path, next)) = take_string(bytes, i)
                 {
+                    declaration = false;
                     out.push(path);
                     i = next;
                     continue;
                 }
             }
-            b'f' if is_word(bytes, i, b"from") => {
+            b'e' if is_word(bytes, i, b"export") => {
+                i = skip_ws(bytes, i + 6);
+                declaration = matches!(bytes.get(i), Some(b'{' | b'*'));
+            }
+            b';' => {
+                declaration = false;
+                i += 1;
+            }
+            b'f' if declaration && is_word(bytes, i, b"from") => {
                 i += 4;
                 i = skip_ws(bytes, i);
                 if i < bytes.len()
                     && (bytes[i] == b'"' || bytes[i] == b'\'')
                     && let Some((path, next)) = take_string(bytes, i)
                 {
+                    declaration = false;
                     out.push(path);
                     i = next;
                     continue;
@@ -63,23 +78,40 @@ fn is_word(bytes: &[u8], i: usize, word: &[u8]) -> bool {
 }
 
 fn is_ident(b: u8) -> bool {
-    b.is_ascii_alphanumeric() || b == b'_'
+    b.is_ascii_alphanumeric() || b == b'_' || b == b'$' || !b.is_ascii()
 }
 
 fn skip_ws(bytes: &[u8], mut i: usize) -> usize {
-    while i < bytes.len() && bytes[i].is_ascii_whitespace() {
-        i += 1;
+    loop {
+        match bytes.get(i) {
+            Some(b) if b.is_ascii_whitespace() => i += 1,
+            Some(b'/') if bytes.get(i + 1) == Some(&b'/') => {
+                while i < bytes.len() && bytes[i] != b'\n' {
+                    i += 1;
+                }
+            }
+            Some(b'/') if bytes.get(i + 1) == Some(&b'*') => {
+                i += 2;
+                while i + 1 < bytes.len() && &bytes[i..i + 2] != b"*/" {
+                    i += 1;
+                }
+                i = (i + 2).min(bytes.len());
+            }
+            _ => break,
+        }
     }
     i
 }
 
 fn skip_string(bytes: &[u8], i: usize) -> usize {
-    take_string(bytes, i).map(|(_, next)| next).unwrap_or(i + 1)
+    take_string(bytes, i)
+        .map(|(_, next)| next)
+        .unwrap_or(bytes.len())
 }
 
 fn take_string(bytes: &[u8], i: usize) -> Option<(String, usize)> {
     let quote = *bytes.get(i)?;
-    if quote != b'"' && quote != b'\'' {
+    if quote != b'"' && quote != b'\'' && quote != b'`' {
         return None;
     }
     let mut j = i + 1;

src/integrity.rs

@@ -0,0 +1,70 @@
+//! dsc-compatible installed-package fsGraph hashing.
+use sha2::{Digest, Sha256};
+use std::io::Read;
+use std::path::{Path, PathBuf};
+
+/// SHA-256 of sorted relative paths, NUL, file bytes, and newline per file.
+/// Excludes dependency/build/cache trees and macOS metadata, as in dsc#167.
+pub fn compute_fs_graph_hash(root: &Path) -> Result<String, String> {
+    let mut files = Vec::new();
+    collect_integrity_files(root, &mut files)?;
+    files.sort();
+    let mut hasher = Sha256::new();
+    for path in files {
+        let rel = path
+            .strip_prefix(root)
+            .map_err(|_| "failed to normalize integrity path".to_string())?;
+        let rel_str = rel.to_string_lossy().replace('\\', "/");
+        hasher.update(rel_str.as_bytes());
+        hasher.update(b"\0");
+        let mut file = std::fs::File::open(&path)
+            .map_err(|err| format!("failed to open {}: {err}", path.display()))?;
+        let mut buf = [0u8; 8192];
+        loop {
+            let read = file
+                .read(&mut buf)
+                .map_err(|err| format!("failed to read {}: {err}", path.display()))?;
+            if read == 0 {
+                break;
+            }
+            hasher.update(&buf[..read]);
+        }
+        hasher.update(b"\n");
+    }
+    Ok(format!("{:x}", hasher.finalize()))
+}
+
+fn collect_integrity_files(current: &Path, out: &mut Vec<PathBuf>) -> Result<(), String> {
+    let mut entries = std::fs::read_dir(current)
+        .map_err(|err| format!("failed to read {}: {err}", current.display()))?
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|err| format!("failed to read {}: {err}", current.display()))?;
+    entries.sort_by_key(|entry| entry.file_name());
+    for entry in entries {
+        let path = entry.path();
+        let name = entry.file_name();
+        let name = name.to_string_lossy();
+        if path.is_dir() {
+            if matches!(
+                name.as_ref(),
+                "ds_modules"
+                    | "php_modules"
+                    | ".git"
+                    | "target"
+                    | "node_modules"
+                    | "dist"
+                    | ".deka"
+                    | ".cache"
+            ) {
+                continue;
+            }
+            collect_integrity_files(&path, out)?;
+        } else if path.is_file() {
+            if name == ".DS_Store" || name.starts_with("._") {
+                continue;
+            }
+            out.push(path);
+        }
+    }
+    Ok(())
+}

src/lib.rs

@@ -9,3 +9,8 @@ pub mod ds_imports;
 pub mod module_spec;
 pub mod modules;
 pub mod project_gate;
+
+pub mod integrity;
+
+#[cfg(test)]
+mod reconcile_tests;