Verify packs with the mdbase-connect runtime engine

.github/workflows/ci.yml

@@ -11,81 +11,60 @@ permissions:
 
 jobs:
   verify:
-    name: verify (${{ matrix.engine }})
+    name: verify
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        # Every pack is installed, upgraded, and tested through both engines.
-        # mdbase-ts also reopens installed collections in both legs.
-        engine: [mdbase-ts, mdbase-cli]
     steps:
       - name: Check out contracts
         uses: actions/checkout@v7
 
-      - name: Read pinned implementation identity
+      - name: Read pinned engine identity
         id: sources
         run: |
-          echo "repository=$(jq -r '.typescript_implementation.repository' sources.json)" >> "$GITHUB_OUTPUT"
-          echo "ref=$(jq -r '.typescript_implementation.ref' sources.json)" >> "$GITHUB_OUTPUT"
-          echo "cli_repository=$(jq -r '.rust_cli.repository' sources.json)" >> "$GITHUB_OUTPUT"
-          echo "cli_ref=$(jq -r '.rust_cli.ref' sources.json)" >> "$GITHUB_OUTPUT"
-          echo "engine_repository=$(jq -r '.rust_cli.engine_repository' sources.json)" >> "$GITHUB_OUTPUT"
-          echo "engine_ref=$(jq -r '.rust_cli.engine_ref' sources.json)" >> "$GITHUB_OUTPUT"
+          echo "repository=$(jq -r '.mdbase_engine.repository' sources.json)" >> "$GITHUB_OUTPUT"
+          echo "ref=$(jq -r '.mdbase_engine.ref' sources.json)" >> "$GITHUB_OUTPUT"
+          echo "package=$(jq -r '.mdbase_engine.package' sources.json)" >> "$GITHUB_OUTPUT"
 
-      - name: Check out TypeScript implementation
+      # Every pack is installed, upgraded and tested through the mdbase engine
+      # that the daemon, the Obsidian plugin and the SDK run (mdbase-connect
+      # runtime/), built from its `mdbase` package at the pinned commit.
+      - name: Check out mdbase engine
         uses: actions/checkout@v7
         with:
           repository: ${{ steps.sources.outputs.repository }}
           ref: ${{ steps.sources.outputs.ref }}
-          path: .sources/mdbase
+          path: .sources/mdbase-connect
+          sparse-checkout: runtime
 
-      - name: Check out mdbase CLI
-        if: matrix.engine == 'mdbase-cli'
-        uses: actions/checkout@v7
-        with:
-          repository: ${{ steps.sources.outputs.cli_repository }}
-          ref: ${{ steps.sources.outputs.cli_ref }}
-          path: .sources/cli/mdbase-connect
-
-      - name: Check out its Rust engine
-        if: matrix.engine == 'mdbase-cli'
-        uses: actions/checkout@v7
-        with:
-          repository: ${{ steps.sources.outputs.engine_repository }}
-          ref: ${{ steps.sources.outputs.engine_ref }}
-          path: .sources/cli/mdbase-rs
+      - name: Install pinned Rust toolchain
+        working-directory: .sources/mdbase-connect/runtime
+        run: rustup toolchain install && rustup show active-toolchain
 
       - uses: Swatinem/rust-cache@v2
-        if: matrix.engine == 'mdbase-cli'
         with:
-          workspaces: .sources/cli/mdbase-connect
-
-      - name: Build mdbase CLI
-        if: matrix.engine == 'mdbase-cli'
-        working-directory: .sources/cli/mdbase-connect
-        run: |
-          cargo build --locked -p mdbase-cli --bin mdbase
-          echo "MDBASE_VERIFY_CLI=$PWD/target/debug/mdbase" >> "$GITHUB_ENV"
+          workspaces: .sources/mdbase-connect/runtime
 
       - uses: actions/setup-node@v7
         with:
           node-version: 22
           cache: npm
 
+      - name: Build mdbase engine
+        working-directory: .sources/mdbase-connect/runtime
+        run: |
+          npm ci --prefix tools/wasm --no-audit --no-fund
+          cargo xtask mdbase-wasm
+          cd "${{ github.workspace }}/.sources/mdbase-connect/${{ steps.sources.outputs.package }}"
+          npm ci --no-audit --no-fund
+          npm run build:native
+          npm run build
+          echo "MDBASE_DIR=$PWD" >> "$GITHUB_ENV"
+          echo "mdbase $(jq -r .version package.json) at $(git rev-parse HEAD)"
+
       - name: Install catalog dependencies
         run: npm ci
 
-      - name: Build TypeScript implementation
-        working-directory: .sources/mdbase
-        run: |
-          npm ci && npm run build
-          echo "mdbase-ts $(jq -r .version package.json) at $(git rev-parse HEAD)"
-
-      - name: Build and verify catalog with ${{ matrix.engine }}
+      - name: Build and verify catalog
         run: npm test
-        env:
-          MDBASE_TS_DIR: .sources/mdbase
 
       - name: Check deterministic publication output
         run: git diff --exit-code -- dist

README.md

@@ -84,12 +84,21 @@ MDBASE_SPEC_DIR=../mdbase-spec npm run sync:runtime
 
 ## Build and verify
 
-Requires Node.js 22+ and a built checkout of
-[`@callumalpass/mdbase`](https://github.com/callumalpass/mdbase) (mdbase-ts)
-0.3.0-rc.9 or later (seed upgrade baseline lists). A sibling `../mdbase` checkout is used by default; set
-`MDBASE_TS_DIR` to override it. CI checks out and builds the tag pinned in
-`sources.json` (`typescript_implementation.ref`), which is the single source
-of truth for the mdbase-ts version the catalog is verified against.
+Requires Node.js 22+ and a built `mdbase` package from
+[mdbase-connect](https://github.com/mdbase-dev/mdbase-connect)'s
+`runtime/packages/mdbase`: the same engine core the daemon, the Obsidian plugin
+and the SDK run. A sibling `../mdbase-connect` checkout is used by default; set
+`MDBASE_DIR` to the package folder to override it. Build it from
+`runtime/` with:
+
+```sh
+npm ci --prefix tools/wasm
+cargo xtask mdbase-wasm
+cd packages/mdbase && npm ci && npm run build:native && npm run build
+```
+
+CI builds the commit pinned in `sources.json` (`mdbase_engine.ref`), which is
+the single source of truth for the engine the catalog is verified against.
 
 ```sh
 npm install
@@ -117,20 +126,10 @@ records declared in `collection.links`; Person 2.0.0 drops the separate `id`.
 The People pack 1.2.0 keeps shipping `mdbase.person` 1.0.0 so Person types
 customised under 1.1.0 continue to validate.
 
-Packs are installed through one of two engines. By default every check
-(catalog verification, the People pack tests, and the TaskNotes upgrade and
-plugin-collection tests) installs through mdbase-ts. To run the same checks
-through a Rust-engine mdbase CLI instead:
-
-```sh
-MDBASE_VERIFY_CLI=/absolute/path/to/mdbase npm test
-```
-
-Installed collections are reopened with mdbase-ts either way. CI runs
-`npm test` once per engine, with the CLI built from the commits pinned in
-`sources.json` (`rust_cli`). The CLI's engine must include mdbase-rs 056db73
-(seed upgrade baseline lists); `rust_cli.ref` is the mdbase-connect commit that
-pins that engine revision.
+Every check (catalog verification, the People pack tests, and the TaskNotes
+upgrade and plugin-collection tests) assesses and applies packs through the
+engine's `assessTypePack` and `applyTypePack`, and reopens installed
+collections with its native `Collection`.
 
 ## Seed upgrade baselines
 
@@ -159,10 +158,11 @@ duplicate or self baselines, and baselines whose type `kind` or `name` differs
 from the desired starter.
 
 Engines replace a starter equal to any listed baseline with the exact new
-starter, and merge an edited starter only against the baseline its lock records
-as its `origin_digest`; an edited type with no recorded origin (installed by an
-engine before 05A) or an unlisted origin is preserved with a reason. An
-unlisted starter is therefore never upgraded, so the catalog keeps an
+starter, and merge an edited starter against the baseline its lock records as
+its `origin_digest`. An edited type with no recorded origin (installed by an
+engine before 05A) or an unlisted origin merges against the nearest listed
+baseline that merges cleanly; failing that it is preserved. An unlisted
+starter is therefore upgraded only by inference, so the catalog keeps an
 invariant, checked by `scripts/seed-baselines.test.mjs`: for every offered
 (non-hidden) pack and seed target, `upgrade_from` lists every starter that any
 earlier version of the pack shipped at that target, including unlisted
@@ -192,9 +192,8 @@ conflicted wherever that mapping was customized. rc.16 is not listed.
 rc.18 ships rc.17's starter unchanged and lists every earlier starter as an
 upgrade baseline (revisions 4, 3, 2 and 1, from rc.16, rc.15, rc.13/rc.14 and
 rc.12), so a collection seeded by any earlier pack upgrades to it. rc.17 is
-hidden. A customised task type whose lock records no origin is preserved, and
-because it still implements the replaced contract the upgrade is blocked for
-review rather than merged against a guessed starter.
+hidden. A customised task type whose lock records no origin merges against
+the nearest earlier starter, keeping its customisations.
 
 ## TaskNotes settings record
 

scripts/pack-engine.mjs

@@ -1,32 +1,113 @@
-import { join, resolve } from "node:path";
+import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
+import { dirname, join, relative, resolve, sep } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import { cliPackEngine } from "./cli-pack-engine.mjs";
 
 const root = fileURLToPath(new URL("../", import.meta.url));
 
-/** The mdbase-ts build the checks import; CI pins it in sources.json. */
-export const mdbase = await import(
-  pathToFileURL(join(resolve(process.env.MDBASE_TS_DIR ?? join(root, "..", "mdbase")), "dist/index.js")).href
-);
+/**
+ * The mdbase engine the checks run: the `mdbase` package built from
+ * mdbase-connect's `runtime/packages/mdbase` (the same core as the daemon and
+ * the Obsidian plugin). CI builds it at the commit pinned in sources.json.
+ */
+const packageDir = resolve(process.env.MDBASE_DIR ?? join(root, "..", "mdbase-connect", "runtime", "packages", "mdbase"));
+const load = (entry) => import(pathToFileURL(join(packageDir, "dist", entry)).href);
+export const mdbase = await load("index.js");
+export const { Collection } = await load("node/index.js");
+
+export const packEngineName = `mdbase ${mdbase.VERSION}`;
 
-/** Which engine installs packs: the Rust mdbase CLI when MDBASE_VERIFY_CLI is set, otherwise mdbase-ts. */
-export const packEngineName = process.env.MDBASE_VERIFY_CLI ? "mdbase CLI" : "mdbase-ts";
+/** Every collection file the engine reads as a resource: all but `.mdbase/` and hidden folders. */
+export async function collectionResources(collectionRoot) {
+  const resources = {};
+  async function walk(directory) {
+    for (const entry of await readdir(directory, { withFileTypes: true })) {
+      if (entry.name.startsWith(".")) continue;
+      const path = join(directory, entry.name);
+      if (entry.isDirectory()) await walk(path);
+      else if (entry.isFile()) resources[relative(collectionRoot, path).split(sep).join("/")] = await readFile(path, "utf8");
+    }
+  }
+  await walk(collectionRoot);
+  return resources;
+}
+
+/** The contract implementations a collection's installed types declare. */
+export async function contractImplementations(collectionRoot, id, version) {
+  const catalog = await mdbase.loadCatalog(await collectionResources(collectionRoot));
+  return catalog.implementations.filter((i) => i.contract === id && i.version === version);
+}
+
+/** Run an engine call as a v0.3-style result: `{ valid, result }` or `{ valid: false, diagnostics }`. */
+export async function attempt(run) {
+  try {
+    return { valid: true, result: await run() };
+  } catch (error) {
+    if (!(error instanceof mdbase.MdbaseError)) throw error;
+    return { valid: false, diagnostics: [diagnostic(error)] };
+  }
+}
+
+function diagnostic(error) {
+  return { code: error.code, message: error.message, help: error.help, details: error.details };
+}
 
 /**
- * A pack installer with one interface for both engines: assess(adoptions)
- * and apply(digest, adoptions) return v0.3 operation results.
+ * A pack installer over a collection folder: assess(adoptions) and
+ * apply(digest, adoptions) return `{ valid, result, diagnostics }`. Apply
+ * writes the engine's exact `writes` and `deletes`, including the lock.
  */
 export async function packEngine(collectionRoot, provision, installedBy) {
-  if (process.env.MDBASE_VERIFY_CLI) {
-    return cliPackEngine(process.env.MDBASE_VERIFY_CLI, collectionRoot, provision, installedBy);
-  }
-  return {
-    assess: async (adoptions = {}) => mdbase.assessTypePack(collectionRoot, provision, {
-      installedBy, adoptResources: adoptions,
-    }),
-    apply: async (digest, adoptions = {}) => mdbase.applyTypePack(collectionRoot, provision, {
-      installedBy, adoptResources: adoptions, expectedAssessmentDigest: digest,
+  const pack = {
+    manifest: JSON.stringify(provision.manifest),
+    sources: Object.fromEntries(provision.resources.map(({ source, document }) => [source, document])),
+  };
+  const options = (adoptions) => ({ installed_by: installedBy, adopt: adoptions });
+  const assess = async (adoptions) => mdbase.assessTypePack({
+    pack, resources: await collectionResources(collectionRoot), options: options(adoptions),
+  });
+  // `live` is already the target's current digest; add the installed one from the receipt.
+  const describe = (assessment) => ({
+    ...assessment,
+    resources: assessment.resources.map((resource) => {
+      const installed = assessment.current?.resources.find(({ target }) => target === resource.target);
+      return {
+        ...resource,
+        current_digest: resource.live ?? undefined,
+        installed_digest: installed?.digest,
+      };
     }),
+  });
+  return {
+    assess: async (adoptions = {}) => {
+      const outcome = await attempt(async () => describe(await assess(adoptions)));
+      if (outcome.valid) outcome.diagnostics = outcome.result.issues;
+      return outcome;
+    },
+    apply: async (expectedDigest, adoptions = {}) => {
+      const outcome = await attempt(async () => {
+        const application = await mdbase.applyTypePack({
+          pack, resources: await collectionResources(collectionRoot), options: options(adoptions), expectedDigest,
+        });
+        for (const { path, document } of application.writes) {
+          const target = resolve(collectionRoot, path);
+          if (!target.startsWith(resolve(collectionRoot) + sep)) throw new Error(`Pack write escapes the collection: ${path}`);
+          await mkdir(dirname(target), { recursive: true });
+          await writeFile(target, document);
+        }
+        for (const path of application.deletes) await rm(resolve(collectionRoot, path), { force: true });
+        return describe(application.assessment);
+      });
+      if (!outcome.valid) {
+        // Say why: the assessment's issues and conflicting resources, as the refusal alone does not.
+        const assessment = await assess(adoptions).catch(() => null);
+        if (assessment) {
+          outcome.diagnostics.push(...assessment.issues, ...assessment.resources
+            .filter(({ action }) => action === "conflict")
+            .map(({ target, reason }) => ({ code: "type_pack_conflict", target, reason })));
+        }
+      }
+      return outcome;
+    },
     close: async () => {},
   };
 }