ci: dogfood interactive AppMap before/after PR comparisons

.github/workflows/appmap-comparison-dogfood.yml

@@ -0,0 +1,258 @@
+name: Dogfood AppMap PR comparison
+
+on:
+  push:
+    branches:
+      - 'agent/**'
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: appmap-comparison-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  COMPARISON_CLI_SHA: aab690a5c4c01802551993718b508718c1139c11
+  COMPARISON_VIEWER_SHA: cfd3a0fef35b016cb0af20ee02ae46fd35414069
+  COMPARISON_REVIEW_ACTION_SHA: 67060904f03c0c8a1492e610f845eefd29e53a78
+  COMPARISON_SKILLS_SHA: 167db8d4b7eb56086c74a9ef9ffe7b2d3c8ab627
+
+jobs:
+  comparison-bundle:
+    name: Record and compare base/head behavior
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out base revision
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.base.sha || github.event.repository.default_branch }}
+          path: base
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Check out head revision
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          path: head
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Check out frozen comparison CLI contract
+        uses: actions/checkout@v6
+        with:
+          repository: tryingET/appmap-js
+          ref: ${{ env.COMPARISON_CLI_SHA }}
+          path: appmap-js
+          fetch-depth: 1
+          persist-credentials: false
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 22
+
+      - name: Install and build both appmap-node revisions
+        shell: bash
+        run: |
+          set -euo pipefail
+          corepack enable
+          (cd base && yarn install --immutable && yarn prepack)
+          (cd head && yarn install --immutable && yarn prepack)
+
+      - name: Record control and visual scenarios
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p comparison/appmaps
+          node head/test/behavioralComparison/record.mjs base control comparison/appmaps/control-base.appmap.json
+          node head/test/behavioralComparison/record.mjs head control comparison/appmaps/control-head.appmap.json
+          node head/test/behavioralComparison/record.mjs base before comparison/appmaps/demo-base.appmap.json
+          node head/test/behavioralComparison/record.mjs head after comparison/appmaps/demo-head.appmap.json
+
+      - name: Build and test comparison contract
+        shell: bash
+        run: |
+          set -euo pipefail
+          corepack enable
+          cd appmap-js
+          test "$(git rev-parse HEAD)" = "$COMPARISON_CLI_SHA"
+          yarn install --immutable
+          yarn build
+          yarn workspace @appland/models test comparison --runInBand
+          yarn workspace @appland/appmap test sequenceDiagramCompare --runInBand
+
+      - name: Sanitize recordings and build portable comparison bundles
+        shell: bash
+        run: |
+          set -euo pipefail
+          CLI="node appmap-js/packages/cli/built/cli.js"
+          BASE_SHA="$(git -C base rev-parse HEAD)"
+          HEAD_SHA="$(git -C head rev-parse HEAD)"
+
+          $CLI sanitize comparison/appmaps/*.appmap.json
+
+          $CLI sequence-diagram-compare \
+            comparison/appmaps/control-base.appmap.json \
+            comparison/appmaps/control-head.appmap.json \
+            --scenario appmap-node-control \
+            --base-revision "$BASE_SHA" \
+            --head-revision "$HEAD_SHA" \
+            --output-file comparison/control.compare.diff.sequence.json
+
+          $CLI sequence-diagram-compare \
+            comparison/appmaps/demo-base.appmap.json \
+            comparison/appmaps/demo-head.appmap.json \
+            --scenario added-authorization-call \
+            --base-revision "$BASE_SHA" \
+            --head-revision "$HEAD_SHA" \
+            --output-file comparison/authorization.compare.diff.sequence.json
+
+      - name: Verify and publish the frozen contract with the artifact
+        shell: bash
+        run: |
+          set -euo pipefail
+          node head/test/behavioralComparison/validate.mjs \
+            comparison/control.compare.diff.sequence.json \
+            comparison/authorization.compare.diff.sequence.json \
+            comparison/summary.md
+
+          mkdir -p comparison/contract/examples
+          cp appmap-js/packages/models/schema/comparison.schema.json comparison/contract/
+          cp appmap-js/packages/models/schema/README.md comparison/contract/
+          cp appmap-js/packages/models/schema/CHANGELOG.md comparison/contract/
+          cp appmap-js/packages/models/schema/examples/*.json comparison/contract/examples/
+
+          BASE_SHA="$(git -C base rev-parse HEAD)"
+          HEAD_SHA="$(git -C head rev-parse HEAD)"
+          cat > comparison/contract-lock.json <<JSON
+          {
+            "comparisonSchema": 1,
+            "appmapNodeBase": "$BASE_SHA",
+            "appmapNodeHead": "$HEAD_SHA",
+            "appmapJs": "$COMPARISON_CLI_SHA",
+            "vscodeAppland": "$COMPARISON_VIEWER_SHA",
+            "reviewAction": "$COMPARISON_REVIEW_ACTION_SHA",
+            "skills": "$COMPARISON_SKILLS_SHA"
+          }
+          JSON
+
+          cat comparison/summary.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload interactive comparison artifact
+        uses: actions/upload-artifact@v6
+        with:
+          name: appmap-pr-comparison-schema-v1
+          path: comparison
+          retention-days: 14
+          if-no-files-found: error
+
+  vscode-viewer:
+    name: Build and exercise comparison viewer
+    needs: comparison-bundle
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out pinned VS Code comparison viewer
+        uses: actions/checkout@v6
+        with:
+          repository: tryingET/vscode-appland
+          ref: ${{ env.COMPARISON_VIEWER_SHA }}
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Verify the pinned viewer revision
+        run: test "$(git rev-parse HEAD)" = "$COMPARISON_VIEWER_SHA"
+
+      - name: Download the generated comparison bundle
+        uses: actions/download-artifact@v8
+        with:
+          name: appmap-pr-comparison-schema-v1
+          path: dogfood-comparison
+
+      - name: Install the generated bundle as the editor test fixture
+        shell: bash
+        run: |
+          set -euo pipefail
+          target="test/fixtures/workspaces/project-diagram-diff/data/diff/minitest/Users_edit_unsuccessful_edit.compare.diff.sequence.json"
+          mkdir -p "$(dirname "$target")"
+          cp dogfood-comparison/authorization.compare.diff.sequence.json "$target"
+          printf 'authorize\n' > "$target.expect"
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 18
+          cache: yarn
+
+      - name: Install, typecheck, and compile
+        run: |
+          corepack enable
+          yarn install --immutable
+          yarn pretest
+          yarn compile
+
+      - name: Prepare Electron integration environment
+        run: |
+          yarn test:precache
+          yarn download-tools
+
+      - name: Open the generated comparison bundle in the extension host
+        shell: bash
+        run: |
+          export XDG_RUNTIME_DIR=/run/user/$(id -u)
+          export DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus
+          sudo mkdir -p "$XDG_RUNTIME_DIR"
+          sudo chown "$(id -u):$(id -g)" "$XDG_RUNTIME_DIR"
+          dbus-daemon --session --address="$DBUS_SESSION_BUS_ADDRESS" --fork --nopidfile
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- \
+            yarn test:integration test/integration/appmapEditor/openSequenceComparison.test.ts
+
+      - name: Package dogfood VSIX
+        run: yarn vsce package --out appmap-comparison-schema-v1-dogfood.vsix
+
+      - name: Upload comparison viewer
+        uses: actions/upload-artifact@v6
+        with:
+          name: appmap-comparison-schema-v1-vsix
+          path: appmap-comparison-schema-v1-dogfood.vsix
+          retention-days: 14
+          if-no-files-found: error
+
+  review-publisher:
+    name: Verify review artifact publisher
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out pinned review-action publisher
+        uses: actions/checkout@v6
+        with:
+          repository: tryingET/review-action
+          ref: ${{ env.COMPARISON_REVIEW_ACTION_SHA }}
+          path: review-action
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Check out pinned comparison skill
+        uses: actions/checkout@v6
+        with:
+          repository: tryingET/skills
+          ref: ${{ env.COMPARISON_SKILLS_SHA }}
+          path: comparison-skills
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Run offline action harness
+        run: |
+          test "$(git -C review-action rev-parse HEAD)" = "$COMPARISON_REVIEW_ACTION_SHA"
+          test "$(git -C comparison-skills rev-parse HEAD)" = "$COMPARISON_SKILLS_SHA"
+          (cd review-action && test/run.sh)
+
+      - name: Validate publisher and skill contract
+        shell: bash
+        run: |
+          set -euo pipefail
+          test -s comparison-skills/appmap-comparison/SKILL.md
+          grep -q 'appmap.comparison' comparison-skills/appmap-comparison/SKILL.md
+          grep -q 'elementIds' comparison-skills/appmap-comparison/SKILL.md
+          grep -q 'appmap-comparison' review-action/scripts/install-skills.sh
+          grep -q 'upload-comparison-artifact' review-action/action.yml

test/behavioralComparison/record.mjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+
+import assert from 'node:assert';
+import { spawnSync } from 'node:child_process';
+import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+
+const [targetRepositoryArg, variant, outputFileArg] = process.argv.slice(2);
+if (!targetRepositoryArg || !variant || !outputFileArg) {
+  console.error('usage: record.mjs <appmap-node-checkout> <control|before|after> <output-file>');
+  process.exit(2);
+}
+if (!['control', 'before', 'after'].includes(variant)) {
+  console.error(`unknown variant: ${variant}`);
+  process.exit(2);
+}
+
+const targetRepository = resolve(targetRepositoryArg);
+const outputFile = resolve(outputFileArg);
+const appmapNode = join(targetRepository, 'bin', 'appmap-node.js');
+const fixtureDir = await mkdtemp(join(tmpdir(), 'appmap-pr-comparison-'));
+
+async function findAppMaps(directory) {
+  const matches = [];
+  const visit = async (current) => {
+    for (const entry of await readdir(current, { withFileTypes: true })) {
+      const path = join(current, entry.name);
+      if (entry.isDirectory()) await visit(path);
+      else if (entry.name.endsWith('.appmap.json')) matches.push(path);
+    }
+  };
+  await visit(directory);
+  return matches;
+}
+
+function fixtureSource(selectedVariant) {
+  const authorizationCall =
+    selectedVariant === 'after'
+      ? `
+function authorize(request) {
+  return request.userId > 0;
+}
+`
+      : '';
+  const authorizationUse = selectedVariant === 'after' ? 'authorize(request);' : '';
+
+  return `
+function parseRequest() {
+  return { userId: 42 };
+}
+${authorizationCall}
+function loadUser(request) {
+  return { id: request.userId, role: 'reader' };
+}
+
+const request = parseRequest();
+${authorizationUse}
+const user = loadUser(request);
+console.log(user.id);
+`;
+}
+
+try {
+  await writeFile(
+    join(fixtureDir, 'appmap.yml'),
+    `name: appmap-pr-comparison\nlanguage: javascript\nappmap_dir: tmp/appmap\npackages:\n  - path: .\n    exclude:\n      - node_modules\n`
+  );
+  await writeFile(join(fixtureDir, 'index.mjs'), fixtureSource(variant));
+
+  const result = spawnSync(process.execPath, [appmapNode, 'index.mjs'], {
+    cwd: fixtureDir,
+    env: {
+      ...process.env,
+      APPMAP_RECORDER_PROCESS_ALWAYS: 'true',
+      TZ: 'UTC',
+    },
+    stdio: 'inherit',
+  });
+  assert.equal(result.status, 0, `appmap-node exited with status ${result.status}`);
+
+  const maps = await findAppMaps(join(fixtureDir, 'tmp'));
+  const processMaps = [];
+  for (const path of maps) {
+    const appmap = JSON.parse(await readFile(path, 'utf8'));
+    if (appmap.metadata?.recorder?.type === 'process') processMaps.push(path);
+  }
+  assert.equal(
+    processMaps.length,
+    1,
+    `expected one process AppMap, found ${processMaps.length}: ${maps.join(', ')}`
+  );
+
+  await mkdir(dirname(outputFile), { recursive: true });
+  await cp(processMaps[0], outputFile);
+  console.log(`Recorded ${variant} behavior to ${outputFile}`);
+} finally {
+  await rm(fixtureDir, { recursive: true, force: true });
+}