From 2282f908352e069efdf3106c50a9e0ad218d7f95 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 10 Sep 2026 19:49:18 +0000 Subject: [PATCH 1/2] test: add MCP directory preflight gate Co-authored-by: Akshay Dodeja --- docs/mcp/home.mdx | 11 +-- packages/mcp/DIRECTORY_PREFLIGHT.md | 24 +++++++ packages/mcp/eval/quality.test.ts | 3 +- packages/mcp/eval/quality.ts | 1 + packages/mcp/src/protocol-compat.test.ts | 90 +++++++++++++++++++++++- 5 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 packages/mcp/DIRECTORY_PREFLIGHT.md diff --git a/docs/mcp/home.mdx b/docs/mcp/home.mdx index 9488da1a..e4d7d793 100644 --- a/docs/mcp/home.mdx +++ b/docs/mcp/home.mdx @@ -398,8 +398,9 @@ List shipments with optional filters and pagination. **Parameters** - `number` *(string, optional)* – shipment, booking, or Bill of Lading identifier - `tracking_stopped` *(boolean, optional)* – filter by whether shipping-line tracking has stopped -- `include_containers` *(boolean, optional)* -- `page`, `page_size` *(number, optional)* +- `include_containers` *(boolean, optional)* – include container relationships (default: false) +- `page` *(number, optional)* – page number, starting at 1 +- `page_size` *(number, optional)* – results per page (default: 25; maximum: 25) **Good for** - "List recent shipments" @@ -415,7 +416,8 @@ List containers with optional filters and pagination. **Parameters** - `include` *(string[], optional)* – include `shipment`, `pod_terminal`, or both -- `page`, `page_size` *(number, optional)* +- `page` *(number, optional)* – page number, starting at 1 +- `page_size` *(number, optional)* – results per page (default: 25; maximum: 25) **Good for** - "List containers in my account" @@ -433,7 +435,8 @@ List tracking requests with optional filters and pagination. - `request_number` *(string, optional)* – tracking request identifier - `status` *(string, optional)* – `created`, `pending`, `succeeded`, or `failed` - `scac` *(string, optional)* – four-letter shipping line SCAC -- `page`, `page_size` *(number, optional)* +- `page` *(number, optional)* – page number, starting at 1 +- `page_size` *(number, optional)* – results per page (default: 25; maximum: 25) **Good for** - "Show failed tracking requests" diff --git a/packages/mcp/DIRECTORY_PREFLIGHT.md b/packages/mcp/DIRECTORY_PREFLIGHT.md new file mode 100644 index 00000000..7dfeb163 --- /dev/null +++ b/packages/mcp/DIRECTORY_PREFLIGHT.md @@ -0,0 +1,24 @@ +# MCP Directory Preflight + +Run this checklist before submitting or resubmitting Terminal49 to the Claude or +ChatGPT directory. + +- [ ] Run `npm run test --workspace @terminal49/mcp -- --run`, + `npm run build --workspace @terminal49/mcp`, and + `npm run lint --workspace @terminal49/mcp`. +- [ ] Compare the live `tools/list` response with the tool reference in + `docs/mcp/home.mdx`. Confirm parameter names, required fields, defaults, + limits, and descriptions match. +- [ ] Inspect successful tool results and server/tool instructions. They must + not contain `_agent_steering`, `_response_contract`, + `presentation_guidance`, `suggested_follow_ups`, or `suggested_tools`. +- [ ] Exercise validation, not-found, entitlement, and upstream failure cases. + Each known failure must identify its cause and next action; do not wrap + non-retryable failures in a generic retry message. +- [ ] Confirm `track_container.number` is required and `get_container` + `transport_events` returns only event counts plus the latest event. Use + `get_container_transport_events` for full history. +- [ ] Run `mcpdemo` against `https://mcp.terminal49.com` using Anthropic's + current [submission requirements](https://claude.com/docs/connectors/building/submission) + and [Software Directory Policy](https://support.claude.com/en/articles/13145358-anthropic-software-directory-policy). + Resolve every failure before resubmitting. diff --git a/packages/mcp/eval/quality.test.ts b/packages/mcp/eval/quality.test.ts index 84bc594a..a8ef3bdf 100644 --- a/packages/mcp/eval/quality.test.ts +++ b/packages/mcp/eval/quality.test.ts @@ -33,6 +33,7 @@ describe('scoreResult', () => { resultWithJson({ nested: { _agent_steering: true, + _response_contract: { purpose: 'Shape the answer' }, presentation_guidance: 'Present this result', suggested_follow_ups: ['Check another container'], suggested_tools: ['get_container'], @@ -47,7 +48,7 @@ describe('scoreResult', () => { expect(score.contractPass).toBe(false); expect(check?.pass).toBe(false); expect(check?.detail).toBe( - '_agent_steering, presentation_guidance, suggested_follow_ups, suggested_tools', + '_agent_steering, _response_contract, presentation_guidance, suggested_follow_ups, suggested_tools', ); }); }); diff --git a/packages/mcp/eval/quality.ts b/packages/mcp/eval/quality.ts index 2ff1822e..f3b29e90 100644 --- a/packages/mcp/eval/quality.ts +++ b/packages/mcp/eval/quality.ts @@ -48,6 +48,7 @@ export interface QualitySpec { const DEFAULT_LATENCY_BUDGET_MS = 8000; const REMOVED_STEERING_FIELDS = new Set([ '_agent_steering', + '_response_contract', 'presentation_guidance', 'suggested_follow_ups', 'suggested_tools', diff --git a/packages/mcp/src/protocol-compat.test.ts b/packages/mcp/src/protocol-compat.test.ts index c3d49360..592c765a 100644 --- a/packages/mcp/src/protocol-compat.test.ts +++ b/packages/mcp/src/protocol-compat.test.ts @@ -3,8 +3,12 @@ import { StreamableHTTPClientTransport, } from '@modelcontextprotocol/client'; import { createMcpHandler } from '@modelcontextprotocol/server'; +import { readFileSync } from 'node:fs'; import { afterEach, describe, expect, it } from 'vite-plus/test'; -import { createTerminal49McpServer } from './server.js'; +import { + createTerminal49McpServer, + TERMINAL49_SERVER_INSTRUCTIONS, +} from './server.js'; const LEGACY_PROTOCOL_VERSIONS = [ '2025-11-25', @@ -29,9 +33,42 @@ type AdvertisedProperty = { type AdvertisedInputSchema = { properties?: Record; + required?: string[]; additionalProperties?: boolean; }; +const DIRECTORY_FORBIDDEN_STEERING_FIELDS = [ + '_agent_steering', + '_response_contract', + 'presentation_guidance', + 'suggested_follow_ups', + 'suggested_tools', +] as const; + +const MCP_DOCS = readFileSync( + new URL('../../../docs/mcp/home.mdx', import.meta.url), + 'utf8', +); + +function documentedToolSection(toolName: string): string { + const marker = `### \`${toolName}\``; + const start = MCP_DOCS.indexOf(marker); + if (start === -1) { + throw new Error(`Missing ${toolName} section in docs/mcp/home.mdx`); + } + + const nextSection = MCP_DOCS.indexOf('\n---', start); + return MCP_DOCS.slice(start, nextSection === -1 ? undefined : nextSection); +} + +function documentedParameters(toolName: string): string[] { + return [ + ...documentedToolSection(toolName).matchAll( + /^- `([^`]+)` \*\([^)]*\)\*/gm, + ), + ].map((match) => match[1]); +} + async function connectClient( options: | { era: 'modern' } @@ -75,6 +112,57 @@ afterEach(async () => { }); describe('MCP protocol compatibility', () => { + it('keeps Directory-prohibited steering fields out of advertised instructions', async () => { + const client = await connectClient({ era: 'modern' }); + const { tools } = await client.listTools(); + const advertisedInstructions = JSON.stringify({ + instructions: TERMINAL49_SERVER_INSTRUCTIONS, + tools, + }); + + for (const field of DIRECTORY_FORBIDDEN_STEERING_FIELDS) { + expect(advertisedInstructions).not.toContain(field); + } + }); + + it('keeps critical MCP docs aligned with tools/list', async () => { + const client = await connectClient({ era: 'modern' }); + const { tools } = await client.listTools(); + const toolSchemas = new Map( + tools.map((tool) => [ + tool.name, + tool.inputSchema as AdvertisedInputSchema, + ]), + ); + + for (const toolName of [ + 'list_shipments', + 'list_containers', + 'list_tracking_requests', + ]) { + const advertised = Object.keys( + toolSchemas.get(toolName)?.properties ?? {}, + ).sort(); + expect(documentedParameters(toolName).sort(), toolName).toEqual( + advertised, + ); + } + + expect(toolSchemas.get('track_container')?.required).toContain('number'); + expect(documentedToolSection('track_container')).toMatch( + /^- `number` \*\(string, required\)\*/m, + ); + + const transportEventsDescription = + toolSchemas.get('get_container')?.properties?.include?.description; + expect(transportEventsDescription).toMatch( + /transport_events: Event summary \(count, rail event count, and latest event\); use get_container_transport_events for the full timeline/, + ); + expect(documentedToolSection('get_container')).toMatch( + /transport_events.*events\.count.*events\.rail_events_count.*events\.latest_event.*get_container_transport_events.*full timeline/, + ); + }); + it('advertises bounded, identifier-only inputs in tools/list', async () => { const client = await connectClient({ era: 'modern' }); const { tools } = await client.listTools(); From 477a221ceacdc596841f8f7383b81fe607b09ca8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 10 Sep 2026 19:51:25 +0000 Subject: [PATCH 2/2] test: run MCP quality scorer in eval suite Co-authored-by: Akshay Dodeja --- packages/mcp/src/protocol-compat.test.ts | 4 +--- packages/mcp/vitest.eval.config.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/mcp/src/protocol-compat.test.ts b/packages/mcp/src/protocol-compat.test.ts index 592c765a..743f510d 100644 --- a/packages/mcp/src/protocol-compat.test.ts +++ b/packages/mcp/src/protocol-compat.test.ts @@ -63,9 +63,7 @@ function documentedToolSection(toolName: string): string { function documentedParameters(toolName: string): string[] { return [ - ...documentedToolSection(toolName).matchAll( - /^- `([^`]+)` \*\([^)]*\)\*/gm, - ), + ...documentedToolSection(toolName).matchAll(/^- `([^`]+)` \*\([^)]*\)\*/gm), ].map((match) => match[1]); } diff --git a/packages/mcp/vitest.eval.config.ts b/packages/mcp/vitest.eval.config.ts index bff284f1..b17814c4 100644 --- a/packages/mcp/vitest.eval.config.ts +++ b/packages/mcp/vitest.eval.config.ts @@ -7,7 +7,7 @@ import { defineConfig } from 'vite-plus/test/config'; */ export default defineConfig({ test: { - include: ['eval/**/*.eval.ts'], + include: ['eval/**/*.eval.ts', 'eval/**/*.test.ts'], testTimeout: 30_000, hookTimeout: 60_000, // One live backend — run serially to stay friendly to rate limits.