Fix LOGIN_TOKEN re-login loop, unthrottled slim /api/adapters, keep /start-trial source

packages/backend/src/adapters/adapters.controller.ts

@@ -10,17 +10,41 @@ import {
   BadRequestException,
   HttpCode,
   HttpException,
+  Headers,
+  Query,
+  Res,
 } from '@nestjs/common';
-import { ApiTags, ApiOperation, ApiBearerAuth } from '@nestjs/swagger';
+import { ApiTags, ApiOperation, ApiBearerAuth, ApiQuery } from '@nestjs/swagger';
+import { SkipThrottle } from '@nestjs/throttler';
+import type { Response } from 'express';
 import { AuthGuard } from '@nestjs/passport';
 import { AdaptersService } from './adapters.service';
 import { LicenseGuardService } from '../license/license-guard.service';
 import { McpServersService } from '../mcp-servers/mcp-servers.service';
 import { ProductEventService, ProductEvents, scrubProviderMessage } from '../audit/product-event.service';
 import { STARTER_PACK_MAX_INSTALL } from './starter-pack';
 
-// Public endpoints (no auth required) — used by the marketing website
+// Public endpoints (no auth required) — used by the marketing website.
+//
+// Not rate limited. The catalog is the same for every caller and changes only
+// with a release: each view is serialised once per process and sent with a
+// strong ETag (a revalidation is a 304) and a Cache-Control that lets the
+// caller keep it for a while. Every render of the website goes through one
+// droplet IP, so the global 100/min per-IP bucket answered its bursts with 429
+// and the site fell back to a static list of half the catalog. Exempting the
+// website's IP instead would hard-code our infrastructure in the app and break
+// on the next droplet move; the endpoint is cheap enough to leave open.
+const PUBLIC_CATALOG_CACHE_CONTROL = 'public, max-age=300, stale-while-revalidate=3600';
+
+/** If-None-Match (a list, `*`, weak or strong tags) names this ETag. */
+export function etagMatches(ifNoneMatch: string | undefined, etag: string): boolean {
+  if (!ifNoneMatch) return false;
+  const bare = (t: string) => t.trim().replace(/^W\//, '');
+  return ifNoneMatch.split(',').some((t) => t.trim() === '*' || bare(t) === bare(etag));
+}
+
 @ApiTags('Adapters')
+@SkipThrottle()
 @Controller('api/adapters')
 export class AdaptersPublicController {
   constructor(private readonly adaptersService: AdaptersService) {}
@@ -29,10 +53,42 @@ export class AdaptersPublicController {
   @ApiOperation({
     summary: 'List all available built-in adapters',
     description:
-      'Returns metadata for all pre-configured adapters. This endpoint is public so the marketing website can render the marketplace dynamically.',
+      'Returns metadata for all pre-configured adapters. This endpoint is public so the marketing website can render the marketplace dynamically. ' +
+      'With `?fields=slim` each adapter carries only what a catalog page shows (no setup instructions, variable help or prerequisites), ' +
+      'plus an `enriched` flag; the instructions of one adapter are at `/api/adapters/:slug/instructions`.',
   })
-  list() {
-    return this.adaptersService.listAll();
+  @ApiQuery({ name: 'fields', required: false, enum: ['slim'] })
+  list(
+    @Query('fields') fields: string | undefined,
+    @Headers('if-none-match') ifNoneMatch: string | undefined,
+    @Res() res: Response,
+  ) {
+    if (fields !== undefined && fields !== 'slim' && fields !== 'full') {
+      throw new BadRequestException('fields must be "slim" (or omitted for the full list)');
+    }
+    const { body, etag } = this.adaptersService.publicCatalog(
+      fields === 'slim' ? 'slim' : 'full',
+    );
+    res.setHeader('ETag', etag);
+    res.setHeader('Cache-Control', PUBLIC_CATALOG_CACHE_CONTROL);
+    if (etagMatches(ifNoneMatch, etag)) {
+      res.status(304).end();
+      return;
+    }
+    res.type('application/json').send(body);
+  }
+
+  // Two path segments, so the authenticated `GET :slug` never sees it.
+  @Get(':slug/instructions')
+  @ApiOperation({
+    summary: "One adapter's setup instructions",
+    description:
+      'The Markdown setup instructions of a listed adapter (null when it has none), for its marketplace page. Public.',
+  })
+  instructions(@Param('slug') slug: string, @Res({ passthrough: true }) res: Response) {
+    const result = this.adaptersService.publicInstructions(slug);
+    res.setHeader('Cache-Control', PUBLIC_CATALOG_CACHE_CONTROL);
+    return result;
   }
 }
 

packages/backend/src/adapters/adapters.service.ts

@@ -10,6 +10,7 @@ import { encrypt } from '../common/crypto/encryption.util';
 import { ConfigService } from '@nestjs/config';
 import { listAdapters, getAdapter, AdapterMeta, AdapterDefinition } from './catalog';
 import { hashInstructions } from './catalog-fingerprint';
+import { SerializedBody, serializeWithEtag, toAdapterSummary } from './catalog-summary';
 import { getRequiredSecret } from '../common/secrets.util';
 import {
   withOperatorProvided,
@@ -48,6 +49,7 @@ import { mcpToolPrefixOf } from '../connectors/mcp-connector-config.util';
 export class AdaptersService {
   private readonly logger = new Logger(AdaptersService.name);
   private readonly encryptionKey: string;
+  private readonly publicCatalogCache = new Map<string, SerializedBody>();
 
   constructor(
     private readonly prisma: PrismaService,
@@ -75,6 +77,30 @@ export class AdaptersService {
       });
   }
 
+  /**
+   * The public catalog as a ready-to-send body: the full list (`full`, what
+   * the connector store loads) or the website's slim one (`slim`, see
+   * AdapterSummary). The catalog only changes with a release, so each view is
+   * built and serialised once per process (per deployment mode) instead of
+   * per request.
+   */
+  publicCatalog(view: 'full' | 'slim'): SerializedBody {
+    const key = `${view}|${this.configService.get<string>('DEPLOYMENT_MODE') ?? ''}`;
+    let entry = this.publicCatalogCache.get(key);
+    if (!entry) {
+      const all = this.listAll();
+      entry = serializeWithEtag(view === 'slim' ? all.map(toAdapterSummary) : all);
+      this.publicCatalogCache.set(key, entry);
+    }
+    return entry;
+  }
+
+  /** Setup instructions of one listed adapter, for its marketplace page. */
+  publicInstructions(slug: string): { slug: string; instructions: string | null } {
+    const adapter = this.getBySlug(slug);
+    return { slug: adapter.slug, instructions: adapter.instructions ?? null };
+  }
+
   /**
    * The adapter as the setup form needs it: the definition, each variable
    * described (label, address / credential / setting, secret, help), and what

packages/backend/src/adapters/catalog-summary.spec.ts

@@ -0,0 +1,158 @@
+import { INestApplication } from '@nestjs/common';
+import { APP_GUARD } from '@nestjs/core';
+import { Test } from '@nestjs/testing';
+import { ThrottlerGuard, ThrottlerModule } from '@nestjs/throttler';
+import { AdaptersPublicController } from './adapters.controller';
+import { AdaptersService } from './adapters.service';
+import { ENRICHED_INSTRUCTIONS_MIN_CHARS, toAdapterSummary } from './catalog-summary';
+
+// Next's data cache refuses items over 2 MB; the website's catalog fetch must stay well under it.
+const NEXT_DATA_CACHE_LIMIT = 2 * 1024 * 1024;
+
+function service(mode?: string): AdaptersService {
+  const svc = Object.create(AdaptersService.prototype) as AdaptersService;
+  (svc as any).configService = { get: (k: string) => (k === 'DEPLOYMENT_MODE' ? mode : undefined) };
+  (svc as any).publicCatalogCache = new Map();
+  return svc;
+}
+
+describe('GET /api/adapters?fields=slim (AdaptersService.publicCatalog)', () => {
+  const svc = service('cloud');
+  const full = JSON.parse(svc.publicCatalog('full').body);
+  const slimBody = svc.publicCatalog('slim').body;
+  const slim = JSON.parse(slimBody);
+
+  it('lists the same adapters as the full catalog, in the same order', () => {
+    expect(slim.map((a: any) => a.slug)).toEqual(full.map((a: any) => a.slug));
+    expect(full).toEqual(JSON.parse(JSON.stringify(svc.listAll())));
+  });
+
+  it('is far below the 2 MB data-cache limit', () => {
+    const bytes = Buffer.byteLength(slimBody, 'utf8');
+    expect(bytes).toBeLessThan(NEXT_DATA_CACHE_LIMIT / 4);
+    expect(bytes).toBeLessThan(Buffer.byteLength(svc.publicCatalog('full').body, 'utf8') / 4);
+  });
+
+  it('leaves out the heavy fields and keeps what the catalog pages read', () => {
+    for (const a of slim) {
+      for (const heavy of ['instructions', 'prerequisites', 'probe', 'aliases', 'appRegistrationUrl', 'optionalEnvVars']) {
+        expect(a).not.toHaveProperty(heavy);
+      }
+      for (const field of ['slug', 'name', 'description', 'region', 'category', 'icon', 'docsUrl', 'requiredEnvVars', 'toolCount', 'enriched']) {
+        expect(a).toHaveProperty(field);
+      }
+      for (const m of Object.values(a.envVarMeta ?? {}) as object[]) {
+        expect(Object.keys(m).every((k) => k === 'kind' || k === 'secret')).toBe(true);
+      }
+    }
+  });
+
+  it('flags the adapters whose setup instructions make a page worth indexing', () => {
+    const bySlug = new Map(full.map((a: any) => [a.slug, a]));
+    for (const a of slim) {
+      const instructions: string | undefined = (bySlug.get(a.slug) as any).instructions;
+      expect(a.enriched).toBe((instructions?.trim().length ?? 0) >= ENRICHED_INSTRUCTIONS_MIN_CHARS);
+    }
+    expect(slim.some((a: any) => a.enriched)).toBe(true);
+  });
+
+  it('keeps the kind and secret flags of each variable', () => {
+    const s = toAdapterSummary({
+      slug: 'x', name: 'X', description: 'd', region: 'intl', category: 'c', icon: 'x', docsUrl: 'u',
+      requiredEnvVars: ['X_URL', 'X_KEY'], toolCount: 1, version: 'v', authType: 'API_KEY',
+      instructions: 'short',
+      envVarMeta: {
+        X_URL: { kind: 'address', label: 'URL', help: 'Your server' },
+        X_KEY: { kind: 'credential', secret: true, link: 'https://x.test' },
+        X_OTHER: { label: 'Only a label' },
+      },
+    } as any);
+    expect(s.envVarMeta).toEqual({ X_URL: { kind: 'address' }, X_KEY: { kind: 'credential', secret: true } });
+    expect(s.enriched).toBe(false);
+  });
+
+  it('is built once per deployment mode and keeps its ETag', () => {
+    const first = svc.publicCatalog('slim');
+    expect(svc.publicCatalog('slim')).toBe(first);
+    expect(first.etag).toMatch(/^"[A-Za-z0-9_-]+"$/);
+    expect(svc.publicCatalog('full').etag).not.toBe(first.etag);
+  });
+
+  it('follows the deployment mode: self-host-only adapters only off the cloud', () => {
+    const selfHosted = service(undefined);
+    const cloudSlugs = new Set(slim.map((a: any) => a.slug));
+    const selfHostSlugs = JSON.parse(selfHosted.publicCatalog('slim').body).map((a: any) => a.slug);
+    expect(selfHostSlugs.length).toBeGreaterThanOrEqual(cloudSlugs.size);
+    expect(slim.some((a: any) => a.selfHostOnly)).toBe(false);
+  });
+});
+
+describe('AdaptersPublicController over HTTP', () => {
+  let app: INestApplication;
+  let base: string;
+
+  beforeAll(async () => {
+    const svc = service('cloud');
+    const moduleRef = await Test.createTestingModule({
+      // A bucket of 2 per minute: anything rate limited answers 429 on the third call.
+      imports: [ThrottlerModule.forRoot([{ ttl: 60_000, limit: 2 }])],
+      controllers: [AdaptersPublicController],
+      providers: [
+        { provide: AdaptersService, useValue: svc },
+        { provide: APP_GUARD, useClass: ThrottlerGuard },
+      ],
+    }).compile();
+    app = moduleRef.createNestApplication();
+    await app.listen(0, '127.0.0.1');
+    base = `${await app.getUrl()}/api/adapters`;
+  });
+
+  afterAll(async () => {
+    await app.close();
+  });
+
+  it('is not rate limited', async () => {
+    const statuses: number[] = [];
+    for (let i = 0; i < 6; i++) {
+      const res = await fetch(`${base}?fields=slim`);
+      statuses.push(res.status);
+      await res.arrayBuffer();
+    }
+    expect(statuses).toEqual([200, 200, 200, 200, 200, 200]);
+  });
+
+  it('sends JSON with an ETag and Cache-Control, and 304 on revalidation', async () => {
+    const res = await fetch(`${base}?fields=slim`);
+    expect(res.headers.get('content-type')).toMatch(/^application\/json/);
+    expect(res.headers.get('cache-control')).toMatch(/public, max-age=\d+/);
+    const etag = res.headers.get('etag');
+    expect(etag).toBeTruthy();
+    expect(Array.isArray(await res.json())).toBe(true);
+
+    const again = await fetch(`${base}?fields=slim`, { headers: { 'If-None-Match': etag! } });
+    expect(again.status).toBe(304);
+  });
+
+  it('keeps the full list as before when fields is omitted', async () => {
+    const res = await fetch(base);
+    const body = await res.json();
+    expect(body[0]).toHaveProperty('slug');
+    expect(body.some((a: any) => typeof a.instructions === 'string')).toBe(true);
+  });
+
+  it('refuses an unknown fields value', async () => {
+    const res = await fetch(`${base}?fields=everything`);
+    expect(res.status).toBe(400);
+  });
+
+  it("serves one adapter's instructions, and 404 for an unknown slug", async () => {
+    const full = await (await fetch(base)).json();
+    const withInstructions = full.find((a: any) => typeof a.instructions === 'string');
+    const res = await fetch(`${base}/${withInstructions.slug}/instructions`);
+    expect(res.status).toBe(200);
+    expect(await res.json()).toEqual({ slug: withInstructions.slug, instructions: withInstructions.instructions });
+
+    const missing = await fetch(`${base}/no-such-adapter/instructions`);
+    expect(missing.status).toBe(404);
+  });
+});

packages/backend/src/adapters/catalog-summary.ts

@@ -0,0 +1,90 @@
+import { createHash } from 'crypto';
+import type { AdapterMeta } from './catalog';
+import type { EnvVarKind } from './env-var-meta';
+
+/**
+ * Setup instructions shorter than this do not make a page worth indexing on
+ * the website (its `isEnrichedAdapter`). Sent as a flag in the slim list so
+ * the site can decide without downloading the instructions.
+ */
+export const ENRICHED_INSTRUCTIONS_MIN_CHARS = 200;
+
+/**
+ * One row of `GET /api/adapters?fields=slim`: what the marketing website's
+ * catalog, marketplace cards, guides and sitemap read, and nothing else.
+ *
+ * The full list carries every adapter's setup instructions, variable help
+ * texts, prerequisites and probe. For 467 adapters that came to 2.6 MB, above
+ * the 2 MB limit of Next's data cache, so the site fetched the whole catalog
+ * again on every render. The instructions of one adapter are at
+ * `GET /api/adapters/:slug/instructions`.
+ */
+export interface AdapterSummary {
+  slug: string;
+  name: string;
+  description: string;
+  region: string;
+  category: string;
+  icon: string;
+  docsUrl: string;
+  requiredEnvVars: string[];
+  toolCount: number;
+  version: string;
+  authType?: string;
+  featured?: boolean;
+  priority?: number;
+  selfHostOnly?: boolean;
+  setupKind?: AdapterMeta['setupKind'];
+  /** Only `kind` and `secret` of each variable: enough to tell a credential
+   *  from an address or a setting. Labels, help, links and patterns stay in
+   *  the full list. */
+  envVarMeta?: Record<string, { kind?: EnvVarKind; secret?: boolean }>;
+  /** The adapter has setup instructions of at least
+   *  ENRICHED_INSTRUCTIONS_MIN_CHARS characters. */
+  enriched: boolean;
+}
+
+export function toAdapterSummary(a: AdapterMeta): AdapterSummary {
+  const summary: AdapterSummary = {
+    slug: a.slug,
+    name: a.name,
+    description: a.description,
+    region: a.region,
+    category: a.category,
+    icon: a.icon,
+    docsUrl: a.docsUrl,
+    requiredEnvVars: a.requiredEnvVars,
+    toolCount: a.toolCount,
+    version: a.version,
+    enriched:
+      (a.instructions?.trim().length ?? 0) >= ENRICHED_INSTRUCTIONS_MIN_CHARS,
+  };
+  if (a.authType) summary.authType = a.authType;
+  if (a.featured) summary.featured = true;
+  if (a.priority) summary.priority = a.priority;
+  if (a.selfHostOnly) summary.selfHostOnly = true;
+  if (a.setupKind) summary.setupKind = a.setupKind;
+  if (a.envVarMeta) {
+    const meta: NonNullable<AdapterSummary['envVarMeta']> = {};
+    for (const [name, m] of Object.entries(a.envVarMeta)) {
+      const slim: { kind?: EnvVarKind; secret?: boolean } = {};
+      if (m?.kind) slim.kind = m.kind;
+      if (m?.secret !== undefined) slim.secret = m.secret;
+      if (Object.keys(slim).length > 0) meta[name] = slim;
+    }
+    if (Object.keys(meta).length > 0) summary.envVarMeta = meta;
+  }
+  return summary;
+}
+
+/** A response body serialised once, with its strong ETag. */
+export interface SerializedBody {
+  body: string;
+  etag: string;
+}
+
+export function serializeWithEtag(value: unknown): SerializedBody {
+  const body = JSON.stringify(value);
+  const hash = createHash('sha1').update(body).digest('base64url');
+  return { body, etag: `"${hash}"` };
+}