diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4eec3bc..67b50ac 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -21,7 +21,7 @@ }, { "name": "trogonstack-nats", - "description": "NATS messaging skills for designing subject hierarchies with naming conventions and wildcard patterns", + "description": "NATS channel design across Core NATS and JetStream, including AsyncAPI fragments, NACK-managed streams, optional consumers, tenancy, authorization, and migrations", "source": "./plugins/trogonstack-nats", "category": "development" }, diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index 869158d..bc13b90 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -20,7 +20,7 @@ { "name": "trogonstack-nats", "source": "plugins/trogonstack-nats", - "description": "NATS messaging skills for designing subject hierarchies with naming conventions and wildcard patterns" + "description": "NATS channel design across Core NATS and JetStream, including AsyncAPI fragments, NACK-managed streams, optional consumers, tenancy, authorization, and migrations" }, { "name": "trogonstack-datadog", diff --git a/.github/workflows/validate-plugins.yml b/.github/workflows/validate-plugins.yml index fa03dad..c42e2f8 100644 --- a/.github/workflows/validate-plugins.yml +++ b/.github/workflows/validate-plugins.yml @@ -27,3 +27,9 @@ jobs: run: npm install --no-save ajv@8 ajv-formats@3 - name: Validate plugin registrations run: node scripts/validate-plugins.mjs + - name: Validate tagged NATS subject examples + run: | + node plugins/trogonstack-nats/skills/nats-design-subject/scripts/validate-subjects.mjs \ + --root plugins/trogonstack-nats/skills/nats-design-subject + node plugins/trogonstack-nats/skills/nats-design-subject/scripts/validate-subjects.mjs \ + --file plugins/trogonstack-nats/skills/nats-design-subject/references/subject-fixtures.yaml diff --git a/plugins/trogonstack-nats/.claude-plugin/plugin.json b/plugins/trogonstack-nats/.claude-plugin/plugin.json index 33dd42a..221bbf1 100644 --- a/plugins/trogonstack-nats/.claude-plugin/plugin.json +++ b/plugins/trogonstack-nats/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "trogonstack-nats", - "description": "NATS messaging skills for designing subject hierarchies with naming conventions and wildcard patterns", + "description": "NATS channel design across Core NATS and JetStream, including AsyncAPI fragments, NACK-managed streams, optional consumers, tenancy, authorization, and migrations", "version": "0.2.0", "author": { "name": "TrogonStack", diff --git a/plugins/trogonstack-nats/.cursor-plugin/plugin.json b/plugins/trogonstack-nats/.cursor-plugin/plugin.json index 83f99d0..54af5fa 100644 --- a/plugins/trogonstack-nats/.cursor-plugin/plugin.json +++ b/plugins/trogonstack-nats/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "trogonstack-nats", "displayName": "TrogonStack NATS", - "description": "NATS messaging skills for designing subject hierarchies with naming conventions and wildcard patterns", + "description": "NATS channel design across Core NATS and JetStream, including AsyncAPI fragments, NACK-managed streams, optional consumers, tenancy, authorization, and migrations", "version": "0.2.0", "author": { "name": "TrogonStack" diff --git a/plugins/trogonstack-nats/plugin.json b/plugins/trogonstack-nats/plugin.json index 2a67ac6..c5899ed 100644 --- a/plugins/trogonstack-nats/plugin.json +++ b/plugins/trogonstack-nats/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "trogonstack-nats", "version": "0.2.0", - "description": "NATS messaging skills for designing subject hierarchies with naming conventions and wildcard patterns", + "description": "NATS channel design across Core NATS and JetStream, including AsyncAPI fragments, NACK-managed streams, optional consumers, tenancy, authorization, and migrations", "author": { "name": "TrogonStack", "url": "https://github.com/TrogonStack" diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/SKILL.md b/plugins/trogonstack-nats/skills/nats-design-subject/SKILL.md index 6a59176..603ed9e 100644 --- a/plugins/trogonstack-nats/skills/nats-design-subject/SKILL.md +++ b/plugins/trogonstack-nats/skills/nats-design-subject/SKILL.md @@ -1,6 +1,6 @@ --- name: nats-design-subject -description: "Design NATS subject hierarchies for messaging patterns (pub/sub, request/reply, streaming). Apply naming conventions, segmentation strategies, and wildcard patterns to create scalable subject architectures. Use when designing NATS messaging systems, choosing account-vs-subject namespace boundaries for multi-tenant communication, designing export/import subjects, or auditing existing subject hierarchies. Do not use for: (1) NATS server or account provisioning, (2) cluster setup, (3) client library implementation or connection code, (4) debugging connectivity or performance issues, (5) choosing between NATS and other messaging systems." +description: "Design and audit NATS channels across Core NATS and JetStream, including subjects, request-reply, streams, consumers, tenancy, authorization, and migrations." allowed-tools: - AskUserQuestion - Write @@ -8,322 +8,221 @@ allowed-tools: - Shell --- -# Design NATS Subject Hierarchy +# Design NATS Channels -Design a subject architecture that subscribers can efficiently navigate using wildcards, with proper segment ordering, account-aware tenant isolation, and growth path. +Design a NATS channel contract that stays understandable as the system grows. A subject is the routing key. Core NATS and JetStream are delivery and storage choices that must remain explicit in the contract. -## Interview Phase +JetStream is not a second application subject tree. Streams bind to ordinary application subjects. The `$JS` subjects are the control plane and need separate permissions and management ownership. -**Skip interview if ALL of these are already specified:** -- Messaging patterns (pub/sub, request/reply, streaming) -- Multi-tenancy needs (single/multi-tenant, scale requirements) -- Security requirements (authorization, tenant isolation) -- Persistence needs (JetStream vs core NATS) +## Start with the channel kind -**Always interview if**: Migrating existing subjects (needs anti-pattern audit first) +Choose the semantic kind before choosing the subject shape: -### Questions +| Kind | Transport | Contract | +| --- | --- | --- | +| RPC | Core NATS request-reply | A caller waits for one response or a timeout. | +| Live signal | Core NATS publish-subscribe | Delivery is best effort while subscribers are connected. | +| Durable event | JetStream | The event can be replayed and consumers acknowledge progress. | +| Durable work | JetStream | A worker pool shares a durable consumer and retries unacknowledged work. | -1. **Scope** — "Is this greenfield design or migrating existing subjects?" - - Impact: Migration needs anti-pattern audit first (see [references/anti-patterns.md](references/anti-patterns.md)) +Do not silently fall back from Core NATS to JetStream. That changes delivery guarantees. If a durable event also needs live observers, publish it on the JetStream subject and let live Core subscribers observe it. The durable consumer remains the correctness path. -2. **Multi-Tenancy & Scale** — "Do you need: (A) Single tenant, (B) Account-per-tenant isolation, (C) Shared account with subject prefixes, (D) Massive scale with regions/shards?" - - Impact: Determines whether tenant identity belongs in the NATS account boundary, the subject, or both +If a stream captures a subject, a Core publish can be stored without giving the publisher a persistence acknowledgement. Use the JetStream publish path when the publisher must know that the server accepted the message for storage. -3. **Messaging Patterns** — "Which patterns do you use? (A) Pub/Sub only, (B) Request/Reply, (C) Streaming/JetStream, (D) All/mix?" - - Impact: JetStream needs stream-aware subject design; request/reply has its own conventions +## Interview only for missing decisions -4. **Security** — "Do you need account-level isolation, subject-based authorization, or both?" - - Impact: Determines account boundaries, exports/imports, tenant prefixes, and permission boundaries +Skip questions that the user already answered. Ask for the missing contract decisions: -5. **Persistence** — "Do you need JetStream persistence or core NATS only?" - - Impact: Determines stream/consumer subject design and retention considerations +- Is this a new design or a migration? +- What channel kinds are needed: RPC, live signal, durable event, or durable work? +- Is tenancy an NATS account boundary, a shared-account subject prefix, or both? +- Which principals publish, subscribe, request, respond, or manage JetStream resources? +- What must survive a disconnect, and what can be lost? +- Who owns stream and consumer provisioning? ---- - -## When to Use - -- Designing a new NATS messaging system -- Planning account-aware multi-tenant subject isolation -- Organizing device telemetry or event streams -- Setting up request/reply patterns across microservices -- Defining event subject structure for event-sourced systems -- Building agentic AI platforms with inter-agent messaging - -## When NOT to Use - -- Configuring NATS server or cluster settings (infrastructure, not subject design) -- Writing NATS client code or connection logic (implementation, not architecture) -- Choosing between NATS and other messaging systems (technology evaluation) -- Debugging existing NATS connectivity or performance issues - ---- +For an existing hierarchy, audit current subjects, subscriptions, streams, consumers, and permissions before proposing replacements. ## Workflow -### 1. Identify Domain Boundaries +### Define the account and domain boundary -List all NATS account boundaries and business domains involved. For strict multi-tenancy, use one account per tenant first; then design short, domain-first subjects inside each account. +Use NATS Accounts for tenant or environment isolation when the boundary needs independent credentials, quotas, JetStream, or KV. Use a subject prefix only for an intentional shared-account surface or an exported platform view. -Use subject tenant prefixes only when accounts are intentionally unavailable or when designing a shared/platform surface that must carry tenant provenance. +Keep tenant identity out of normal tenant-local subjects when the account already supplies it. Put tenant provenance into platform aggregation subjects only when the data crosses an explicit export or import boundary. -### 2. Choose a Pattern +### Write the channel contract -Match the user's scenario to a pattern: - -| Use Case | Pattern | Example | -|----------|---------|---------| -| **Simple Domain** | `{domain}.{action}.{scope}` | `orders.created.us-west` | -| **Multi-Region** | `{domain}.{action}.{region}.{id}` | `devices.telemetry.us-east.sensor-456` | -| **Multi-Tenant SaaS** | Account per tenant, subjects: `{domain}.{action}.{id}` | `analytics.processed.report-123` in account `acme-corp` | -| **Shared Account Fallback** | `{tenant}.{domain}.{action}.{id}` | `acme-corp.analytics.processed.report-123` | -| **Multi-Tenant AI** | Account per tenant, subjects: `agents.{action}.{agent-id}.{task-id}` | `agents.task-assigned.agent-xyz.task-123` in account `tenant-abc` | -| **Request/Reply** | `{service}.request` / `{service}.reply` | `orders.request` / `orders.reply` | -| **Event Sourcing** | `{aggregate}.{action}.v{version}.{id}` | `orders.order.created.v1.order-123` | - -For full pattern details with subscriber paths and scaling guidance, read [references/patterns.md](references/patterns.md). - -### 3. Order Segments Strategically - -Apply these rules when ordering subject segments left-to-right inside the selected account: - -- **Broad to specific**: Domain → Action → Scope → Identifier -- **Low-cardinality left, high-cardinality right**: Regions (few values) before IDs (millions of values) -- **Never put IDs or UUIDs before actions** +Record one logical channel before creating subjects or streams. Use the small [AsyncAPI fragments](references/gateway-asyncapi-fragments.yaml) for application send and receive behavior. Use the [NACK JetStream resources](references/nack-jetstream.yaml) for Kubernetes-owned streams and optional consumers. The compact record below is review metadata, not a replacement wire specification: +```yaml +id: gateway.ingress.raw.github.v1 +kind: durable-event +subjects: + - gateway.ingress.raw.github.v1. +transport: jetstream +stream: GATEWAY_GITHUB_PRIMARY +owner: gitops +retention: 7d +ack: explicit ``` -✓ GOOD: orders.created.us-west.order-123 - ↑ ↑ - low-card high-card -✗ BAD: orders.order-123.us-west.created - ↑ - high-card early (kills wildcard filtering) +For an RPC channel: + +```yaml +id: gateway.rpc.v1.health +kind: rpc +request_subjects: + - gateway.rpc.v1.health +reply_subjects: + - _INBOX. +transport: core +storage: none +owner: gateway ``` -Why this matters: NATS wildcard matching scans left-to-right. High-cardinality values on the left force subscribers into inefficient `orders.*.us-west.created` patterns that must match thousands of IDs. - -For common ordering mistakes and migration strategies, read [references/anti-patterns.md](references/anti-patterns.md). +The channel record is the source for the subjects, stream filters, consumer filters, permissions, and management status. Do not make an application silently create production streams that are also declared by GitOps or an operator. -### 4. Plan Subscriber Paths +### Choose the subject shape -For each domain, document how subscribers will filter: +Use a stable, low-cardinality prefix followed by the fields subscribers need to filter: +```text +... ``` -orders.> → All order events -orders.created.> → All order creation events -orders.created.us-west.> → Orders created in US West -orders.created.us-west.order-123 → Specific order -``` - -Design subjects for subscribers, not publishers. Subscribers determine how you organize — a good hierarchy lets them efficiently filter with wildcards. - -### 5. Design Security Model (if multi-tenant) - -If the user needs tenant isolation or role-based access: - -- Prefer NATS Accounts for tenant isolation; each tenant gets its own subject namespace -- Use exports/imports for cross-account federation instead of assuming cross-tenant visibility -- Use tenant IDs in subjects only for shared-account fallbacks or platform aggregation surfaces -- Separate admin/platform subjects from user operations (`_admin.>` or platform account subjects) -- Apply least-privilege permissions per service - -For account-based tenancy, authorization patterns, and tenant isolation examples, read [references/security.md](references/security.md). - -### 6. Design JetStream Streams (if persistence needed) - -If the user needs JetStream: - -- Treat JetStream streams, consumers, and KV buckets as account-scoped resources -- Reuse stream/KV names across tenant accounts when the topology is identical -- Use one stream per domain inside each tenant account -- Use per-tenant streams in one shared account only as a fallback -- Consumer filters for fine-grained routing -- Account and domain retention limits (financial: years, telemetry: days) -- Keep to 4-6 subject segments — use consumer filters instead of deeper hierarchies - -For stream design, consumer patterns, and migration from core NATS, read [references/jetstream.md](references/jetstream.md). - -### 7. Validate and Write Output - -Present the design using this template: -```markdown -# NATS Subject Architecture: [System Name] - -## Domain Overview -[Describe the domains and their interactions] +Examples: -## Subject Hierarchy - -### Domain: [Name] -- `domain.action.{scope}.{id}` -- `domain.action.{scope}.{id}` +```text +orders.created.us-west.order-123 +orders.shipped.us-west.order-123 +gateway.ingress.raw.github.v1.pull_request +gateway.ingress.raw.slack.v1.event_callback +``` -Subscriber paths: -- `domain.>` — All events -- `domain.action.>` — Specific action +Publishers send concrete subjects. Only subscriptions, permissions, and stream or consumer filters use wildcards. -[Repeat for each domain] +- `*` matches exactly one token. +- `>` matches one or more trailing tokens and must be the final token. +- A wildcard is a complete token. `foo*` and `foo.>bar` are not NATS wildcards. +- Keep the subject readable and bounded. NATS documents a reasonable maximum of about 16 tokens and a subject length below 256 characters. A shorter project convention is fine when it is stated as a convention. +- Use one spelling and casing convention. Lowercase with hyphens is a good project convention, while NATS also permits underscores and case-sensitive names. +- Keep `$SYS`, `$JS`, `$KV`, `$O`, `$SRV`, and `_INBOX` for their protocol roles unless the design explicitly documents a boundary. -## Multi-Tenancy Model -[NATS Accounts, exports/imports, or shared-account subject prefixes] +Do not claim a performance effect from token order without a benchmark. Place fields where they make the intended subscription and authorization patterns obvious. -## Security Model -[Authorization rules per role/service, if applicable] +### Plan subscriber and permission paths -## JetStream Streams -[Account-scoped stream definitions and consumer filters, if applicable] +For every subject template, document the intended filters: -## Quality Validation -[Run checklist below] +```yaml nats-subjects +- kind: stream + subjects: + - gateway.ingress.raw.github.v1.> + - gateway.ingress.raw.slack.v1.> +- kind: consumer + subjects: + - gateway.ingress.raw.github.v1.pull_request ``` -### Example Output - -```markdown -# NATS Subject Architecture: IoT Smart Building Platform - -## Domain Overview +For a request, document the request subject and reply-to behavior. NATS request-reply uses a generated inbox. Responders publish to the request's reply subject; they do not invent a shared reply subject for each correlation ID. -Smart building system with 10,000+ sensors across multiple regions sending temperature, humidity, and occupancy data. Needs real-time monitoring, regional aggregation, and alerting. +For each principal, list publish and subscribe permissions separately. A broad `>` permission also reaches the JetStream control plane, so scope permissions to the channel and the required `$JS` or `_INBOX` subjects. -## Subject Hierarchy +### Design JetStream storage and consumers -### Domain: Devices -- `devices.telemetry.{region}.{device-id}.{metric}` -- `devices.telemetry.us-west.sensor-456.temperature` -- `devices.telemetry.us-west.sensor-456.humidity` -- `devices.telemetry.eu-central.sensor-789.occupancy` +Choose a stream boundary from retention, storage, replication, throughput, ownership, and consumer topology. One stream per domain is an option, not a rule. -Subscriber paths: -- `devices.telemetry.>` — All telemetry (global monitoring) -- `devices.telemetry.us-west.>` — Regional dashboard (US West) -- `devices.telemetry.>.>.temperature` — All temperature readings +Select the retention policy deliberately: -### Domain: Alerts -- `alerts.triggered.{severity}.{region}.{device-id}` -- `alerts.triggered.critical.us-west.sensor-456` +- `limits` keeps replayable history under message, byte, or age limits. +- `workqueue` removes messages after one durable consumer acknowledges them. +- `interest` keeps messages while a matching consumer has interest. -Subscriber paths: -- `alerts.triggered.critical.>` — Critical alerts only -- `alerts.triggered.>.us-west.>` — Regional alert dashboard +Define storage, replicas, discard behavior, duplicate windows, maximum message size, and retention. Define consumers separately with durable identity, subject filters, pull or push delivery, acknowledgement policy, acknowledgement wait or backoff, maximum delivery attempts, and maximum pending acknowledgements. -## Multi-Tenancy Model +Create a consumer only when a reader needs durable delivery. A durable event stream may have zero, one, or many consumers. The producer's event contract does not imply a consumer. -Not applicable (single organization) +Keep stream subjects disjoint within an account. JetStream rejects a stream whose subjects overlap another stream in the same account. Use sourcing or mirroring to copy messages into another stream. Work-queue consumers must not have overlapping filters. Use a subject filter for routing. Do not assume arbitrary header predicates are available in JetStream consumers. -## Security Model +### Define ownership and management -- Building operators: `devices.telemetry.>`, `alerts.>` (subscribe only) -- Alert service: `alerts.>` (publish + subscribe) -- Admin: `>` (full access) +A single control-plane owner must provision each stream and consumer. In Kubernetes, NACK is the control-plane owner for these JetStream resources. The management view should show: -## JetStream Streams - -Stream: `telemetry-us-west` - Subjects: `devices.telemetry.us-west.>` - Retention: 24h (high volume) - -Stream: `alerts` - Subjects: `alerts.>` - Retention: 30d - -## Quality Validation - -✓ All segments follow broad-to-specific order -✓ Device IDs at rightmost position -✓ Naming consistent (lowercase, hyphens) -✓ Regional filtering efficient -✓ 4 segments (within 4-6 limit) +```text +logical channel + -> subject or subject filter + -> Core or JetStream transport + -> stream and any provisioned consumers + -> owner + -> desired state and observed state + -> retention, backlog, and last error ``` -For complete real-world examples across microservices, IoT, SaaS, event sourcing, and agentic AI platforms, read [references/use-cases.md](references/use-cases.md). +For the Trogon gateway, GitOps applies the NACK resources while the gateway publishes and validates the ingress subjects. The application must not create or update competing stream definitions. The ownership must be visible in the channel contract. ---- - -## Quick Start (Simple Cases) +### Plan migrations -If you're designing a simple single-domain system without multi-tenancy: +Before changing a subject: -1. **Use Pattern 1** (Simple Domain): `{domain}.{action}.{id}` -2. **Skip references** — follow the workflow above -3. **Example**: `orders.created.order-123`, `payments.authorized.payment-456` +1. Inventory concrete subjects, wildcard subscriptions, streams, consumers, and ACLs. +2. Check wildcard syntax and pattern overlap. +3. Define the target contract and its stream ownership. +4. Decide whether dual publishing is safe for ordering, idempotency, and storage cost. +5. Move consumers before removing the old subject. +6. Observe old and new traffic before deleting the old path. -For multi-region, multi-tenant, or event-sourcing needs, continue with full workflow and read references as needed. +Make consumers idempotent with an application key when a migration can deliver a message twice. `Nats-Msg-Id` deduplicates only inside one stream and its duplicate window. It does not deduplicate across the old and new streams. ---- +## Output -## Reference Navigation +Write a design that includes: -The skill includes 5 detailed reference documents — read them as needed during workflow steps: - -- **[patterns.md](references/patterns.md)**: Read when choosing initial pattern (step 2) -- **[anti-patterns.md](references/anti-patterns.md)**: Read when migrating or auditing existing subjects -- **[security.md](references/security.md)**: Read when multi-tenancy/authorization needed (step 5) -- **[jetstream.md](references/jetstream.md)**: Read when persistence needed (step 6) -- **[use-cases.md](references/use-cases.md)**: Read for complete worked examples per domain - -Don't read all references upfront — use them progressively as the workflow requires. - ---- - -## Naming Rules +```markdown +# NATS Channel Architecture: [System] -- All lowercase with hyphens: `orders.created` ✓ -- Never underscores: `orders_created` ✗ -- Never mixed case: `Orders.Created` ✗ -- Keep to 4-6 segments maximum +## Account and tenancy boundary ---- +## Channel contracts -## Quality Checklist +## Subject templates and valid wildcard paths -- [ ] All segments follow broad-to-specific order (`{domain}.{action}.{scope}.{id}`) -- [ ] No UUID/ID fields appear before action/scope segments -- [ ] Naming consistent (all lowercase, hyphens, no underscores) -- [ ] Documented subscriber wildcard paths for each domain -- [ ] No subjects deeper than 6 segments -- [ ] Multi-tenancy boundary is clear (account-per-tenant, shared-account prefix fallback, or both) -- [ ] Security subjects defined for admin/monitoring access -- [ ] No conflicting patterns (e.g., `orders.created.123` vs `orders.123.created`) -- [ ] High-cardinality decision documented (why ID placement chosen) +## Core NATS request-reply and live signals -## Testing Your Design +## JetStream streams and consumers -Validate your subject hierarchy before deployment: +## Permissions and reserved subjects -```bash -# Start local NATS server -nats-server -D +## Provisioning owner and management readback -# Test subscriber wildcards -nats sub "orders.>" # Should match all order subjects -nats sub "orders.created.>" # Should match only creations -nats sub "orders.created.us-west.>" # Should match region-specific +## Migration and validation plan +``` -# Test publishing -nats pub "orders.created.us-west.order-123" "test message" +## Validation -# Verify JetStream streams (if applicable) -nats stream info orders-stream -nats consumer info orders-stream order-consumer -``` +Before presenting the design: -For existing deployments, audit current subjects: +- Validate every concrete publisher subject. +- Validate every wildcard expression. +- Verify that `>` appears only as the final token. +- Verify that request replies use a generated inbox or an explicitly documented alternative. +- Check stream and consumer filter overlap. +- Check permissions against the actual subject and JetStream control-plane requirements. +- Mark machine-readable Markdown fixtures with the `nats-subjects` info tag. For example, use ` ```yaml nats-subjects` and put a `kind` and `subjects` list in each YAML item. Keep the list shape even when it has one value. Ordinary examples stay untagged. +- Run the bundled subject validator when it is available: ```bash -# List active subjects (requires monitoring enabled) -nats server report jetstream - -# Check subject permissions -nats server check connection --account +node plugins/trogonstack-nats/skills/nats-design-subject/scripts/validate-subjects.mjs --root plugins/trogonstack-nats/skills/nats-design-subject ``` -## Reference Documentation - -- **[Patterns](references/patterns.md)**: 6 hierarchy patterns with subscriber paths and scaling guidance -- **[Anti-Patterns](references/anti-patterns.md)**: common mistakes with detection, fixes, and migration strategies -- **[Security & Multi-Tenancy](references/security.md)**: Authorization patterns and tenant isolation -- **[JetStream Design](references/jetstream.md)**: Stream filters, consumer subjects, and retention policies -- **[Use Cases](references/use-cases.md)**: Complete examples for microservices, IoT, SaaS, event sourcing, agentic AI +## References + +- [Channel contracts and Core versus JetStream](references/channel-contract.md) +- [AsyncAPI channel fragments](references/gateway-asyncapi-fragments.yaml) +- [NACK JetStream resources](references/nack-jetstream.yaml) +- [Tagged subject validation examples](references/channel-contract.md#gateway-example) +- [Subject fixture YAML](references/subject-fixtures.yaml) +- [Subject patterns](references/patterns.md) +- [JetStream streams and consumers](references/jetstream.md) +- [Subject anti-patterns](references/anti-patterns.md) +- [Security and tenancy](references/security.md) +- [Worked use cases](references/use-cases.md) diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/anti-patterns.md b/plugins/trogonstack-nats/skills/nats-design-subject/references/anti-patterns.md index a71fef8..3fe9945 100644 --- a/plugins/trogonstack-nats/skills/nats-design-subject/references/anti-patterns.md +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/anti-patterns.md @@ -23,12 +23,12 @@ orders.created.us-west.order-789 - Kills hierarchical filtering - Subscriber must use wildcard: `orders.*.created.us-west.>` (matches thousands) - Cannot efficiently say "all orders created in US West" -- High memory usage for wildcard subscriptions +- Makes the intended subscriber filters harder to express **Detection**: ```nats # If you're writing subscribers like this, you have the problem: -subscribers: orders.*.created.> # Wildcard in middle = high-cardinality left +subscribers: orders.*.created.> # Wildcard in middle = high-cardinality identifiers early in the subject ``` **Fix**: @@ -57,7 +57,7 @@ Phase 3 (week 5): **Real Example (E-Commerce)**: ``` -BEFORE (❌ high-cardinality left): +BEFORE (❌ high-cardinality identifiers early in the subject): orders.12345678.created (customer ID on left) orders.12345678.shipped (must filter by customer first) orders.12345679.created (10 million customers = 10 million prefixes) @@ -124,7 +124,7 @@ admins.us-west.updated.admin-456 (L2=region) Frustration: Subscriber needs: users.*.updated.> # Some have profile in L2 -users.>.updated.> # Others don't +users.updated.> # Others don't CONSISTENT ORDERING (✓): users.profile.updated.us-west.user-123 (always: domain.context.action.region.id) @@ -133,7 +133,7 @@ users.admin.updated.us-west.user-456 (always same order) Subscriber: users.profile.updated.us-west.> users.admin.updated.us-west.> -users.>.updated.us-west.> # All same ordering! +users.*.updated.us-west.> # All same ordering! ``` --- @@ -145,7 +145,7 @@ users.>.updated.us-west.> # All same ordering! ✗ BAD (9 segments): orders.created.v1.us-west.us-west-2a.customer-123.order-456.line-item-789.warehouse-101 -✓ GOOD (4-5 segments): +✓ GOOD (bounded subject with details in the message): orders.created.us-west.order-456 (other details in message headers/payload) ``` @@ -173,7 +173,7 @@ echo "orders.created.v1.us-west.customer.order-456.line-item.warehouse" | tr '.' **Fix**: 1. Move low-cardinality data to message headers/properties 2. Use JetStream consumer filters for fine-grained routing -3. Keep subjects to 4-5 segments max +3. Keep subjects within the project segment budget. Move details that do not need subject filtering into headers or payloads. **Better Approach Using JetStream**: ``` @@ -388,10 +388,12 @@ grep '[A-Z]' subjects.txt grep '_' subjects.txt ``` -**Standard**: -- All lowercase -- Use hyphens, never underscores -- Never use CamelCase or snake_case in segments +**Project convention example**: +- Use lowercase segments +- Use hyphens when separating words +- Keep one casing and separator policy across the hierarchy + +NATS permits underscores and case-sensitive names. Enforce a stricter style only when the project has chosen it. **Real Example**: ``` @@ -429,7 +431,7 @@ orders.created.eu-central.order-456 // Subscribers documented: Dashboard: orders.> (all regional order events) -Region monitor: orders.>.us-west.> (all orders in region) +Region monitor: orders.*.us-west.> (all orders in region) Created events only: orders.created.> (all creations, all regions) ``` @@ -451,8 +453,8 @@ For each domain, document: 2. Subscriber patterns: - All events: domain.> - Domain-specific: domain.action.> - - Scope-specific: domain.>.scope.> - - Specific resource: domain.>.scope.resource-id + - Scope-specific: domain.*.scope.> + - Specific resource: domain.*.scope.resource-id 3. Validate each pattern works and is efficient ``` @@ -474,7 +476,7 @@ Planned subscribers: - All telemetry: devices.telemetry.> - Regional: devices.telemetry.us-west.> - Specific device: devices.telemetry.us-west.sensor-456.> -- Specific metric: devices.telemetry.>.>.temperature +- Specific metric: devices.telemetry.*.*.temperature ``` --- @@ -608,7 +610,7 @@ Using `{tenant}.>` as the main isolation boundary in a system that needs strict ✓ Action/event type is always L2 unless a fixed namespace token is required ✓ No IDs or UUIDs before action/scope ✓ Consistent casing (lowercase) and separators (hyphens) -✓ Max 5-6 segments per subject +✓ Subject depth follows the project convention and stays within NATS limits ✓ Subscriber patterns documented ✓ No two wildcard subscriptions can match the same subject ✓ IDs at consistent positions across all subject types diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/channel-contract.md b/plugins/trogonstack-nats/skills/nats-design-subject/references/channel-contract.md new file mode 100644 index 0000000..605b0b7 --- /dev/null +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/channel-contract.md @@ -0,0 +1,175 @@ +# NATS Channel Contracts + +A NATS subject is a routing key. Core NATS and JetStream determine how the message is delivered and whether it is stored. Keep those decisions in one channel contract. + +JetStream does not require a separate application subject namespace. A stream captures ordinary subjects. `$JS` subjects are the JetStream control plane and must remain separate from application channels. + +Use [AsyncAPI 3.0](https://www.asyncapi.com/docs/reference/specification/v3.0.0) with the small [channel fragments](gateway-asyncapi-fragments.yaml) to describe application behavior. Its `send` and `receive` operations belong to the application that owns the document. Use [NACK JetStream resources](nack-jetstream.yaml) for Kubernetes-owned streams and optional consumers. Do not put a downstream consumer into a loosely coupled producer contract. A reader can publish its own receive-side contract and own a JetStream consumer separately. + +The NATS-specific examples below use list-valued `subjects`, `request_subjects`, and `reply_subjects` fields, even when one value is present. AsyncAPI keeps its singular `address` field, and JetStream examples keep exact server fields such as `filter_subject`. + +| Concern | Core NATS | JetStream | +| --- | --- | --- | +| Delivery | Live publish-subscribe or request-reply | Stream-backed delivery through consumers | +| Disconnect | A disconnected subscriber misses the message | A durable consumer can resume or replay | +| Publisher result | Publish has no storage acknowledgement | JetStream publish can return a stream acknowledgement | +| State | No stream or consumer state | Stream and consumer state must be provisioned and observed | +| Use | RPC and best-effort live signals | Durable events and work that require replay or retry | + +## Channel kinds + +### RPC + +Use Core NATS request-reply when the caller needs one response now: + +```yaml +id: gateway.rpc.v1.health +kind: rpc +request_subjects: + - gateway.rpc.v1.health +reply_subjects: + - _INBOX. +transport: core +storage: none +``` + +The client creates a unique inbox and sends it as the reply subject. The responder uses that reply subject. A timeout and retry policy are part of the contract. + +### Live signal + +Use Core NATS publish-subscribe for best-effort notifications: + +```yaml +id: gateway.signal.v1.configuration_changed +kind: live-signal +subjects: + - gateway.signal.v1.configuration_changed +transport: core +storage: none +``` + +A disconnected subscriber misses the message. Do not use this kind for an event that must be replayed. + +### Durable event + +Use JetStream for a fact that must survive a disconnect or be replayed: + +```yaml +id: gateway.ingress.raw.github.v1 +kind: durable-event +subjects: + - gateway.ingress.raw.github.v1. +transport: jetstream +stream: GITHUB_PRIMARY +retention: 7d +ack: explicit +``` + +Core subscribers may observe the same published subject for low-latency reactions. Durable consumers remain the correctness path. + +The publisher must choose its acknowledgement requirement explicitly. A Core publish to a subject captured by a stream does not tell the publisher that storage completed. A JetStream publish acknowledgement is the appropriate contract when the publisher needs that proof. + +Use the same shape for a second ingress source when it has a separate ownership or retention policy: + +```yaml +id: gateway.ingress.slack.v1 +kind: durable-event +subjects: + - gateway.ingress.raw.slack.v1. +transport: jetstream +stream: SLACK_PRIMARY +retention: 7d +ack: explicit +``` + +### Durable work + +Use a JetStream work-queue stream and add a durable consumer only when worker instances need shared, retryable work: + +```yaml +id: gateway.work.v1.installation_sync +kind: durable-work +subjects: + - gateway.work.v1.installation_sync +transport: jetstream +stream: GATEWAY_WORK +retention: workqueue +consumer: installation-sync +ack: explicit +``` + +Set redelivery, backoff, maximum attempts, and dead-letter behavior as part of the work contract. + +The `consumer` field belongs to the worker's receive-side contract. Omit it from a publisher-only event contract. + +## Gateway example + +```text +gateway.ingress.raw.github.v1.pull_request +gateway.ingress.raw.slack.v1.event_callback +gateway.events.v1.conversation.message +gateway.rpc.v1.health +gateway.work.v1.installation_sync +``` + +The GitHub and Slack ingress streams can remain separate when their retention, permissions, or consumers differ. A stream may exist before a reader is deployed. The important rule is that request subjects do not sit under the ingress event wildcard. + +The `nats-subjects` tag marks the examples that the bundled validator reads. The `yaml` language tag keeps the fixture readable as a small data document. Narrative examples use ordinary fences and are not treated as machine-readable contracts. + +```yaml nats-subjects +- kind: publish + subjects: + - gateway.ingress.raw.github.v1.pull_request +- kind: subscribe + subjects: + - gateway.ingress.raw.github.v1.> +- kind: stream + subjects: + - gateway.ingress.raw.github.v1.> +# This validates an optional reader filter. It does not provision a consumer. +- kind: consumer + subjects: + - gateway.ingress.raw.github.v1.pull_request +- kind: publish + subjects: + - gateway.ingress.raw.slack.v1.event_callback +- kind: subscribe + subjects: + - gateway.ingress.raw.slack.v1.> +- kind: publish + subjects: + - gateway.rpc.v1.health +- kind: deliver + subjects: + - _INBOX.generated +``` + +## Ownership + +Declare one owner for each stream and consumer. In Kubernetes, NACK owns the JetStream resources: + +```yaml +owner: gitops +provisioner: nack +publisher: trogon-gateway +``` + +The application can validate that a required stream exists. It should not silently create a second definition that competes with NACK. Apply mTLS connection settings to NACK through its Helm values and Kubernetes Secrets, not through application startup code. + +## Management readback + +Management should expose the same descriptor with desired and observed state. Consumer fields are optional and appear only when a reader owns one: + +```text +channel: gateway.ingress.raw.github.v1 +subjects: + - gateway.ingress.raw.github.v1.> +transport: jetstream +stream: GITHUB_PRIMARY +stream_state: ready +consumers: optional, owned by readers +backlog: 0 +owner: gitops +``` + +Core channels omit stream fields and instead expose responder, queue group, timeout, and authorization status. diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/gateway-asyncapi-fragments.yaml b/plugins/trogonstack-nats/skills/nats-design-subject/references/gateway-asyncapi-fragments.yaml new file mode 100644 index 0000000..4c8892a --- /dev/null +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/gateway-asyncapi-fragments.yaml @@ -0,0 +1,72 @@ +# AsyncAPI 3.0 fragments. +# Merge the channels and operations below into the owning application document. +# Message definitions remain in the application's existing components section. +channels: + githubIngress: + address: gateway.ingress.raw.github.v1.{event} + parameters: + event: + description: GitHub event name. + enum: + - pull_request + - issues + - installation + messages: + githubWebhook: + $ref: '#/components/messages/githubWebhook' + slackIngress: + address: gateway.ingress.raw.slack.v1.{event} + parameters: + event: + description: Slack event name. + enum: + - event_callback + - url_verification + messages: + slackWebhook: + $ref: '#/components/messages/slackWebhook' + healthRequest: + address: gateway.rpc.v1.health + messages: + healthRequest: + $ref: '#/components/messages/healthRequest' + healthReply: + address: null + messages: + healthResponse: + $ref: '#/components/messages/healthResponse' + installationSync: + address: gateway.work.v1.installation_sync + messages: + installationSync: + $ref: '#/components/messages/installationSync' +operations: + publishGithubWebhook: + action: send + channel: + $ref: '#/channels/githubIngress' + messages: + - $ref: '#/channels/githubIngress/messages/githubWebhook' + publishSlackWebhook: + action: send + channel: + $ref: '#/channels/slackIngress' + messages: + - $ref: '#/channels/slackIngress/messages/slackWebhook' + receiveHealthRequest: + action: receive + channel: + $ref: '#/channels/healthRequest' + messages: + - $ref: '#/channels/healthRequest/messages/healthRequest' + reply: + channel: + $ref: '#/channels/healthReply' + messages: + - $ref: '#/channels/healthReply/messages/healthResponse' + publishInstallationSync: + action: send + channel: + $ref: '#/channels/installationSync' + messages: + - $ref: '#/channels/installationSync/messages/installationSync' diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/jetstream.md b/plugins/trogonstack-nats/skills/nats-design-subject/references/jetstream.md index 27f9913..00d492d 100644 --- a/plugins/trogonstack-nats/skills/nats-design-subject/references/jetstream.md +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/jetstream.md @@ -4,6 +4,14 @@ This reference covers subject patterns specific to NATS JetStream (streams, cons **Related references**: For core subject hierarchy patterns see [patterns.md](patterns.md). For stream-level security and tenant isolation see [security.md](security.md). For common design mistakes see [anti-patterns.md](anti-patterns.md). +The configuration blocks in this reference are illustrative. Confirm field names, server support, and client behavior against the NATS server version and client library used by the deployment. + +## Core NATS and JetStream + +Core NATS delivers to subscribers that are connected at publish time and does not replay messages. JetStream stores messages in a stream and delivers them through stateful consumers with acknowledgements and redelivery. + +Use Core NATS for request-reply and best-effort live signals. Use JetStream for replayable events and durable work. If live observers also need an event, let them subscribe to the event subject while a JetStream consumer remains the correctness path. Do not silently switch a request subject between Core NATS and JetStream. + ## JetStream Subject Basics ### Streams vs Publishers @@ -39,7 +47,7 @@ consumer: { ## Pattern 1: Simple Domain Stream -**Use when**: Single domain, simple event types, single consumer. +**Use when**: One retention and ownership policy covers the domain and its consumers. **Stream Definition**: ```nats @@ -76,7 +84,7 @@ consumer: { # Consumer 3: Regional (US West) consumer: { name: "us-west-orders" - filter_subject: "orders.>.us-west.>" # All order actions in US West + filter_subject: "orders.*.us-west.>" # All order actions in US West } ``` @@ -84,7 +92,7 @@ consumer: { ## Pattern 2: Multi-Domain Streams (One per Domain) -**Use when**: Clear domain boundaries, separate scaling needs, independent retention. +**Use when**: Streams have separate retention, scaling, replication, ownership, or consumer topologies. **Stream Definitions**: ```nats @@ -326,29 +334,29 @@ stream: { stream: { name: "orders-by-region" # Derived stream - sources: [{ - name: "orders-raw" - }] + subjects: ["orders-by-region.>"] # Disjoint from orders.> } consumer: { name: "transform-by-region" filter_subject: "orders.created.>" # Source - deliver_subject: "orders.region-{region}.created.>" # Transform target + deliver_subject: "transform.orders.input" # Concrete push destination outside orders.> } ``` **Transform Flow**: ``` Input: orders.created.us-west.order-123 - ↓ (Consumer with subject transform) -Output: orders.region-us-west.created.order-123 + ↓ consumer delivers to transform.orders.input +Transform service publishes: orders-by-region.us-west.created.order-123 Input: orders.created.eu-central.order-456 - ↓ -Output: orders.region-eu-central.created.order-456 + ↓ consumer delivers to transform.orders.input +Transform service publishes: orders-by-region.eu-central.created.order-456 ``` +A push consumer delivery subject is a concrete subject. Wildcards and `{region}` placeholders do not transform a delivery subject. Use a server-supported stream subject transform for the target server version or perform the transform in a service that publishes the derived subject. Keep the delivery subject and the derived subjects outside the source stream filter. A delivery subject inside the filter forms a cycle, and a derived subject inside the filter is stored again in the raw stream. + **Use Cases**: - Normalize subject hierarchies - Fan-out to multiple derived streams @@ -456,7 +464,7 @@ stream: { ## High-Volume Subject Design (IoT) -**Challenge**: Device telemetry generates 1M+ events/sec with high cardinality device IDs. +**Challenge**: Device telemetry generates high event volume with high-cardinality device IDs. **Subject Design**: ``` @@ -487,7 +495,7 @@ stream: { # Consumer 1: Temperature metrics only consumer: { name: "temp-aggregator" - filter_subject: "devices.telemetry.us-west.>.temperature" + filter_subject: "devices.telemetry.us-west.*.temperature" } # Consumer 2: Specific device @@ -534,7 +542,7 @@ orders.created.order-1 orders.updated.order-1 ← Same order-id, maintains ordering orders.shipped.order-1 -Consumer: filter_subject: "orders.>order-1" +Consumer: filter_subject: "orders.*.order-1" Reads in insertion order: created → updated → shipped ``` @@ -558,7 +566,7 @@ Payload: { orderId: "order-123", total: 99.99, ... } - Subject stays readable (not bloated) - Headers for filtering/metadata - Payload for business data -- Supports header-based filtering in consumers (future) +- Consumer filters match subjects. Inspect message headers in application code when routing on metadata. --- @@ -571,10 +579,11 @@ When designing JetStream subjects: - [ ] **Retention Policy**: max_age set appropriately per domain? - [ ] **Storage Strategy**: File vs memory decided per workload? - [ ] **Ordering Semantics**: Aggregate ID in subject for ordered consumers? -- [ ] **Headers Used**: Heavy data in headers, not subject? +- [ ] **Headers Used**: Metadata is in headers or payload instead of an unbounded subject? - [ ] **Mirroring**: Cross-region streams planned? -- [ ] **Scaling Plan**: Stream per region/tenant/domain as needed? -- [ ] **Backpressure**: flow_control enabled for slow consumers? +- [ ] **Scaling Plan**: Stream boundaries follow retention, ownership, and consumer topology? +- [ ] **Consumer Semantics**: Pull or push, ack policy, redelivery, backoff, and max pending are explicit? +- [ ] **Backpressure**: Pull limits or flow control are configured for slow consumers? - [ ] **Monitoring**: Subjects planned for stream/consumer metrics? --- diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/nack-jetstream.yaml b/plugins/trogonstack-nats/skills/nats-design-subject/references/nack-jetstream.yaml new file mode 100644 index 0000000..8256466 --- /dev/null +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/nack-jetstream.yaml @@ -0,0 +1,60 @@ +# NACK JetStream resources. +# These resources assume the NACK JetStream controller is installed in the cluster. +# Configure NACK's NATS connection and mTLS through its Helm values and Kubernetes Secrets. +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: github-primary + namespace: nats +spec: + name: GITHUB_PRIMARY + subjects: + - gateway.ingress.raw.github.v1.> + retention: limits + storage: file + replicas: 3 + maxAge: 168h +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: slack-primary + namespace: nats +spec: + name: SLACK_PRIMARY + subjects: + - gateway.ingress.raw.slack.v1.> + retention: limits + storage: file + replicas: 3 + maxAge: 168h +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: gateway-work + namespace: nats +spec: + name: GATEWAY_WORK + subjects: + - gateway.work.v1.installation_sync + retention: workqueue + storage: file + replicas: 3 + maxConsumers: 1 +--- +# Optional. Apply this only when the installation-sync worker is deployed. +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: installation-sync + namespace: nats +spec: + streamName: GATEWAY_WORK + durableName: installation-sync + filterSubject: gateway.work.v1.installation_sync + deliverPolicy: all + ackPolicy: explicit + ackWait: 30s + maxDeliver: 5 diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/patterns.md b/plugins/trogonstack-nats/skills/nats-design-subject/references/patterns.md index 4798f08..d7e2a00 100644 --- a/plugins/trogonstack-nats/skills/nats-design-subject/references/patterns.md +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/patterns.md @@ -1,6 +1,6 @@ # NATS Subject Hierarchy Patterns -This reference covers 6 proven subject hierarchy patterns and 5 segmentation strategies for different use cases. +This reference gives reusable subject hierarchy patterns and segmentation strategies for common NATS designs. **Related references**: For authorization on these patterns see [security.md](security.md). For JetStream stream design see [jetstream.md](jetstream.md). For common mistakes see [anti-patterns.md](anti-patterns.md). @@ -27,7 +27,7 @@ inventory.> # All inventory events **Best For**: E-commerce backend, simple microservices, IoT with single location. -**Scaling**: Works for 10k-100k events/sec with moderate subscriber count. Add region/tenant at layer 3 when scaling. +**Scaling**: Start with one domain subject and add region or tenant only when a subscriber or authorization boundary requires it. Benchmark throughput and subscriber counts for the target deployment. --- @@ -50,14 +50,14 @@ Examples: orders.> # All orders (any region) orders.created.> # All order creations (any region) orders.created.us-west.> # Order creations in US West -orders.>.us-west.> # All order actions in US West (less efficient) +orders.*.us-west.> # All order actions in US West ``` **Best For**: Global e-commerce, distributed IoT networks, compliance-required data residency. **Patterns**: -- Region as L3: Efficient regional filtering -- Region as L4: Less efficient, use L3 for regional subscriptions +- Region as L3: Makes the regional filter direct +- Region as L4: Requires a one-token wildcard before the region --- @@ -128,14 +128,14 @@ acme-corp.orders.created.order-123 ``` Request Subject: {service}.request.{request-type} -Reply Subject: {service}.reply.{correlation-id} +Reply-To: _INBOX. Examples: orders.request.get-order orders.request.calculate-total payments.request.authorize -Reply inbox: _INBOX.{auto-generated-id} +The client creates a unique `_INBOX.` reply subject and sends it as the request reply-to. ``` **Implementation**: @@ -228,7 +228,7 @@ devices.telemetry.2026-01-24.sensor-456 devices.telemetry.2026-01-23.sensor-456 Subscriber: devices.telemetry.2026-01-24.> (current day only) -Subscriber: devices.telemetry.>.> (all days) +Subscriber: devices.telemetry.*.> (all days) ``` **Best For**: API evolution, device telemetry, multi-version deployments. @@ -298,8 +298,8 @@ Organize by geographic region: **When**: Global systems, data residency requirements, compliance. **Subscribers**: -- `orders.>.us-west.>` - Orders in US West -- `orders.>.eu-central.>` - Orders in EU Central +- `orders.*.us-west.>` - Orders in US West +- `orders.*.eu-central.>` - Orders in EU Central - `orders.>` - All orders globally **Use Case**: GDPR compliance (EU data stays in EU). @@ -327,7 +327,8 @@ sensors.humidity.us-west.sensor-456 **Subscribers**: - `sensors.temperature.us-west.>` - All temperature in US West -- `sensors.>.us-west.sensor-456` - All metrics from specific sensor +- `sensors.*.us-west.sensor-456` - All metrics from specific sensor (four-token form) +- `sensors.*.us-west.sensor-456.*` - All metrics from specific sensor (timestamped form) - `sensors.>` - All sensor data **JetStream Streams**: @@ -338,7 +339,7 @@ Subject: sensors.> Consumers filter by: - sensors.temperature.> (temperature only) - sensors.humidity.> (humidity only) -- sensors.>.us-west.> (region only) +- sensors.*.us-west.> (region only) ``` --- @@ -391,8 +392,8 @@ Order Service: orders.> Payment Service: payments.> Inventory Service: inventory.> Shipping Service: shipping.> -Regional Dashboard (US West): >.us-west.> -Regional Dashboard (EU): >.eu-central.> +Regional Dashboard (US West): *.*.us-west.> +Regional Dashboard (EU): *.*.eu-central.> ``` --- @@ -434,9 +435,9 @@ devices.telemetry.ap-south.device-789.current **Subscribers**: ``` -Temperature Monitor: devices.telemetry.>.>.temperature +Temperature Monitor: devices.telemetry.*.*.temperature US East Aggregator: devices.telemetry.us-east.> -Device 456 Dashboard: devices.telemetry.>.sensor-456.> +Device 456 Dashboard: devices.telemetry.*.sensor-456.> All Telemetry: devices.telemetry.> ``` @@ -446,13 +447,13 @@ Stream: iot-telemetry Subject: devices.telemetry.> Consumer: temperature-only -Filter: devices.telemetry.>.>.temperature +Filter: devices.telemetry.*.*.temperature Consumer: us-east-only Filter: devices.telemetry.us-east.> Consumer: sensor-456 -Filter: devices.telemetry.>.sensor-456.> +Filter: devices.telemetry.*.sensor-456.> ``` --- @@ -491,11 +492,11 @@ LLM agent capability discovery: agents.capabilities.llm-agent ## Comparison Matrix -| Pattern | Best For | Complexity | Scalability | Auth Support | -|---------|----------|-----------|-------------|--------------| -| Simple | Learning, single domain | Low | 100k events/sec | Good | -| Multi-Region | Global systems | Medium | 1M+ events/sec | Good | -| Multi-Tenant | SaaS, account isolation | Medium-High | 1M+ events/sec | Excellent | -| Request/Reply | Sync services | Low | 10k req/sec | Good | -| Event Sourcing | CQRS, event-sourced | Medium | 100k events/sec | Good | -| Temporal | IoT, time-series | High | 10M+ events/sec | Good | +| Pattern | Best For | Complexity | Scaling decision | Auth Support | +|---------|----------|-------------|------------------|--------------| +| Simple | Learning, single domain | Low | Add scope only when needed | Good | +| Multi-Region | Global systems | Medium | Split by region when ownership or retention differs | Good | +| Multi-Tenant | SaaS, account isolation | Medium-High | Prefer account boundaries before subject prefixes | Excellent | +| Request/Reply | Sync services | Low | Size from request latency and responder capacity | Good | +| Event Sourcing | CQRS, event-sourced | Medium | Size from retention and replay needs | Good | +| Temporal | IoT, time-series | High | Partition by retention and consumer topology | Good | diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/security.md b/plugins/trogonstack-nats/skills/nats-design-subject/references/security.md index 4bd2296..daeb9b3 100644 --- a/plugins/trogonstack-nats/skills/nats-design-subject/references/security.md +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/security.md @@ -18,6 +18,7 @@ Use tenant prefixes in subjects only when: ## Subject-Based Authorization Basics NATS authorization works by allowing/denying subjects via permissions. Your subject hierarchy directly enables or blocks access. +Keep `$SYS`, `$JS`, `$KV`, `$O`, `$SRV`, and `_INBOX` under explicit control. A broad `>` permission includes JetStream control subjects and can grant more authority than the application needs. ### Permission Types @@ -34,9 +35,8 @@ permissions: ### Wildcard Rules -- `>` matches any number of segments (multi-level) +- `>` matches one or more trailing tokens and must be the final token - `*` matches exactly one segment -- Applied left-to-right ```nats Subject: orders.created.us-west.order-123 @@ -46,7 +46,7 @@ Matches: ✓ orders.created.> (any created order) ✓ orders.created.us-west.> (any created in region) ✓ orders.*.us-west.> (any action in region) -✗ orders.*.> (order action, but only one level) +✓ orders.*.> (any action and its remaining tokens) ``` --- @@ -76,7 +76,7 @@ payments.authorized.eu-central.payment-101 ```nats # ACME Corp user in account acme-corp user acme-admin { - username: "admin@acme-corp.com" + username: acme-admin permissions { publish { allow: ["orders.>", "payments.>"] @@ -89,7 +89,7 @@ user acme-admin { # StartUp Inc user in account startup-inc user startup-admin { - username: "admin@startup-inc.com" + username: startup-admin permissions { publish { allow: ["orders.>", "payments.>"] @@ -294,7 +294,7 @@ user system-service { **Effectiveness**: ✓ Excellent. Clean separation of concerns. **Pros**: -- Tenant subjects remain simple (4-5 segments) +- Tenant-local subjects remain short and follow the project segment convention - Admin operations completely separate - Platform observability isolated - Tenant accounts do not need broad platform-admin credentials diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/subject-fixtures.yaml b/plugins/trogonstack-nats/skills/nats-design-subject/references/subject-fixtures.yaml new file mode 100644 index 0000000..155cf15 --- /dev/null +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/subject-fixtures.yaml @@ -0,0 +1,24 @@ +- kind: publish + subjects: + - gateway.ingress.raw.github.v1.pull_request +- kind: subscribe + subjects: + - gateway.ingress.raw.github.v1.> +- kind: stream + subjects: + - gateway.ingress.raw.github.v1.> +- kind: consumer + subjects: + - gateway.ingress.raw.github.v1.pull_request +- kind: publish + subjects: + - gateway.ingress.raw.slack.v1.event_callback +- kind: subscribe + subjects: + - gateway.ingress.raw.slack.v1.> +- kind: publish + subjects: + - gateway.rpc.v1.health +- kind: deliver + subjects: + - _INBOX.generated diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/references/use-cases.md b/plugins/trogonstack-nats/skills/nats-design-subject/references/use-cases.md index 2f11c15..724a2cf 100644 --- a/plugins/trogonstack-nats/skills/nats-design-subject/references/use-cases.md +++ b/plugins/trogonstack-nats/skills/nats-design-subject/references/use-cases.md @@ -62,7 +62,7 @@ Shipping Service subscribes to: Order Dashboard subscribes to: - orders.> (all order events, any region) -- orders.>.us-west.> (regional dashboard) +- orders.*.us-west.> (regional dashboard) ``` ### Multi-Region Deployment @@ -127,10 +127,10 @@ Temperature Alerting: - devices.temperature.> (all temperatures, any region) Regional Aggregator: -- devices.>.us-west.> (all metrics in region) +- devices.*.us-west.> (all metrics in region) Specific Sensor Debug: -- devices.>.us-west.sensor-456 (all metrics from one sensor) +- devices.*.us-west.sensor-456 (all metrics from one sensor) ``` ### Aggregation Workflow @@ -233,7 +233,7 @@ Account: {tenant} Subject: {domain}.{action}.{region}.{id} EU customers: events.*.eu-central.> in the tenant account -US customers: events.*.us-*.> in the tenant account +US East customers: events.*.us-east.> in the tenant account ``` For authorization configuration see [security.md](security.md) Pattern 1 (Tenant Isolation with NATS Accounts) and Pattern 4 (Cross-Tenant Analytics). For account-scoped stream setup see [jetstream.md](jetstream.md) Pattern 3. @@ -412,10 +412,10 @@ For tenant isolation authorization see [security.md](security.md) Pattern 5 (AI ## Comparison Matrix -| Use Case | Subject Depth | Domain Count | Subscriber Count | Throughput | Retention | +| Use Case | Subject Depth | Domain Count | Subscriber Count | Sizing input | Retention | |----------|---|---|---|---|---| -| Microservices | 4 | 3-10 | 50-100 | 10k-100k/sec | 30d-7y | -| IoT | 4 | 1 (many dimensions) | 20-50 | 1M+/sec | 1-7 days | -| Multi-Tenant SaaS | 4 | 3-8 | 50-200 | 100k-1M/sec | 30-90d | -| Event Sourcing | 4 | 5-20 | 100+ | 10k-100k/sec | forever | -| Agentic AI | 4 | 2-3 | 100-1000+ | 10k-100k/sec | 30-365d | +| Microservices | 4 | 3-10 | 50-100 | Request rate and latency | 30d-7y | +| IoT | 4 | 1 (many dimensions) | 20-50 | Device rate and payload size | 1-7 days | +| Multi-Tenant SaaS | 4 | 3-8 | 50-200 | Tenant and consumer load | 30-90d | +| Event Sourcing | 4 | 5-20 | 100+ | Replay and retention load | forever | +| Agentic AI | 4 | 2-3 | 100-1000+ | Task rate and worker concurrency | 30-365d | diff --git a/plugins/trogonstack-nats/skills/nats-design-subject/scripts/validate-subjects.mjs b/plugins/trogonstack-nats/skills/nats-design-subject/scripts/validate-subjects.mjs new file mode 100644 index 0000000..8cf5272 --- /dev/null +++ b/plugins/trogonstack-nats/skills/nats-design-subject/scripts/validate-subjects.mjs @@ -0,0 +1,264 @@ +#!/usr/bin/env node + +import { readFile, readdir, stat } from "node:fs/promises"; +import { join } from "node:path"; + +const markdownTag = "nats-subjects"; +const kinds = new Set(["publish", "subscribe", "stream", "consumer", "deliver"]); +const usage = `Usage: + node validate-subjects.mjs --subject [--kind publish|subscribe|stream|consumer|deliver] + node validate-subjects.mjs --file + node validate-subjects.mjs --root + +A YAML fixture file contains kind records with a subjects list. Legacy line fixtures remain supported. +Markdown files contain YAML-like records in fenced blocks tagged with ${markdownTag}: + +\`\`\`yaml ${markdownTag} +- kind: publish + subjects: + - events.created +\`\`\` + +Blank lines and lines beginning with # are ignored.`; + +function fail(message) { + console.error(`error: ${message}`); + process.exitCode = 1; +} + +function validate(subject, kind, location) { + const label = location ? `${location}: ` : ""; + if (!kinds.has(kind)) { + fail(`${label}unknown subject kind ${kind}`); + return; + } + if (!subject) { + fail(`${label}subject is empty`); + return; + } + if (subject.length > 255) { + fail(`${label}subject exceeds the recommended 255-character limit`); + } + if (/\s|\0/.test(subject)) { + fail(`${label}subject contains whitespace or a NUL character`); + } + const tokens = subject.split("."); + if (tokens.some((token) => token.length === 0)) { + fail(`${label}subject contains an empty token`); + } + for (const [index, token] of tokens.entries()) { + if (token.includes("*") && token !== "*") { + fail(`${label}* must be a complete token`); + } + if (token.includes(">") && token !== ">") { + fail(`${label}> must be a complete token`); + } + if (token === ">" && index !== tokens.length - 1) { + fail(`${label}> must be the final token`); + } + } + if ((kind === "publish" || kind === "deliver") && tokens.some((token) => token === "*" || token === ">")) { + fail(`${label}${kind} subjects must be concrete and cannot contain wildcards`); + } + if (kind === "deliver" && subject.startsWith("$")) { + fail(`${label}delivery subjects must not use a reserved server prefix`); + } +} + +function unquote(value) { + const trimmed = value.trim(); + if (trimmed.length >= 2) { + const first = trimmed[0]; + const last = trimmed.at(-1); + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return trimmed.slice(1, -1); + } + } + return trimmed; +} + +function parseLegacyFixture(content, file) { + let entries = 0; + for (const [index, line] of content.split(/\r?\n/).entries()) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) { + continue; + } + const separator = trimmed.indexOf(" "); + if (separator < 1) { + fail(`${file}:${index + 1}: expected ' '`); + continue; + } + const kind = trimmed.slice(0, separator); + const subject = trimmed.slice(separator + 1).trim(); + validate(subject, kind, `${file}:${index + 1}`); + entries += 1; + } + return entries; +} + +function parseTaggedBlock(lines, file, openingLine) { + let currentKind; + let awaitingSubjects = false; + let readingSubjects = false; + let entries = 0; + + for (const [index, line] of lines.entries()) { + const location = `${file}:${openingLine + index + 1}`; + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) { + continue; + } + + const kindMatch = trimmed.match(/^(?:-\s*)?kind:\s*([A-Za-z-]+)\s*$/); + if (kindMatch) { + if (awaitingSubjects) { + fail(`${location}: previous kind has no subjects`); + } + currentKind = kindMatch[1]; + awaitingSubjects = true; + readingSubjects = false; + continue; + } + + if (trimmed === "subjects:") { + if (!currentKind) { + fail(`${location}: subjects must follow a kind`); + } else { + readingSubjects = true; + } + continue; + } + + const subjectMatch = trimmed.match(/^[-]\s+(.+)$/); + if (subjectMatch && readingSubjects) { + validate(unquote(subjectMatch[1]), currentKind, location); + entries += 1; + awaitingSubjects = false; + continue; + } + + fail(`${location}: expected 'kind: ', 'subjects:', or a subject list item`); + } + + if (awaitingSubjects) { + fail(`${file}:${openingLine}: final kind has no subjects`); + } + if (entries === 0) { + fail(`${file}:${openingLine}: tagged block has no subject entries`); + } + return entries; +} + +function parseMarkdown(content, file) { + const lines = content.split(/\r?\n/); + let inFence = false; + let fenceMarker = ""; + let info = ""; + let openingLine = 0; + let body = []; + let entries = 0; + + for (const [index, line] of lines.entries()) { + const fence = line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/); + if (!inFence && fence) { + inFence = true; + fenceMarker = fence[1]; + info = fence[2].trim(); + openingLine = index + 1; + body = []; + continue; + } + const closing = line.match(/^ {0,3}(`{3,}|~{3,})[ \t]*$/); + if (inFence && closing && closing[1][0] === fenceMarker[0] && closing[1].length >= fenceMarker.length) { + if (info.split(/\s+/).includes(markdownTag)) { + entries += parseTaggedBlock(body, file, openingLine); + } + inFence = false; + fenceMarker = ""; + info = ""; + body = []; + continue; + } + if (inFence) { + body.push(line); + } + } + + if (inFence) { + fail(`${file}:${openingLine}: fenced block is not closed`); + } + return entries; +} + +async function markdownFiles(root) { + const details = await stat(root); + if (details.isFile()) { + return root.endsWith(".md") ? [root] : []; + } + const files = []; + for (const entry of await readdir(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory()) { + files.push(...await markdownFiles(path)); + } else if (entry.isFile() && entry.name.endsWith(".md")) { + files.push(path); + } + } + return files; +} + +const args = process.argv.slice(2); +let file; +let root; +let subject; +let kind = "subscribe"; +for (let index = 0; index < args.length; index += 1) { + const argument = args[index]; + if (argument === "--file") { + file = args[++index]; + } else if (argument === "--root") { + root = args[++index]; + } else if (argument === "--subject") { + subject = args[++index]; + } else if (argument === "--kind") { + kind = args[++index]; + } else if (argument === "--help" || argument === "-h") { + console.log(usage); + process.exit(0); + } else { + fail(`unknown argument ${argument}`); + console.error(usage); + process.exit(1); + } +} + +if (!file && !root && !subject) { + console.error(usage); + process.exit(1); +} + +let entries = 0; +if (subject) { + validate(subject, kind, "command line"); + entries = 1; +} else if (file) { + const content = await readFile(file, "utf8"); + entries = file.endsWith(".md") + ? parseMarkdown(content, file) + : file.endsWith(".yaml") || file.endsWith(".yml") + ? parseTaggedBlock(content.split(/\r?\n/), file, 0) + : parseLegacyFixture(content, file); +} else { + for (const path of await markdownFiles(root)) { + entries += parseMarkdown(await readFile(path, "utf8"), path); + } +} + +if (entries === 0) { + fail(`no subject entries found${root ? ` under ${root}` : ""}`); +} +if (process.exitCode) { + process.exit(process.exitCode); +} +console.log(`subjects valid (${entries} entr${entries === 1 ? "y" : "ies"})`);