From 62c9d9721dc72d5d16a73461a2c7a4fa05aad825 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 07:19:30 +0200 Subject: [PATCH 01/16] fix: pass --effort through to the claude-cli backend --effort was parsed and then only ever reached the API path, so a `--backend claude-cli` run silently ignored it and used the CLI's own session default. The flag now maps onto `claude --effort`. --- zendesk_triage/triage.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index d9d423b..165b8d1 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -697,7 +697,7 @@ def tickets_from_payload(payload, source): return validate_findings(found, source) -def analyze_via_claude_cli(model, compact_tickets, timeout=1800): +def analyze_via_claude_cli(model, effort, compact_tickets, timeout=1800): """Classify the batch with the local `claude` CLI instead of the Anthropic API. Local debugging path: it authenticates as Claude Code, so no ANTHROPIC_API_KEY is @@ -710,6 +710,8 @@ def analyze_via_claude_cli(model, compact_tickets, timeout=1800): cmd = ["claude", "-p", "--output-format", "json"] if model: cmd += ["--model", model] + if effort: + cmd += ["--effort", effort] try: # Prompt goes over stdin: a full batch can exceed the argv size limit. proc = subprocess.run( @@ -1194,7 +1196,7 @@ def main(): return if args.backend == "claude-cli": - analyzer = partial(analyze_via_claude_cli, model) + analyzer = partial(analyze_via_claude_cli, model, args.effort) else: client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY analyzer = partial(analyze, client, model, args.effort) From c68433e4af70962fa49931e323a4a96a313bc6bd Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 07:23:38 +0200 Subject: [PATCH 02/16] feat: enforce the schema on the claude-cli backend with --json-schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI path used to describe the shape in prose and parse a JSON object back out of the model's reply, which is how platform, app_version and reported_session_id silently came back empty after being added to the schema. `claude --json-schema` takes the same SCHEMA the API path uses and returns it validated in the envelope's `structured_output`, so the hand-maintained field list (and the tests guarding it) can go. An envelope with no `structured_output` means the CLI ignored the flag (pre-v2.1.205) — that exits with the version to check rather than rendering an empty digest. --- zendesk_triage/test_triage.py | 53 +++++++++++++++--------- zendesk_triage/triage.py | 78 +++++++++++++---------------------- 2 files changed, 62 insertions(+), 69 deletions(-) diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index e2f880e..41357c4 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -439,23 +439,6 @@ def test_every_category_is_explained_in_the_system_prompt(self): missing = [c for c in triage.CATEGORIES if c not in triage.SYSTEM_PROMPT] self.assertEqual(missing, []) - def test_every_category_is_listed_in_the_cli_instructions(self): - missing = [c for c in triage.CATEGORIES if c not in triage.CLI_JSON_INSTRUCTIONS] - self.assertEqual(missing, []) - - def test_every_schema_field_is_listed_in_the_cli_instructions(self): - """The CLI backend has no structured-output enforcement, so a field absent - from these instructions comes back empty — which is how platform, app_version - and reported_session_id silently went unpopulated.""" - fields = triage.SCHEMA["properties"]["tickets"]["items"]["properties"] - missing = [f for f in fields if f not in triage.CLI_JSON_INSTRUCTIONS] - self.assertEqual(missing, []) - - def test_every_enum_value_is_listed_in_the_cli_instructions(self): - for values in (triage.CATEGORIES, triage.SEVERITIES, triage.PLATFORMS): - for value in values: - self.assertIn(value, triage.CLI_JSON_INSTRUCTIONS) - def test_schema_enum_matches_the_category_list(self): item = triage.SCHEMA["properties"]["tickets"]["items"] self.assertEqual(item["properties"]["category"]["enum"], triage.CATEGORIES) @@ -796,8 +779,8 @@ def test_an_empty_list_is_valid(self): self.assertEqual(triage.tickets_from_payload({"tickets": []}, "x"), []) def test_a_finding_missing_renderer_keys_exits(self): - """The claude-cli backend has no structured-output enforcement, so an entry - without category/severity would otherwise KeyError inside build_summary_embed.""" + """A hand-edited --backend file findings list has nothing enforcing its shape, + so an entry without category/severity would KeyError in build_summary_embed.""" for entry in ({"id": 1}, {"id": 1, "category": "bug_report"}, {"category": "bug_report", "severity": "major"}): with self.assertRaises(SystemExit): @@ -808,6 +791,38 @@ def test_a_non_object_entry_exits(self): triage.tickets_from_payload({"tickets": [["not", "an", "object"]]}, "x") +class TestFindingsFromCliEnvelope(unittest.TestCase): + def envelope(self, **overrides): + base = { + "subtype": "success", + "is_error": False, + "structured_output": {"tickets": [finding(1)]}, + } + base.update(overrides) + return base + + def test_reads_structured_output(self): + self.assertEqual(triage.findings_from_cli_envelope(self.envelope()), [finding(1)]) + + def test_missing_structured_output_exits(self): + """A CLI too old for --json-schema returns prose in `result` and no + structured_output; without this check the digest comes out silently empty.""" + stale = self.envelope(result='{"tickets": []}') + del stale["structured_output"] + with self.assertRaises(SystemExit): + triage.findings_from_cli_envelope(stale) + + def test_a_reported_cli_error_exits(self): + for envelope in (self.envelope(is_error=True), + self.envelope(subtype="error_max_turns")): + with self.assertRaises(SystemExit): + triage.findings_from_cli_envelope(envelope) + + def test_a_cost_field_is_reported_not_fatal(self): + result = triage.findings_from_cli_envelope(self.envelope(total_cost_usd=0.42)) + self.assertEqual(result, [finding(1)]) + + class TestExtractJsonObject(unittest.TestCase): def test_bare_object(self): self.assertEqual(triage.extract_json_object('{"a": 1}'), {"a": 1}) diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 165b8d1..e964543 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -617,40 +617,6 @@ def build_analysis_prompt(compact_tickets): ) -def _cli_field_lines(): - """Describe every schema field for the CLI path, derived from TICKET_PROPERTIES. - - The API path has structured outputs to enforce the shape; the CLI path only has - this text. Hardcoding the field list here is how three fields (platform, - app_version, reported_session_id) silently came back empty on the CLI backend - after being added to the schema. - """ - lines = [] - for name, spec in TICKET_PROPERTIES.items(): - shape = spec.get("type", "string") - if "enum" in spec: - shape += "; one of: " + ", ".join(spec["enum"]) - description = spec.get("description", "") - lines.append(f"- {name} ({shape}){': ' + description if description else ''}") - return "\n".join(lines) - - -# The API path gets the shape enforced by structured outputs. The CLI path has no -# such enforcement, so the shape is spelled out here — from the same schema. -CLI_JSON_INSTRUCTIONS = textwrap.dedent( - """ - Return ONLY a single JSON object — no prose, no explanation, no markdown code - fence. The object has exactly one key, "tickets", whose value is an array with - one object per input ticket, each with exactly these keys: - __FIELDS__ - - Every key is required on every object. Use an empty string for text fields you - cannot fill, and the enum's catch-all value ('unknown', 'not_applicable', 'other') - rather than inventing a new one. - """ -).strip().replace("__FIELDS__", _cli_field_lines()) - - def extract_json_object(text): """Pull the outermost JSON object out of model prose (tolerates code fences).""" start = text.find("{") @@ -697,17 +663,37 @@ def tickets_from_payload(payload, source): return validate_findings(found, source) +def findings_from_cli_envelope(envelope): + """Pull the findings out of a `claude -p --output-format json` envelope. + + With --json-schema the shape lands in `structured_output`, already validated + against SCHEMA. An envelope without that key means the CLI ignored the flag + (it predates v2.1.205), which would otherwise surface as an empty digest. + """ + if envelope.get("is_error") or envelope.get("subtype") != "success": + sys.exit(f"`claude` reported an error: {envelope.get('result') or envelope}") + cost = envelope.get("total_cost_usd") + if cost is not None: + print(f"claude CLI reported ${cost:.4f} for this batch.") + payload = envelope.get("structured_output") + if not isinstance(payload, dict): + sys.exit("`claude` returned no structured_output: --json-schema needs Claude " + "Code v2.1.205 or newer (check `claude --version`).") + return tickets_from_payload(payload, "`claude` CLI") + + def analyze_via_claude_cli(model, effort, compact_tickets, timeout=1800): """Classify the batch with the local `claude` CLI instead of the Anthropic API. - Local debugging path: it authenticates as Claude Code, so no ANTHROPIC_API_KEY is - needed. There is no structured-output enforcement here, so the response is parsed - defensively and the schema is described in the prompt. + It authenticates as Claude Code, so no ANTHROPIC_API_KEY is needed, and + --json-schema enforces the same SCHEMA the API path uses — so the response needs + no prose parsing and no hand-maintained field list in the prompt. """ - prompt = "\n\n".join( - [SYSTEM_PROMPT, CLI_JSON_INSTRUCTIONS, build_analysis_prompt(compact_tickets)] - ) - cmd = ["claude", "-p", "--output-format", "json"] + prompt = "\n\n".join([SYSTEM_PROMPT, build_analysis_prompt(compact_tickets)]) + # --bare would be the faster startup, but it reads ANTHROPIC_API_KEY only and + # never touches OAuth credentials, which is exactly what CI authenticates with. + cmd = ["claude", "-p", "--output-format", "json", + "--json-schema", json.dumps(SCHEMA)] if model: cmd += ["--model", model] if effort: @@ -724,14 +710,7 @@ def analyze_via_claude_cli(model, effort, compact_tickets, timeout=1800): if proc.returncode != 0: sys.exit(f"`claude` failed ({proc.returncode}): {proc.stderr[:500]}") - envelope = extract_json_object(proc.stdout) - if envelope.get("is_error") or envelope.get("subtype") != "success": - sys.exit(f"`claude` reported an error: {envelope.get('result') or envelope}") - cost = envelope.get("total_cost_usd") - if cost is not None: - print(f"claude CLI reported ${cost:.4f} for this batch.") - result = extract_json_object(envelope.get("result") or "") - return tickets_from_payload(result, "`claude` CLI") + return findings_from_cli_envelope(extract_json_object(proc.stdout)) def dump_batch(path, compact_tickets, model): @@ -739,7 +718,6 @@ def dump_batch(path, compact_tickets, model): payload = { "model": model, "system_prompt": SYSTEM_PROMPT, - "instructions": CLI_JSON_INSTRUCTIONS, "schema": SCHEMA, "tickets": compact_tickets, } From 680c9d4fb9bc19aa95bfff6e98006d710dfd81c9 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 07:39:06 +0200 Subject: [PATCH 03/16] feat: default to the `opus` alias instead of a pinned model id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI resolves an alias against whatever the authenticated plan allows, so a new Opus release needs no edit here and a plan without Opus access degrades instead of 404-ing on an id it can't serve. Documents the trade-off and when to pin a full id instead. Note: `--backend api` needs an explicit `--model claude-opus-4-8` until the follow-up commit removes that path — the Anthropic API takes ids, not Claude Code aliases. --- README.md | 4 +++- zendesk_triage/triage.py | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3570f70..bccf063 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ Two caveats worth knowing: | `--state` | flag | *(unset)* | Dedup state file. The workflow points this at the cached `.triage-state/seen.json` | | `--state-retention-days` | flag | `30` | Forget state entries older than N days | | `ZENDESK_QUERY` | env / `--query` | *(unset)* | Explicit Zendesk search query. Overrides `--window-hours` entirely | -| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `claude-opus-4-8` | Set to a cheaper model (e.g. `claude-haiku-4-5`) to reduce cost on large batches | +| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `opus` | Model alias (`opus`, `sonnet`, `haiku`) or a full id. Set it to `sonnet` to reduce cost on large batches | | `--max-tickets` | workflow input / flag | `1000` (workflow) / `100` (flag) | Runaway guard on tickets analyzed per run, **not** a batch size. The workflow passes `1000`; a bare `python triage.py` uses the script's own `DEFAULT_MAX_TICKETS` of `100`. Zendesk's search API caps a query at 1000 results, so higher values don't fetch more | | `--batch-size` | flag | `400` | Split batches larger than this across multiple requests | | `--review-star-floor` | flag | `3` | Classify app-store reviews at or below N stars; count the rest | @@ -150,6 +150,8 @@ Two caveats worth knowing: | `--no-hydrate` | flag | off | Skip fetching comments for content-free tickets | | `--effort` | flag | `medium` | Claude reasoning effort (`low`–`max`) | +> **Why an alias and not a pinned model id:** the alias is resolved at run time against whatever the authenticated plan allows, so a new Opus release needs no edit here, and a plan without Opus access falls back rather than failing on an id it can't serve. Pin a full id (`claude-opus-4-8`) only when you need a specific version — for reproducing a past run, say. + #### Batch size vs. ticket cap These do different jobs, and conflating them is how you get a silently truncated digest: diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index e964543..3082641 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -27,7 +27,7 @@ ANTHROPIC_API_KEY Claude API key (read by the SDK automatically) DISCORD_WEBHOOK_URL Discord incoming webhook ZENDESK_QUERY (optional) Zendesk search query; see DEFAULT_QUERY - ZENDESK_TRIAGE_MODEL (optional) Claude model id; defaults to claude-opus-4-8 + ZENDESK_TRIAGE_MODEL (optional) Claude model alias or id; defaults to `opus` Usage: # real run (CI): reads everything from the environment @@ -97,7 +97,10 @@ def window_label(hours): days = hours // 24 return f"created in the past {days} day{'s' if days > 1 else ''}" return f"created in the past {hours}h" -DEFAULT_MODEL = "claude-opus-4-8" +# An alias, not a pinned id: the CLI resolves `opus` to the newest Opus the +# authenticated plan allows, so a model release needs no edit here and a plan +# without Opus access degrades instead of 404-ing on a dead id. +DEFAULT_MODEL = "opus" DEFAULT_MAX_TICKETS = 100 DESCRIPTION_CHARS = 1500 # per-ticket description sent to Claude (triage only) # One classification runs ~100 output tokens per ticket, and adaptive thinking draws @@ -1041,7 +1044,8 @@ def main(): parser.add_argument("--window-hours", type=int, metavar="N", help="Only analyze unsolved tickets created in the last N hours. " "The scheduled daily run uses 48.") - parser.add_argument("--model", help="Claude model id (else ZENDESK_TRIAGE_MODEL, else claude-opus-4-8).") + parser.add_argument("--model", help="Claude model alias (opus, sonnet, haiku) or full " + "id (else ZENDESK_TRIAGE_MODEL, else opus).") parser.add_argument("--effort", default="medium", choices=["low", "medium", "high", "xhigh", "max"], help="Claude reasoning effort (default: medium).") parser.add_argument("--max-tickets", type=int, default=DEFAULT_MAX_TICKETS, From ae86e2d4514ca51682371afabe85599a6d17ab10 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 07:41:07 +0200 Subject: [PATCH 04/16] feat: run the scheduled triage through Claude Code instead of the API Installs Claude Code on the runner (stable channel, version echoed into the log) and authenticates with a CLAUDE_CODE_OAUTH_TOKEN secret from `claude setup-token`, so the job runs on a Claude subscription rather than API billing. ANTHROPIC_API_KEY is dropped from the step rather than left alongside: an API key outranks the OAuth token in Claude Code's credential precedence, and in -p mode a present key is always used, so keeping it would silently bill the API. DISABLE_AUTOUPDATER pins the run to the version the log reports. Documents the token's one-year life, that runs draw on one person's subscription limits, and why --bare must not be added (it reads ANTHROPIC_API_KEY only and never OAuth credentials). --- .github/workflows/zendesk_triage.yml | 23 ++++++++++++++++++++++- README.md | 28 ++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.github/workflows/zendesk_triage.yml b/.github/workflows/zendesk_triage.yml index eccd46b..fe50b9a 100644 --- a/.github/workflows/zendesk_triage.yml +++ b/.github/workflows/zendesk_triage.yml @@ -55,6 +55,20 @@ jobs: - name: Install dependencies run: pip install -r zendesk_triage/requirements.txt + # Classification runs through Claude Code, not the Anthropic API, so the job + # authenticates with a subscription OAuth token instead of an API key. The + # `stable` channel is ~a week behind `latest` and skips releases with known + # major regressions, which is what a scheduled job wants; the version is + # echoed because a CLI older than v2.1.205 ignores --json-schema and the + # script's error for that names this line. + - name: Install Claude Code + run: | + curl -fsSL https://claude.ai/install.sh | bash -s stable + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Report Claude Code version + run: claude --version + # Unique key so every run writes a fresh entry; the restore-keys prefix pulls # in the most recent previous one. run_attempt is in the key because cache # entries are immutable: a re-run reuses run_id, so without it attempt 2's save @@ -97,18 +111,25 @@ jobs: fi echo "Window: ${window}h, max tickets: ${max}" + # ANTHROPIC_API_KEY is deliberately absent: it outranks the OAuth token in + # Claude Code's credential precedence, and in -p mode a key that is present is + # always used — so setting it here would silently bill the API instead. + # DISABLE_AUTOUPDATER keeps the CLI from downloading a new version mid-run, + # so the version echoed above is the one that actually classifies. - name: Run triage env: ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }} ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }} ZENDESK_API_TOKEN: ${{ secrets.ZENDESK_API_TOKEN }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + DISABLE_AUTOUPDATER: "1" DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} ZENDESK_QUERY: ${{ github.event.inputs.query }} ZENDESK_TRIAGE_MODEL: ${{ vars.ZENDESK_TRIAGE_MODEL }} run: | mkdir -p .triage-state python zendesk_triage/triage.py \ + --backend claude-cli \ --window-hours "${{ steps.cfg.outputs.window }}" \ --max-tickets "${{ steps.cfg.outputs.max }}" \ --state .triage-state/seen.json diff --git a/README.md b/README.md index bccf063..04c7706 100644 --- a/README.md +++ b/README.md @@ -131,9 +131,26 @@ Two caveats worth knowing: | `ZENDESK_SUBDOMAIN` | Zendesk subdomain (`mycompany` → `mycompany.zendesk.com`) | | `ZENDESK_EMAIL` | Agent email used for Zendesk API-token auth | | `ZENDESK_API_TOKEN` | Zendesk API token | -| `ANTHROPIC_API_KEY` | Claude API key | +| `CLAUDE_CODE_OAUTH_TOKEN` | Claude subscription OAuth token — see [Claude authentication](#claude-authentication) | | `DISCORD_WEBHOOK_URL` | Discord webhook (reused from the failure-notification setup) | +### Claude Authentication + +Classification runs through Claude Code (`--backend claude-cli`), not the Anthropic API, so the job authenticates with a **subscription OAuth token** rather than an API key. Generate one on a machine where you're logged into Claude Code: + +``` +claude setup-token +``` + +It runs the browser authorization flow and prints the token once — it is not saved anywhere. Store it as the `CLAUDE_CODE_OAUTH_TOKEN` repo secret. Requires a Pro, Max, Team, or Enterprise plan; see [Generate a long-lived token](https://code.claude.com/docs/en/authentication#generate-a-long-lived-token). + +Four things worth knowing before you rely on it: + +- **The token lasts one year.** It expires silently from the workflow's point of view — the run just fails to authenticate. Put the renewal date somewhere you'll see it. +- **Never add `ANTHROPIC_API_KEY` to that step.** An API key [outranks the OAuth token](https://code.claude.com/docs/en/authentication#authentication-precedence) in Claude Code's credential precedence, and in `-p` mode a key that is present is always used — so the job would quietly bill the API instead of the subscription. +- **Runs draw on that subscription's usage limits**, not API credits, and the token is tied to whoever minted it. A scheduled run competes with that person's own interactive Claude Code usage, and hitting a limit fails the run (the tickets stay eligible and get picked up by the next one, per the dedup rules above). Anthropic's own guidance is to use an API key for a secret shared across an org for exactly this reason. +- **The CLI must be v2.1.205 or newer** for `--json-schema`. The workflow installs the `stable` channel and echoes `claude --version` into the run log, because the script's "no structured_output" error points here. + ### Optional Configuration | Setting | Where | Default | Description | @@ -219,10 +236,10 @@ export ZENDESK_SUBDOMAIN=... ZENDESK_EMAIL=... ZENDESK_API_TOKEN=... ANTHROPIC_A python zendesk_triage/triage.py --window-hours 48 --dry-run ``` -No `ANTHROPIC_API_KEY`? Two debug backends skip the Anthropic API entirely: +Locally, `--backend claude-cli` needs no token at all — it reuses your own Claude Code login: ``` -# classify via the local `claude` CLI (authenticates as Claude Code) +# what CI runs: classify through Claude Code python zendesk_triage/triage.py --backend claude-cli --window-hours 48 --dry-run # or dump the batch, classify it by hand, and feed the findings back @@ -230,7 +247,10 @@ python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 python zendesk_triage/triage.py --backend file --findings /tmp/findings.json --dry-run ``` -The `claude-cli` backend has no structured-output enforcement, so its field values are looser than the API path's (e.g. `"en"` where the schema asks for `"English"`), and each invocation carries ~25K tokens of Claude Code system-prompt overhead. Use it for debugging, not for scheduled runs. +Two properties of the `claude-cli` backend to keep in mind: + +- Each invocation carries ~25K tokens of Claude Code system-prompt overhead on top of the batch, so the per-run cost is dominated by that on small windows. It's one invocation per chunk, not per ticket — a typical 48h window is a single call. +- **Do not add `--bare`.** It's otherwise the right flag for a scripted call (it skips hook, plugin, MCP and `CLAUDE.md` discovery, so the runner behaves the same as your laptop), but bare mode reads `ANTHROPIC_API_KEY` or an `apiKeyHelper` **only** — it never touches OAuth credentials, which is exactly what both CI and your local login use. See [bare mode](https://code.claude.com/docs/en/headless#start-faster-with-bare-mode); the docs say it will become the default for `-p` in a future release, so this is worth re-checking on CLI upgrades. ## Workflow Failure Notificaiton From 2d7c19dae5b7c59fe3c786e2f170bd197eb4648c Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 07:44:54 +0200 Subject: [PATCH 05/16] refactor: drop the Anthropic API backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing uses it now that the scheduled run goes through Claude Code, and keeping it meant maintaining two classification paths plus a dependency the job never loads. Removes analyze(), the anthropic import and pin, and the `api` backend choice — `claude-cli` becomes the default, leaving `file` for rendering findings classified elsewhere. The one behaviour that went with it was the explicit max-tokens message on a truncated batch, since only the API path could read stop_reason. A truncated reply now closes no JSON and yields no structured_output, so that error names both possible causes and the --batch-size to lower. --- README.md | 21 ++++----- zendesk_triage/requirements.txt | 1 - zendesk_triage/triage.py | 79 ++++++++++----------------------- 3 files changed, 31 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 04c7706..5aba25e 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Runs automatically every Monday at 00:00 UTC. ## Zendesk Ticket Triage -Claude reviews recently-created unsolved Zendesk tickets via the API and posts a summary to Discord that links back to each original ticket and highlights the ones worth looking into. For each ticket it assigns a category, infers severity, guesses a likely root cause, identifies platform and app version, groups likely duplicates into clusters, and ranks by priority. +Claude reviews recently-created unsolved Zendesk tickets fetched from the Zendesk API and posts a summary to Discord that links back to each original ticket and highlights the ones worth looking into. For each ticket it assigns a category, infers severity, guesses a likely root cause, identifies platform and app version, groups likely duplicates into clusters, and ranks by priority. ### Categories @@ -93,7 +93,7 @@ Detection uses the Zendesk `via.channel`, which identified reviews with no false Twitter DM tickets arrive with `description` identical to `subject` — both just `"Conversation with "` — which is 15% of non-review tickets and unclassifiable as fetched. For those only, `hydrate_descriptions` fetches a page of up to 10 comments and joins every body that differs from the subject into the description; later replies often carry the actual detail. Hydration is an enrichment, so an HTTP error or an unreachable endpoint leaves the ticket as-is rather than failing the run (`--no-hydrate` to skip it entirely). -The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time window, sends the whole batch to Claude in one structured-output request, and posts Discord embeds: a summary embed plus one embed per highlighted ticket (linking to the ticket in Zendesk). +The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time window, classifies the whole batch in one schema-enforced request through the `claude` CLI, and posts Discord embeds: a summary embed plus one embed per highlighted ticket (linking to the ticket in Zendesk). The summary embed accounts for the batch in full, so nothing is dropped silently: @@ -176,7 +176,7 @@ These do different jobs, and conflating them is how you get a silently truncated - **`--max-tickets`** bounds how much of the Zendesk result set is fetched. At the workflow's 1000 it never binds on a 48h window (~45 tickets); it exists so a spam flood or a wide `reset_state` backfill can't run away. 1000 is also [Zendesk's own search result limit](https://developer.zendesk.com/api-reference/ticketing/ticket-management/search/#results-limit) — the API returns `422` for any page past it, so the fetch stops at 1000 regardless of what you pass, and reports the matched-vs-analyzed gap rather than failing. - **`--batch-size`** bounds how many tickets go into a *single* model request. Anything larger is split across requests and the findings are concatenated. -The split is necessary because output tokens, not context, are the binding constraint. Measured on real tickets: **~118 input tokens and ~102 output tokens per ticket**, with adaptive thinking drawing from the same `max_tokens` budget. +The split is necessary because output tokens, not context, are the binding constraint. Measured on real tickets: **~118 input tokens and ~102 output tokens per ticket**, with adaptive thinking drawing from the same output budget. | Batch | Input | Output needed | Fits in one request? | | ----- | ----- | ------------- | -------------------- | @@ -184,7 +184,7 @@ The split is necessary because output tokens, not context, are the binding const | 400 (`--batch-size`) | ~47K | ~41K | Yes, with room for thinking | | 1000 (`--max-tickets`) | ~118K | ~102K | **No** — leaves only ~26K of the 128K output ceiling for thinking | -If a single request ever does hit the ceiling, the script exits with that explicit reason rather than failing on an incomplete-JSON parse error. +If a single request ever does hit the ceiling, the JSON never closes and no `structured_output` comes back — the script exits naming that and the `--batch-size` to lower, rather than rendering a digest that is silently short. > Chunking is per-request, so `cluster` labels and `priority_rank` are only meaningful within a chunk. Batches large enough to split are ones where completing at all matters more than cross-chunk cluster fidelity. @@ -228,19 +228,14 @@ Offline tests covering the window arithmetic, dedup partitioning, state round-tr ### Local Testing +Classification reuses your own Claude Code login, so no token is needed locally — only the Zendesk credentials: + ``` pip install -r zendesk_triage/requirements.txt -export ZENDESK_SUBDOMAIN=... ZENDESK_EMAIL=... ZENDESK_API_TOKEN=... ANTHROPIC_API_KEY=... +export ZENDESK_SUBDOMAIN=... ZENDESK_EMAIL=... ZENDESK_API_TOKEN=... -# fetch + analyze, print the Discord payload, post nothing +# fetch + classify, print the Discord payload, post nothing python zendesk_triage/triage.py --window-hours 48 --dry-run -``` - -Locally, `--backend claude-cli` needs no token at all — it reuses your own Claude Code login: - -``` -# what CI runs: classify through Claude Code -python zendesk_triage/triage.py --backend claude-cli --window-hours 48 --dry-run # or dump the batch, classify it by hand, and feed the findings back python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 diff --git a/zendesk_triage/requirements.txt b/zendesk_triage/requirements.txt index 00d0e6e..d80d9fc 100644 --- a/zendesk_triage/requirements.txt +++ b/zendesk_triage/requirements.txt @@ -1,2 +1 @@ -anthropic==0.116.0 requests==2.32.3 diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 3082641..0f44b02 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -2,10 +2,10 @@ """ Daily Zendesk ticket triage with Claude, delivered to Discord. -Fetches open Zendesk tickets (broad query by default, not just bugs), sends the -whole batch to Claude in a single structured-output request, and posts a Discord -summary that links back to each original ticket and highlights the ones worth -looking into (crashes, data loss, legal requests, security/legislation, etc.). +Fetches open Zendesk tickets (broad query by default, not just bugs), classifies the +whole batch in one schema-enforced request through the local `claude` CLI, and posts +a Discord summary that links back to each original ticket and highlights the ones +worth looking into (crashes, data loss, legal requests, security/legislation, etc.). Because this repo is public, ticket content is never written to the job summary or anywhere public: in a normal run the only place ticket detail goes is the Discord @@ -20,11 +20,13 @@ bug_report | low_star_review | legal_request | security_or_legislation | question | feature_request | other +Classification authenticates as Claude Code: your own login locally, or a +CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token` in CI. No Claude API key involved. + Config (env vars, or CLI flags for local runs): ZENDESK_SUBDOMAIN e.g. "mycompany" -> https://mycompany.zendesk.com ZENDESK_EMAIL agent email for API token auth ZENDESK_API_TOKEN Zendesk API token - ANTHROPIC_API_KEY Claude API key (read by the SDK automatically) DISCORD_WEBHOOK_URL Discord incoming webhook ZENDESK_QUERY (optional) Zendesk search query; see DEFAULT_QUERY ZENDESK_TRIAGE_MODEL (optional) Claude model alias or id; defaults to `opus` @@ -42,10 +44,8 @@ # or an explicit query, which overrides --window-hours python triage.py --query "type:ticket status:open tags:bug" --max-tickets 50 - # local debugging without an ANTHROPIC_API_KEY: classify via the `claude` CLI - python triage.py --backend claude-cli --dry-run --max-tickets 20 - - # or split it in two: dump the batch, classify it by hand, feed it back + # split classification out entirely: dump the batch, classify it by hand, + # feed the findings back in to render python triage.py --dump-batch /tmp/batch.json --max-tickets 20 python triage.py --backend file --findings /tmp/findings.json --dry-run """ @@ -61,7 +61,6 @@ from datetime import datetime, timedelta, timezone from functools import partial -import anthropic import requests # Open, pending, new, and on-hold tickets, newest first. Broad on purpose: we @@ -104,10 +103,9 @@ def window_label(hours): DEFAULT_MAX_TICKETS = 100 DESCRIPTION_CHARS = 1500 # per-ticket description sent to Claude (triage only) # One classification runs ~100 output tokens per ticket, and adaptive thinking draws -# from the same max_tokens budget. 400 keeps a chunk far under the 128K output +# from the same output budget. 400 keeps a chunk far under the model's 128K output # ceiling; batches larger than this are split rather than truncated. DEFAULT_BATCH_SIZE = 400 -MAX_OUTPUT_TOKENS = 128000 # ---- Taxonomy -------------------------------------------------------------- # @@ -670,8 +668,9 @@ def findings_from_cli_envelope(envelope): """Pull the findings out of a `claude -p --output-format json` envelope. With --json-schema the shape lands in `structured_output`, already validated - against SCHEMA. An envelope without that key means the CLI ignored the flag - (it predates v2.1.205), which would otherwise surface as an empty digest. + against SCHEMA. An envelope without that key would otherwise surface as an empty + digest, so it exits naming both causes: a CLI too old for the flag, or a reply + that ran out of output tokens before the JSON closed. """ if envelope.get("is_error") or envelope.get("subtype") != "success": sys.exit(f"`claude` reported an error: {envelope.get('result') or envelope}") @@ -680,8 +679,10 @@ def findings_from_cli_envelope(envelope): print(f"claude CLI reported ${cost:.4f} for this batch.") payload = envelope.get("structured_output") if not isinstance(payload, dict): - sys.exit("`claude` returned no structured_output: --json-schema needs Claude " - "Code v2.1.205 or newer (check `claude --version`).") + sys.exit("`claude` returned no structured_output. Either the CLI predates " + "--json-schema (needs v2.1.205 or newer — check `claude --version`) " + f"or the batch outgrew the output ceiling: lower --batch-size " + f"(currently splitting at {DEFAULT_BATCH_SIZE}).") return tickets_from_payload(payload, "`claude` CLI") @@ -707,7 +708,8 @@ def analyze_via_claude_cli(model, effort, compact_tickets, timeout=1800): cmd, input=prompt, capture_output=True, text=True, timeout=timeout ) except FileNotFoundError: - sys.exit("`claude` not found on PATH. Install Claude Code, or use --backend api.") + sys.exit("`claude` not found on PATH. Install Claude Code: " + "https://code.claude.com/docs/en/setup") except subprocess.TimeoutExpired: sys.exit(f"`claude` timed out after {timeout}s. Try a smaller --max-tickets.") if proc.returncode != 0: @@ -762,37 +764,6 @@ def analyze_in_chunks(analyzer, compact_tickets, batch_size): return findings -def analyze(client, model, effort, compact_tickets): - prompt = build_analysis_prompt(compact_tickets) - with client.messages.stream( - model=model, - max_tokens=MAX_OUTPUT_TOKENS, - thinking={"type": "adaptive"}, - output_config={ - "format": {"type": "json_schema", "schema": SCHEMA}, - "effort": effort, - }, - system=SYSTEM_PROMPT, - messages=[{"role": "user", "content": prompt}], - ) as stream: - message = stream.get_final_message() - - if message.stop_reason == "refusal": - sys.exit("Claude refused to process the batch.") - if message.stop_reason == "max_tokens": - # Structured output truncated mid-JSON: json.loads below would fail with a - # baffling parse error, so say what actually went wrong. - sys.exit( - f"Claude hit the {MAX_OUTPUT_TOKENS} output-token limit on a batch of " - f"{len(compact_tickets)} tickets, so the JSON is incomplete. " - f"Lower --batch-size (currently splitting at {DEFAULT_BATCH_SIZE})." - ) - text = next((b.text for b in message.content if b.type == "text"), None) - if not text: - sys.exit("Claude returned no structured output.") - return tickets_from_payload(json.loads(text), "Claude") - - # ---- Discord rendering ----------------------------------------------------- SEVERITY_COLOR = { @@ -1056,9 +1027,9 @@ def main(): f"(default: {DEFAULT_BATCH_SIZE}).") parser.add_argument("--dry-run", action="store_true", help="Fetch and analyze, then print the Discord payload instead of posting.") - parser.add_argument("--backend", default="api", choices=["api", "claude-cli", "file"], - help="Where classification happens: the Anthropic API (default), the " - "local `claude` CLI (no API key needed), or a findings file.") + parser.add_argument("--backend", default="claude-cli", choices=["claude-cli", "file"], + help="Where classification happens: Claude Code (default), or a " + "findings file classified elsewhere.") parser.add_argument("--findings", help="Findings JSON to render instead of classifying (--backend file).") parser.add_argument("--review-star-floor", type=int, default=DEFAULT_REVIEW_STAR_FLOOR, @@ -1177,11 +1148,7 @@ def main(): print("Classify it, then: --backend file --findings --dry-run") return - if args.backend == "claude-cli": - analyzer = partial(analyze_via_claude_cli, model, args.effort) - else: - client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY - analyzer = partial(analyze, client, model, args.effort) + analyzer = partial(analyze_via_claude_cli, model, args.effort) findings = analyze_in_chunks(analyzer, compact, args.batch_size) # Keep only findings whose id maps to a fetched ticket, in case of drift. From 6bcc483ea9f04202cedce52922c662f966ac3c08 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 08:05:04 +0200 Subject: [PATCH 06/16] feat: restore the Anthropic API backend alongside Claude Code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings back analyze(), the anthropic dependency and the `api` backend, and teaches the workflow to choose: `api` when an ANTHROPIC_API_KEY secret exists, `claude-cli` otherwise, overridable per dispatch. Adding the secret therefore switches the scheduled job over with no workflow edit, and the job keeps running on the subscription token until then. Because both credentials are in scope in the run step, it unsets the one the chosen backend doesn't use — an API key outranks the OAuth token inside `claude`, so leaving it set would make a claude-cli run bill the API instead. The Claude Code install is skipped entirely on the API path. DEFAULT_MODEL stays the `opus` alias, so the API path maps aliases to ids via API_MODEL_ALIASES (newest model per family, matching what the CLI's own alias resolution lands on). --- .github/workflows/zendesk_triage.yml | 56 ++++++++++++++---- README.md | 39 ++++++++++--- zendesk_triage/requirements.txt | 1 + zendesk_triage/test_triage.py | 20 +++++++ zendesk_triage/triage.py | 86 ++++++++++++++++++++++++---- 5 files changed, 172 insertions(+), 30 deletions(-) diff --git a/.github/workflows/zendesk_triage.yml b/.github/workflows/zendesk_triage.yml index fe50b9a..b2adf06 100644 --- a/.github/workflows/zendesk_triage.yml +++ b/.github/workflows/zendesk_triage.yml @@ -27,6 +27,11 @@ on: description: "Ignore saved state and re-report everything in the window" type: boolean default: false + backend: + description: "Where classification runs (auto: API key if configured, else Claude Code)" + type: choice + options: [auto, api, claude-cli] + default: auto # Two overlapping runs would race on the same state file, and the loser's # reported tickets would be forgotten. Queue instead of cancelling, so a @@ -55,18 +60,41 @@ jobs: - name: Install dependencies run: pip install -r zendesk_triage/requirements.txt - # Classification runs through Claude Code, not the Anthropic API, so the job - # authenticates with a subscription OAuth token instead of an API key. The - # `stable` channel is ~a week behind `latest` and skips releases with known - # major regressions, which is what a scheduled job wants; the version is - # echoed because a CLI older than v2.1.205 ignores --json-schema and the - # script's error for that names this line. + # Classification can run through the Anthropic API or through Claude Code, and + # the credential differs: an org-owned ANTHROPIC_API_KEY, or a subscription + # OAuth token from `claude setup-token`. `auto` prefers the API key when the + # secret exists, because it doesn't draw on one person's subscription quota — + # so adding that secret switches the job over with no edit here, and the job + # keeps running on the subscription until then. The secret is tested through + # a boolean rather than shelled out, so its value never reaches the runner. + - name: Select backend + id: backend + env: + FORCED: ${{ github.event.inputs.backend }} + HAS_API_KEY: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + if [ -n "$FORCED" ] && [ "$FORCED" != "auto" ]; then + backend="$FORCED" + elif [ "$HAS_API_KEY" = "true" ]; then + backend="api" + else + backend="claude-cli" + fi + echo "backend=$backend" >> "$GITHUB_OUTPUT" + echo "Classification backend: $backend" + + # Only the claude-cli backend needs the CLI. The `stable` channel is ~a week + # behind `latest` and skips releases with known major regressions, which is what + # a scheduled job wants; the version is echoed because a CLI older than + # v2.1.205 ignores --json-schema and the script's error for that names this. - name: Install Claude Code + if: steps.backend.outputs.backend == 'claude-cli' run: | curl -fsSL https://claude.ai/install.sh | bash -s stable echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Report Claude Code version + if: steps.backend.outputs.backend == 'claude-cli' run: claude --version # Unique key so every run writes a fresh entry; the restore-keys prefix pulls @@ -111,25 +139,33 @@ jobs: fi echo "Window: ${window}h, max tickets: ${max}" - # ANTHROPIC_API_KEY is deliberately absent: it outranks the OAuth token in - # Claude Code's credential precedence, and in -p mode a key that is present is - # always used — so setting it here would silently bill the API instead. + # Both credentials are in scope, so the step must drop the one the chosen + # backend doesn't use: an API key outranks the OAuth token in Claude Code's + # credential precedence, and in -p mode a key that is present is always used, + # so leaving it set would make a claude-cli run silently bill the API instead. # DISABLE_AUTOUPDATER keeps the CLI from downloading a new version mid-run, # so the version echoed above is the one that actually classifies. - name: Run triage env: + BACKEND: ${{ steps.backend.outputs.backend }} ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }} ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }} ZENDESK_API_TOKEN: ${{ secrets.ZENDESK_API_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} DISABLE_AUTOUPDATER: "1" DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} ZENDESK_QUERY: ${{ github.event.inputs.query }} ZENDESK_TRIAGE_MODEL: ${{ vars.ZENDESK_TRIAGE_MODEL }} run: | + if [ "$BACKEND" = "api" ]; then + unset CLAUDE_CODE_OAUTH_TOKEN + else + unset ANTHROPIC_API_KEY + fi mkdir -p .triage-state python zendesk_triage/triage.py \ - --backend claude-cli \ + --backend "$BACKEND" \ --window-hours "${{ steps.cfg.outputs.window }}" \ --max-tickets "${{ steps.cfg.outputs.max }}" \ --state .triage-state/seen.json diff --git a/README.md b/README.md index 5aba25e..4f305f5 100644 --- a/README.md +++ b/README.md @@ -131,24 +131,40 @@ Two caveats worth knowing: | `ZENDESK_SUBDOMAIN` | Zendesk subdomain (`mycompany` → `mycompany.zendesk.com`) | | `ZENDESK_EMAIL` | Agent email used for Zendesk API-token auth | | `ZENDESK_API_TOKEN` | Zendesk API token | -| `CLAUDE_CODE_OAUTH_TOKEN` | Claude subscription OAuth token — see [Claude authentication](#claude-authentication) | +| **One Claude credential** | `ANTHROPIC_API_KEY` **or** `CLAUDE_CODE_OAUTH_TOKEN` — see [Claude authentication](#claude-authentication) | | `DISCORD_WEBHOOK_URL` | Discord webhook (reused from the failure-notification setup) | ### Claude Authentication -Classification runs through Claude Code (`--backend claude-cli`), not the Anthropic API, so the job authenticates with a **subscription OAuth token** rather than an API key. Generate one on a machine where you're logged into Claude Code: +Classification can reach Claude two ways, and they need different credentials: + +| Backend | Credential | Billing | +| ------- | ---------- | ------- | +| `api` | `ANTHROPIC_API_KEY` from the [Claude Console](https://platform.claude.com) | Per token, to the Console organization | +| `claude-cli` | `CLAUDE_CODE_OAUTH_TOKEN` from `claude setup-token` | Draws on that subscription's usage limits | + +**The workflow picks for itself.** The `Select backend` step uses `api` when an `ANTHROPIC_API_KEY` secret exists and `claude-cli` otherwise, so adding that secret switches the job over with no edit to the workflow, and until then it keeps running on the subscription. The choice is echoed into the run log, and a `workflow_dispatch` run can force either backend to test one without touching secrets. + +Prefer the API key where you have one: it's an organization-owned credential that doesn't expire annually, doesn't consume an individual's quota, and costs a fraction of a run through Claude Code (measured on this batch shape: ~$0.01 versus ~$0.31, because each `claude -p` invocation carries ~25K tokens of Claude Code system prompt). + +#### Notes on the API key + +An API key only exists inside a **Claude Console organization** (`platform.claude.com`), which is separate from a claude.ai Pro/Max/Team/Enterprise subscription with its own membership and billing — a claude.ai admin console has no API keys at all. If nobody can find one, the likely answer is that no Console organization exists yet rather than a permissions problem. + +#### Notes on the subscription token + +Generate it on a machine where you're logged into Claude Code: ``` claude setup-token ``` -It runs the browser authorization flow and prints the token once — it is not saved anywhere. Store it as the `CLAUDE_CODE_OAUTH_TOKEN` repo secret. Requires a Pro, Max, Team, or Enterprise plan; see [Generate a long-lived token](https://code.claude.com/docs/en/authentication#generate-a-long-lived-token). - -Four things worth knowing before you rely on it: +It runs the browser authorization flow and prints the token once — it is not saved anywhere. Requires a Pro, Max, Team, or Enterprise plan; see [Generate a long-lived token](https://code.claude.com/docs/en/authentication#generate-a-long-lived-token). Then: - **The token lasts one year.** It expires silently from the workflow's point of view — the run just fails to authenticate. Put the renewal date somewhere you'll see it. -- **Never add `ANTHROPIC_API_KEY` to that step.** An API key [outranks the OAuth token](https://code.claude.com/docs/en/authentication#authentication-precedence) in Claude Code's credential precedence, and in `-p` mode a key that is present is always used — so the job would quietly bill the API instead of the subscription. -- **Runs draw on that subscription's usage limits**, not API credits, and the token is tied to whoever minted it. A scheduled run competes with that person's own interactive Claude Code usage, and hitting a limit fails the run (the tickets stay eligible and get picked up by the next one, per the dedup rules above). Anthropic's own guidance is to use an API key for a secret shared across an org for exactly this reason. +- **The two credentials must not both be live in the run step.** An API key [outranks the OAuth token](https://code.claude.com/docs/en/authentication#authentication-precedence) in Claude Code's credential precedence, and in `-p` mode a key that is present is always used, so a `claude-cli` run with a key in scope would quietly bill the API instead. The workflow unsets whichever credential the chosen backend doesn't use. +- **Runs draw on that subscription's usage limits**, not API credits, and the token is tied to whoever minted it — so a scheduled run competes with that person's own interactive Claude Code usage. Hitting a limit fails the run (the tickets stay eligible and get picked up by the next one, per the dedup rules above). If your organization has [usage credits](https://support.claude.com/en/articles/12429409-manage-usage-credits-for-paid-claude-plans) enabled, usage continues past the allowance at standard API rates instead of stopping, which turns a failed run into a billed one. + > A [separate monthly Agent SDK credit](https://support.claude.com/en/articles/15036540-use-the-claude-agent-sdk-with-your-claude-plan) was announced for 2026-06-15 and then **paused** — `claude -p` still draws on subscription limits as described here. Worth re-reading that page before assuming otherwise. - **The CLI must be v2.1.205 or newer** for `--json-schema`. The workflow installs the `stable` channel and echoes `claude --version` into the run log, because the script's "no structured_output" error points here. ### Optional Configuration @@ -159,7 +175,8 @@ Four things worth knowing before you rely on it: | `--state` | flag | *(unset)* | Dedup state file. The workflow points this at the cached `.triage-state/seen.json` | | `--state-retention-days` | flag | `30` | Forget state entries older than N days | | `ZENDESK_QUERY` | env / `--query` | *(unset)* | Explicit Zendesk search query. Overrides `--window-hours` entirely | -| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `opus` | Model alias (`opus`, `sonnet`, `haiku`) or a full id. Set it to `sonnet` to reduce cost on large batches | +| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `opus` | Model alias (`opus`, `sonnet`, `haiku`) or a full id. Set it to `sonnet` to reduce cost on large batches. On `--backend api` the alias is mapped to an id by `API_MODEL_ALIASES` | +| `backend` | workflow input / `--backend` | `auto` (workflow) / `claude-cli` (flag) | `api`, `claude-cli`, or `file`. The workflow's `auto` resolves to `api` when an `ANTHROPIC_API_KEY` secret exists — see [Claude authentication](#claude-authentication) | | `--max-tickets` | workflow input / flag | `1000` (workflow) / `100` (flag) | Runaway guard on tickets analyzed per run, **not** a batch size. The workflow passes `1000`; a bare `python triage.py` uses the script's own `DEFAULT_MAX_TICKETS` of `100`. Zendesk's search API caps a query at 1000 results, so higher values don't fetch more | | `--batch-size` | flag | `400` | Split batches larger than this across multiple requests | | `--review-star-floor` | flag | `3` | Classify app-store reviews at or below N stars; count the rest | @@ -228,7 +245,7 @@ Offline tests covering the window arithmetic, dedup partitioning, state round-tr ### Local Testing -Classification reuses your own Claude Code login, so no token is needed locally — only the Zendesk credentials: +The default backend reuses your own Claude Code login, so no Claude credential is needed locally — only the Zendesk ones: ``` pip install -r zendesk_triage/requirements.txt @@ -237,6 +254,10 @@ export ZENDESK_SUBDOMAIN=... ZENDESK_EMAIL=... ZENDESK_API_TOKEN=... # fetch + classify, print the Discord payload, post nothing python zendesk_triage/triage.py --window-hours 48 --dry-run +# exercise the path CI uses once an API key is configured +export ANTHROPIC_API_KEY=... +python zendesk_triage/triage.py --backend api --window-hours 48 --dry-run + # or dump the batch, classify it by hand, and feed the findings back python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 python zendesk_triage/triage.py --backend file --findings /tmp/findings.json --dry-run diff --git a/zendesk_triage/requirements.txt b/zendesk_triage/requirements.txt index d80d9fc..00d0e6e 100644 --- a/zendesk_triage/requirements.txt +++ b/zendesk_triage/requirements.txt @@ -1 +1,2 @@ +anthropic==0.116.0 requests==2.32.3 diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index 41357c4..bc9c6d1 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -791,6 +791,26 @@ def test_a_non_object_entry_exits(self): triage.tickets_from_payload({"tickets": [["not", "an", "object"]]}, "x") +class TestResolveApiModel(unittest.TestCase): + """The CLI resolves aliases itself; the API takes ids, so only that path maps.""" + + def test_every_alias_maps_to_an_id(self): + for alias, model_id in triage.API_MODEL_ALIASES.items(): + self.assertEqual(triage.resolve_api_model(alias), model_id) + self.assertTrue(model_id.startswith("claude-"), model_id) + + def test_the_default_model_is_mappable(self): + """DEFAULT_MODEL is an alias, so --backend api would 404 without an entry.""" + self.assertIn(triage.DEFAULT_MODEL, triage.API_MODEL_ALIASES) + + def test_a_full_id_passes_through(self): + self.assertEqual(triage.resolve_api_model("claude-opus-4-8"), "claude-opus-4-8") + + def test_an_unknown_value_passes_through(self): + """A model newer than this table should reach the API rather than be rewritten.""" + self.assertEqual(triage.resolve_api_model("claude-future-9"), "claude-future-9") + + class TestFindingsFromCliEnvelope(unittest.TestCase): def envelope(self, **overrides): base = { diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 0f44b02..828e783 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -3,9 +3,9 @@ Daily Zendesk ticket triage with Claude, delivered to Discord. Fetches open Zendesk tickets (broad query by default, not just bugs), classifies the -whole batch in one schema-enforced request through the local `claude` CLI, and posts -a Discord summary that links back to each original ticket and highlights the ones -worth looking into (crashes, data loss, legal requests, security/legislation, etc.). +whole batch in one schema-enforced request, and posts a Discord summary that links +back to each original ticket and highlights the ones worth looking into (crashes, +data loss, legal requests, security/legislation, etc.). Because this repo is public, ticket content is never written to the job summary or anywhere public: in a normal run the only place ticket detail goes is the Discord @@ -20,13 +20,18 @@ bug_report | low_star_review | legal_request | security_or_legislation | question | feature_request | other -Classification authenticates as Claude Code: your own login locally, or a -CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token` in CI. No Claude API key involved. +Two ways to reach Claude, chosen with --backend: + api the Anthropic API, with an ANTHROPIC_API_KEY (org-owned credential, + per-token billing, structured outputs enforced by the API) + claude-cli the local `claude` CLI, authenticating as Claude Code — your own + login locally, or a CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token` + in CI. No API key, but it draws on that subscription's usage limits. Config (env vars, or CLI flags for local runs): ZENDESK_SUBDOMAIN e.g. "mycompany" -> https://mycompany.zendesk.com ZENDESK_EMAIL agent email for API token auth ZENDESK_API_TOKEN Zendesk API token + ANTHROPIC_API_KEY Claude API key, for --backend api (read by the SDK itself) DISCORD_WEBHOOK_URL Discord incoming webhook ZENDESK_QUERY (optional) Zendesk search query; see DEFAULT_QUERY ZENDESK_TRIAGE_MODEL (optional) Claude model alias or id; defaults to `opus` @@ -44,6 +49,9 @@ # or an explicit query, which overrides --window-hours python triage.py --query "type:ticket status:open tags:bug" --max-tickets 50 + # classify through the Anthropic API instead of Claude Code + python triage.py --backend api --dry-run + # split classification out entirely: dump the batch, classify it by hand, # feed the findings back in to render python triage.py --dump-batch /tmp/batch.json --max-tickets 20 @@ -61,6 +69,7 @@ from datetime import datetime, timedelta, timezone from functools import partial +import anthropic import requests # Open, pending, new, and on-hold tickets, newest first. Broad on purpose: we @@ -100,12 +109,22 @@ def window_label(hours): # authenticated plan allows, so a model release needs no edit here and a plan # without Opus access degrades instead of 404-ing on a dead id. DEFAULT_MODEL = "opus" +# The Anthropic API takes model ids, not Claude Code aliases, so the same alias has +# to be mapped for --backend api. Each entry is the newest model in its family, +# which is what the CLI's own alias resolution lands on — pass a full id to pin a +# specific version instead. +API_MODEL_ALIASES = { + "opus": "claude-opus-5", + "sonnet": "claude-sonnet-5", + "haiku": "claude-haiku-4-5", +} DEFAULT_MAX_TICKETS = 100 DESCRIPTION_CHARS = 1500 # per-ticket description sent to Claude (triage only) # One classification runs ~100 output tokens per ticket, and adaptive thinking draws # from the same output budget. 400 keeps a chunk far under the model's 128K output # ceiling; batches larger than this are split rather than truncated. DEFAULT_BATCH_SIZE = 400 +MAX_OUTPUT_TOKENS = 128000 # ---- Taxonomy -------------------------------------------------------------- # @@ -610,6 +629,15 @@ def compact_ticket(ticket): } +def resolve_api_model(model): + """Map a Claude Code model alias onto the id the Anthropic API expects. + + Anything that isn't a known alias passes through untouched, so a pinned id + (`claude-opus-4-8`) or a model newer than this table still works. + """ + return API_MODEL_ALIASES.get(model, model) + + def build_analysis_prompt(compact_tickets): return ( "Classify every ticket in this batch and return one object per ticket.\n\n" @@ -708,8 +736,8 @@ def analyze_via_claude_cli(model, effort, compact_tickets, timeout=1800): cmd, input=prompt, capture_output=True, text=True, timeout=timeout ) except FileNotFoundError: - sys.exit("`claude` not found on PATH. Install Claude Code: " - "https://code.claude.com/docs/en/setup") + sys.exit("`claude` not found on PATH. Install Claude Code " + "(https://code.claude.com/docs/en/setup), or use --backend api.") except subprocess.TimeoutExpired: sys.exit(f"`claude` timed out after {timeout}s. Try a smaller --max-tickets.") if proc.returncode != 0: @@ -764,6 +792,37 @@ def analyze_in_chunks(analyzer, compact_tickets, batch_size): return findings +def analyze(client, model, effort, compact_tickets): + prompt = build_analysis_prompt(compact_tickets) + with client.messages.stream( + model=model, + max_tokens=MAX_OUTPUT_TOKENS, + thinking={"type": "adaptive"}, + output_config={ + "format": {"type": "json_schema", "schema": SCHEMA}, + "effort": effort, + }, + system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": prompt}], + ) as stream: + message = stream.get_final_message() + + if message.stop_reason == "refusal": + sys.exit("Claude refused to process the batch.") + if message.stop_reason == "max_tokens": + # Structured output truncated mid-JSON: json.loads below would fail with a + # baffling parse error, so say what actually went wrong. + sys.exit( + f"Claude hit the {MAX_OUTPUT_TOKENS} output-token limit on a batch of " + f"{len(compact_tickets)} tickets, so the JSON is incomplete. " + f"Lower --batch-size (currently splitting at {DEFAULT_BATCH_SIZE})." + ) + text = next((b.text for b in message.content if b.type == "text"), None) + if not text: + sys.exit("Claude returned no structured output.") + return tickets_from_payload(json.loads(text), "Claude") + + # ---- Discord rendering ----------------------------------------------------- SEVERITY_COLOR = { @@ -1027,9 +1086,10 @@ def main(): f"(default: {DEFAULT_BATCH_SIZE}).") parser.add_argument("--dry-run", action="store_true", help="Fetch and analyze, then print the Discord payload instead of posting.") - parser.add_argument("--backend", default="claude-cli", choices=["claude-cli", "file"], - help="Where classification happens: Claude Code (default), or a " - "findings file classified elsewhere.") + parser.add_argument("--backend", default="claude-cli", choices=["claude-cli", "api", "file"], + help="Where classification happens: Claude Code (default, no API " + "key needed), the Anthropic API (needs ANTHROPIC_API_KEY), or " + "a findings file classified elsewhere.") parser.add_argument("--findings", help="Findings JSON to render instead of classifying (--backend file).") parser.add_argument("--review-star-floor", type=int, default=DEFAULT_REVIEW_STAR_FLOOR, @@ -1148,7 +1208,11 @@ def main(): print("Classify it, then: --backend file --findings --dry-run") return - analyzer = partial(analyze_via_claude_cli, model, args.effort) + if args.backend == "api": + client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY + analyzer = partial(analyze, client, resolve_api_model(model), args.effort) + else: + analyzer = partial(analyze_via_claude_cli, model, args.effort) findings = analyze_in_chunks(analyzer, compact, args.batch_size) # Keep only findings whose id maps to a fetched ticket, in case of drift. From 3c4e0ab5c6542b05405f9a9b59eef89c02d110ce Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 08:14:07 +0200 Subject: [PATCH 07/16] harden: strip the claude-cli invocation down to classification only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ticket text is written by strangers and the runner has a checkout of this repo, so the CLI ran with more attached to it than the job needs: the instructions rode on stdin next to the untrusted payload, and the session loaded the runner's and the repo's hooks, plugins, skills and CLAUDE.md along with the full tool surface. Now: SYSTEM_PROMPT travels as --system-prompt (stdin carries only the ticket JSON), --setting-sources "" loads no config from either machine, --strict-mcp-config with no config means no MCP servers, and the tool surface is denied by name. A session ends up with one tool, StructuredOutput, and no MCP servers. Dropping the agent preamble and the tool definitions also took a two-ticket fixture from ~$0.29 to ~$0.015, so the README's cost comparison is corrected too. Measured on v2.1.218, and the reason the deny list is by name rather than `*`: a wildcard also denies StructuredOutput, which is how --json-schema is implemented, so the run comes back as prose. `--permission-mode dontAsk` is not a substitute either — a session with no allow rules still executed Bash(echo …). Tests guard the parts that fail silently. --- README.md | 15 +++++++-- zendesk_triage/test_triage.py | 31 +++++++++++++++++ zendesk_triage/triage.py | 63 +++++++++++++++++++++++++++++++---- 3 files changed, 100 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4f305f5..b528d76 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ Classification can reach Claude two ways, and they need different credentials: **The workflow picks for itself.** The `Select backend` step uses `api` when an `ANTHROPIC_API_KEY` secret exists and `claude-cli` otherwise, so adding that secret switches the job over with no edit to the workflow, and until then it keeps running on the subscription. The choice is echoed into the run log, and a `workflow_dispatch` run can force either backend to test one without touching secrets. -Prefer the API key where you have one: it's an organization-owned credential that doesn't expire annually, doesn't consume an individual's quota, and costs a fraction of a run through Claude Code (measured on this batch shape: ~$0.01 versus ~$0.31, because each `claude -p` invocation carries ~25K tokens of Claude Code system prompt). +Prefer the API key where you have one, but for credential reasons rather than cost: it's organization-owned, doesn't expire annually, and doesn't consume an individual's quota. The two are close on cost — a `claude -p` run used to carry ~25K tokens of Claude Code system prompt and tool definitions on top of the batch, but the [locked-down invocation](#how-the-claude-cli-invocation-is-locked-down) removes both, which took a two-ticket fixture from ~$0.29 to ~$0.015. #### Notes on the API key @@ -263,9 +263,18 @@ python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 python zendesk_triage/triage.py --backend file --findings /tmp/findings.json --dry-run ``` -Two properties of the `claude-cli` backend to keep in mind: +### How the `claude-cli` invocation is locked down + +Ticket text is written by strangers and the runner has a checkout of this repo, so the CLI is invoked with as little around it as possible: our own `--system-prompt` in place of Claude Code's, `--setting-sources ""` (no hooks, plugins, skills, allow-rules or `CLAUDE.md` from either the runner or the repo), `--strict-mcp-config` with no config (no MCP servers), and an explicit `--disallowed-tools` list. A session then exposes one tool, `StructuredOutput`, and no MCP servers. Removing the agent preamble and the tool definitions is also what makes this path cheap. + +Three findings from `v2.1.218` that explain why it's written that way — all worth re-testing after a CLI upgrade: + +- **`--disallowed-tools "*"` can't be used**, tempting as it is. It empties the surface, but `--json-schema` is itself implemented as a `StructuredOutput` tool, so the wildcard denies that too and the run returns prose with no `structured_output`. Allow-listing `StructuredOutput` alongside the wildcard leaves the tool present but still doesn't produce structured output. +- **`--permission-mode dontAsk` is not a boundary.** A session with no allow rules still ran `Bash(echo …)`, because the mode permits a read-only command set. It's kept as a backstop, not as the control. +- **The deny list is therefore by name, and will go stale** as tools are added. Naming only the obvious ones (`Bash`, `Read`, `Write`, …) left 19 others live, including several with outward side effects. To see what a session really exposes, read the `init` event: `echo hi | claude -p --output-format stream-json --verbose [flags] | grep '"subtype":"init"'`. + +One more, on the flag not used: -- Each invocation carries ~25K tokens of Claude Code system-prompt overhead on top of the batch, so the per-run cost is dominated by that on small windows. It's one invocation per chunk, not per ticket — a typical 48h window is a single call. - **Do not add `--bare`.** It's otherwise the right flag for a scripted call (it skips hook, plugin, MCP and `CLAUDE.md` discovery, so the runner behaves the same as your laptop), but bare mode reads `ANTHROPIC_API_KEY` or an `apiKeyHelper` **only** — it never touches OAuth credentials, which is exactly what both CI and your local login use. See [bare mode](https://code.claude.com/docs/en/headless#start-faster-with-bare-mode); the docs say it will become the default for `-p` in a future release, so this is worth re-checking on CLI upgrades. ## Workflow Failure Notificaiton diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index bc9c6d1..cdc6123 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -791,6 +791,37 @@ def test_a_non_object_entry_exits(self): triage.tickets_from_payload({"tickets": [["not", "an", "object"]]}, "x") +class TestCliIsolation(unittest.TestCase): + """The claude-cli invocation is locked down because ticket text is untrusted and + the runner has a checkout. Each of these fails silently if broken: a wrong deny + list returns prose instead of findings, a missing flag loads the repo's config.""" + + def test_structured_output_is_never_denied(self): + """--json-schema is implemented as the StructuredOutput tool, so denying it — + or passing a `*` wildcard — makes the run return prose and no findings.""" + denied = triage.CLI_DENIED_TOOLS.split() + self.assertNotIn("StructuredOutput", denied) + self.assertNotIn("*", denied) + + def test_tools_with_side_effects_are_denied(self): + denied = triage.CLI_DENIED_TOOLS.split() + for tool in ("Bash", "Write", "Edit", "WebFetch", "WebSearch", "Task"): + self.assertIn(tool, denied) + + def test_the_system_prompt_travels_as_a_flag(self): + """Not on stdin with the tickets: stdin is untrusted input, the prompt isn't.""" + self.assertIn("--system-prompt", triage.CLI_ISOLATION_ARGS) + self.assertIn(triage.SYSTEM_PROMPT, triage.CLI_ISOLATION_ARGS) + + def test_no_setting_sources_are_loaded(self): + """An empty value is what keeps hooks, plugins, skills and CLAUDE.md out.""" + args = triage.CLI_ISOLATION_ARGS + self.assertEqual(args[args.index("--setting-sources") + 1], "") + + def test_mcp_config_is_strict(self): + self.assertIn("--strict-mcp-config", triage.CLI_ISOLATION_ARGS) + + class TestResolveApiModel(unittest.TestCase): """The CLI resolves aliases itself; the API takes ids, so only that path maps.""" diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 828e783..8c8ccf3 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -692,6 +692,54 @@ def tickets_from_payload(payload, source): return validate_findings(found, source) +# Ticket text is untrusted input written by strangers, and the runner has a checkout +# of this repo, so the CLI invocation is stripped to just "classify this text": +# --system-prompt ours replaces Claude Code's, so there is no agent preamble +# and no per-machine section (cwd, env, git status) either +# --setting-sources "" loads no user/project/local settings, so no hooks, plugins, +# skills, allow-rules or CLAUDE.md from the runner or the repo +# --strict-mcp-config with no --mcp-config, that means no MCP servers at all +# --disallowed-tools the tools below, by name — see CLI_DENIED_TOOLS +# --permission-mode dontAsk, as a backstop rather than the boundary +# Everything Claude needs is in the prompt, so a ticket that tries to talk its way +# into running something should have nothing to reach for. +# --bare would give the same isolation and a faster start, but it reads +# ANTHROPIC_API_KEY only and never OAuth credentials, which is what this path uses. +# +# Two things measured on v2.1.218 that constrain how the tool surface is closed: +# - `--disallowed-tools "*"` does empty the surface, but it also denies +# StructuredOutput, which is how --json-schema is implemented — the run then +# returns prose and no structured_output. So the tools have to be named. +# - `--permission-mode dontAsk` is not a boundary on its own: a session with no +# allow rules still executed `Bash(echo …)`, because the mode permits a +# read-only command set. It stays as a backstop, not as the control. +# A named list goes stale as tools are added, so re-check what a session actually +# exposes after a CLI upgrade — the `init` event lists it: +# echo hi | claude -p --output-format stream-json --verbose [flags] \ +# | grep '"subtype":"init"' +CLI_DENIED_TOOLS = " ".join([ + # filesystem and execution + "Bash", "BashOutput", "KillShell", "Read", "Write", "Edit", "NotebookEdit", + "Glob", "Grep", + # network + "WebFetch", "WebSearch", + # delegation and session control + "Task", "TaskOutput", "TaskStop", "Workflow", "Skill", "SlashCommand", + "ToolSearch", "TodoWrite", "EnterWorktree", "ExitWorktree", "Monitor", + "ScheduleWakeup", + # anything that reaches outside the run + "Artifact", "SendMessage", "PushNotification", "RemoteTrigger", "DesignSync", + "CronCreate", "CronDelete", "CronList", "ShareOnboardingGuide", "ReportFindings", +]) +CLI_ISOLATION_ARGS = [ + "--system-prompt", SYSTEM_PROMPT, + "--setting-sources", "", + "--strict-mcp-config", + "--permission-mode", "dontAsk", + "--disallowed-tools", CLI_DENIED_TOOLS, +] + + def findings_from_cli_envelope(envelope): """Pull the findings out of a `claude -p --output-format json` envelope. @@ -721,19 +769,22 @@ def analyze_via_claude_cli(model, effort, compact_tickets, timeout=1800): --json-schema enforces the same SCHEMA the API path uses — so the response needs no prose parsing and no hand-maintained field list in the prompt. """ - prompt = "\n\n".join([SYSTEM_PROMPT, build_analysis_prompt(compact_tickets)]) - # --bare would be the faster startup, but it reads ANTHROPIC_API_KEY only and - # never touches OAuth credentials, which is exactly what CI authenticates with. cmd = ["claude", "-p", "--output-format", "json", - "--json-schema", json.dumps(SCHEMA)] + "--json-schema", json.dumps(SCHEMA), *CLI_ISOLATION_ARGS] if model: cmd += ["--model", model] if effort: cmd += ["--effort", effort] try: - # Prompt goes over stdin: a full batch can exceed the argv size limit. + # Only the ticket payload goes over stdin — the instructions are the system + # prompt above. It goes over stdin rather than argv because a full batch can + # exceed the argv size limit. proc = subprocess.run( - cmd, input=prompt, capture_output=True, text=True, timeout=timeout + cmd, + input=build_analysis_prompt(compact_tickets), + capture_output=True, + text=True, + timeout=timeout, ) except FileNotFoundError: sys.exit("`claude` not found on PATH. Install Claude Code " From ced2ecde26f73f752a32716aab815fa071924245 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 08:22:02 +0200 Subject: [PATCH 08/16] refactor: keep CI on the Anthropic API, claude-cli for local runs only Drops the backend selection step, the Claude Code install and the CLAUDE_CODE_OAUTH_TOKEN secret from the workflow: CI now always passes --backend api. That keeps the runner off a ~270MB CLI download and keeps CI on an org-owned credential rather than an individual's subscription. The claude-cli backend and its lockdown stay in the script as the local default, so a local run still needs no Claude credential. Its notes move to the local-testing section, including the one that bites there instead of in CI: an exported ANTHROPIC_API_KEY outranks your Claude Code login, so a claude-cli run with one in the shell bills the API instead. --- .github/workflows/zendesk_triage.yml | 62 +++------------------------- README.md | 54 +++++++----------------- zendesk_triage/triage.py | 10 ++--- 3 files changed, 26 insertions(+), 100 deletions(-) diff --git a/.github/workflows/zendesk_triage.yml b/.github/workflows/zendesk_triage.yml index b2adf06..a7d30ac 100644 --- a/.github/workflows/zendesk_triage.yml +++ b/.github/workflows/zendesk_triage.yml @@ -27,11 +27,6 @@ on: description: "Ignore saved state and re-report everything in the window" type: boolean default: false - backend: - description: "Where classification runs (auto: API key if configured, else Claude Code)" - type: choice - options: [auto, api, claude-cli] - default: auto # Two overlapping runs would race on the same state file, and the loser's # reported tickets would be forgotten. Queue instead of cancelling, so a @@ -60,43 +55,6 @@ jobs: - name: Install dependencies run: pip install -r zendesk_triage/requirements.txt - # Classification can run through the Anthropic API or through Claude Code, and - # the credential differs: an org-owned ANTHROPIC_API_KEY, or a subscription - # OAuth token from `claude setup-token`. `auto` prefers the API key when the - # secret exists, because it doesn't draw on one person's subscription quota — - # so adding that secret switches the job over with no edit here, and the job - # keeps running on the subscription until then. The secret is tested through - # a boolean rather than shelled out, so its value never reaches the runner. - - name: Select backend - id: backend - env: - FORCED: ${{ github.event.inputs.backend }} - HAS_API_KEY: ${{ secrets.ANTHROPIC_API_KEY != '' }} - run: | - if [ -n "$FORCED" ] && [ "$FORCED" != "auto" ]; then - backend="$FORCED" - elif [ "$HAS_API_KEY" = "true" ]; then - backend="api" - else - backend="claude-cli" - fi - echo "backend=$backend" >> "$GITHUB_OUTPUT" - echo "Classification backend: $backend" - - # Only the claude-cli backend needs the CLI. The `stable` channel is ~a week - # behind `latest` and skips releases with known major regressions, which is what - # a scheduled job wants; the version is echoed because a CLI older than - # v2.1.205 ignores --json-schema and the script's error for that names this. - - name: Install Claude Code - if: steps.backend.outputs.backend == 'claude-cli' - run: | - curl -fsSL https://claude.ai/install.sh | bash -s stable - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Report Claude Code version - if: steps.backend.outputs.backend == 'claude-cli' - run: claude --version - # Unique key so every run writes a fresh entry; the restore-keys prefix pulls # in the most recent previous one. run_attempt is in the key because cache # entries are immutable: a re-run reuses run_id, so without it attempt 2's save @@ -139,33 +97,23 @@ jobs: fi echo "Window: ${window}h, max tickets: ${max}" - # Both credentials are in scope, so the step must drop the one the chosen - # backend doesn't use: an API key outranks the OAuth token in Claude Code's - # credential precedence, and in -p mode a key that is present is always used, - # so leaving it set would make a claude-cli run silently bill the API instead. - # DISABLE_AUTOUPDATER keeps the CLI from downloading a new version mid-run, - # so the version echoed above is the one that actually classifies. + # Classification goes through the Anthropic API here, never the `claude` CLI: + # CI gets an org-owned credential that doesn't draw on an individual's + # subscription quota, and the runner needs no 270MB CLI download. The + # claude-cli backend stays in the script for local runs — see the README. - name: Run triage env: - BACKEND: ${{ steps.backend.outputs.backend }} ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }} ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }} ZENDESK_API_TOKEN: ${{ secrets.ZENDESK_API_TOKEN }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - DISABLE_AUTOUPDATER: "1" DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} ZENDESK_QUERY: ${{ github.event.inputs.query }} ZENDESK_TRIAGE_MODEL: ${{ vars.ZENDESK_TRIAGE_MODEL }} run: | - if [ "$BACKEND" = "api" ]; then - unset CLAUDE_CODE_OAUTH_TOKEN - else - unset ANTHROPIC_API_KEY - fi mkdir -p .triage-state python zendesk_triage/triage.py \ - --backend "$BACKEND" \ + --backend api \ --window-hours "${{ steps.cfg.outputs.window }}" \ --max-tickets "${{ steps.cfg.outputs.max }}" \ --state .triage-state/seen.json diff --git a/README.md b/README.md index b528d76..e0714da 100644 --- a/README.md +++ b/README.md @@ -131,41 +131,14 @@ Two caveats worth knowing: | `ZENDESK_SUBDOMAIN` | Zendesk subdomain (`mycompany` → `mycompany.zendesk.com`) | | `ZENDESK_EMAIL` | Agent email used for Zendesk API-token auth | | `ZENDESK_API_TOKEN` | Zendesk API token | -| **One Claude credential** | `ANTHROPIC_API_KEY` **or** `CLAUDE_CODE_OAUTH_TOKEN` — see [Claude authentication](#claude-authentication) | +| `ANTHROPIC_API_KEY` | Claude API key — see [Claude authentication](#claude-authentication) | | `DISCORD_WEBHOOK_URL` | Discord webhook (reused from the failure-notification setup) | ### Claude Authentication -Classification can reach Claude two ways, and they need different credentials: +The workflow classifies through the Anthropic API (`--backend api`) with an `ANTHROPIC_API_KEY`. That keeps CI on an organization-owned credential that doesn't draw on any individual's subscription quota, and keeps the runner free of a ~270MB Claude Code download. The `claude-cli` backend stays in the script for [local runs](#local-testing) and is not used in CI. -| Backend | Credential | Billing | -| ------- | ---------- | ------- | -| `api` | `ANTHROPIC_API_KEY` from the [Claude Console](https://platform.claude.com) | Per token, to the Console organization | -| `claude-cli` | `CLAUDE_CODE_OAUTH_TOKEN` from `claude setup-token` | Draws on that subscription's usage limits | - -**The workflow picks for itself.** The `Select backend` step uses `api` when an `ANTHROPIC_API_KEY` secret exists and `claude-cli` otherwise, so adding that secret switches the job over with no edit to the workflow, and until then it keeps running on the subscription. The choice is echoed into the run log, and a `workflow_dispatch` run can force either backend to test one without touching secrets. - -Prefer the API key where you have one, but for credential reasons rather than cost: it's organization-owned, doesn't expire annually, and doesn't consume an individual's quota. The two are close on cost — a `claude -p` run used to carry ~25K tokens of Claude Code system prompt and tool definitions on top of the batch, but the [locked-down invocation](#how-the-claude-cli-invocation-is-locked-down) removes both, which took a two-ticket fixture from ~$0.29 to ~$0.015. - -#### Notes on the API key - -An API key only exists inside a **Claude Console organization** (`platform.claude.com`), which is separate from a claude.ai Pro/Max/Team/Enterprise subscription with its own membership and billing — a claude.ai admin console has no API keys at all. If nobody can find one, the likely answer is that no Console organization exists yet rather than a permissions problem. - -#### Notes on the subscription token - -Generate it on a machine where you're logged into Claude Code: - -``` -claude setup-token -``` - -It runs the browser authorization flow and prints the token once — it is not saved anywhere. Requires a Pro, Max, Team, or Enterprise plan; see [Generate a long-lived token](https://code.claude.com/docs/en/authentication#generate-a-long-lived-token). Then: - -- **The token lasts one year.** It expires silently from the workflow's point of view — the run just fails to authenticate. Put the renewal date somewhere you'll see it. -- **The two credentials must not both be live in the run step.** An API key [outranks the OAuth token](https://code.claude.com/docs/en/authentication#authentication-precedence) in Claude Code's credential precedence, and in `-p` mode a key that is present is always used, so a `claude-cli` run with a key in scope would quietly bill the API instead. The workflow unsets whichever credential the chosen backend doesn't use. -- **Runs draw on that subscription's usage limits**, not API credits, and the token is tied to whoever minted it — so a scheduled run competes with that person's own interactive Claude Code usage. Hitting a limit fails the run (the tickets stay eligible and get picked up by the next one, per the dedup rules above). If your organization has [usage credits](https://support.claude.com/en/articles/12429409-manage-usage-credits-for-paid-claude-plans) enabled, usage continues past the allowance at standard API rates instead of stopping, which turns a failed run into a billed one. - > A [separate monthly Agent SDK credit](https://support.claude.com/en/articles/15036540-use-the-claude-agent-sdk-with-your-claude-plan) was announced for 2026-06-15 and then **paused** — `claude -p` still draws on subscription limits as described here. Worth re-reading that page before assuming otherwise. -- **The CLI must be v2.1.205 or newer** for `--json-schema`. The workflow installs the `stable` channel and echoes `claude --version` into the run log, because the script's "no structured_output" error points here. +If you go looking for that key and can't find one: an API key only exists inside a **Claude Console organization** (`platform.claude.com`), which is a separate organization from a claude.ai Pro/Max/Team/Enterprise subscription, with its own membership and billing. A claude.ai admin console has no API keys in it at all, so the usual answer is that no Console organization exists yet rather than that you're missing a permission. ### Optional Configuration @@ -176,7 +149,7 @@ It runs the browser authorization flow and prints the token once — it is not s | `--state-retention-days` | flag | `30` | Forget state entries older than N days | | `ZENDESK_QUERY` | env / `--query` | *(unset)* | Explicit Zendesk search query. Overrides `--window-hours` entirely | | `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `opus` | Model alias (`opus`, `sonnet`, `haiku`) or a full id. Set it to `sonnet` to reduce cost on large batches. On `--backend api` the alias is mapped to an id by `API_MODEL_ALIASES` | -| `backend` | workflow input / `--backend` | `auto` (workflow) / `claude-cli` (flag) | `api`, `claude-cli`, or `file`. The workflow's `auto` resolves to `api` when an `ANTHROPIC_API_KEY` secret exists — see [Claude authentication](#claude-authentication) | +| `--backend` | flag | `claude-cli` (flag) / `api` (workflow) | Where classification happens: `claude-cli` for local runs, `api` for CI, or `file` to render findings classified elsewhere | | `--max-tickets` | workflow input / flag | `1000` (workflow) / `100` (flag) | Runaway guard on tickets analyzed per run, **not** a batch size. The workflow passes `1000`; a bare `python triage.py` uses the script's own `DEFAULT_MAX_TICKETS` of `100`. Zendesk's search API caps a query at 1000 results, so higher values don't fetch more | | `--batch-size` | flag | `400` | Split batches larger than this across multiple requests | | `--review-star-floor` | flag | `3` | Classify app-store reviews at or below N stars; count the rest | @@ -245,27 +218,32 @@ Offline tests covering the window arithmetic, dedup partitioning, state round-tr ### Local Testing -The default backend reuses your own Claude Code login, so no Claude credential is needed locally — only the Zendesk ones: +Locally the default backend is `claude-cli`, which reuses your own Claude Code login — so no Claude credential is needed, only the Zendesk ones: ``` pip install -r zendesk_triage/requirements.txt export ZENDESK_SUBDOMAIN=... ZENDESK_EMAIL=... ZENDESK_API_TOKEN=... -# fetch + classify, print the Discord payload, post nothing +# fetch + classify through your Claude Code login, print the payload, post nothing python zendesk_triage/triage.py --window-hours 48 --dry-run -# exercise the path CI uses once an API key is configured -export ANTHROPIC_API_KEY=... -python zendesk_triage/triage.py --backend api --window-hours 48 --dry-run +# exercise exactly what CI runs +ANTHROPIC_API_KEY=... python zendesk_triage/triage.py --backend api --window-hours 48 --dry-run # or dump the batch, classify it by hand, and feed the findings back python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 python zendesk_triage/triage.py --backend file --findings /tmp/findings.json --dry-run ``` -### How the `claude-cli` invocation is locked down +Two things about the local `claude-cli` path: + +- **It spends your own subscription usage**, shared with your interactive Claude Code and chat usage — one invocation per chunk, so a 48h window is a single call (~$0.015 of equivalent usage on a small batch). +- **An exported `ANTHROPIC_API_KEY` silently takes over.** It [outranks your login](https://code.claude.com/docs/en/authentication#authentication-precedence) in Claude Code's credential precedence, and in `-p` mode a key that is present is always used — so with one exported in your shell, `--backend claude-cli` bills the API rather than using your subscription. `unset ANTHROPIC_API_KEY` if you want the subscription path. +- **It needs Claude Code v2.1.205 or newer** for `--json-schema`. On an older CLI the run exits with "no structured_output" naming that version; check `claude --version`. + +#### How the `claude-cli` invocation is locked down -Ticket text is written by strangers and the runner has a checkout of this repo, so the CLI is invoked with as little around it as possible: our own `--system-prompt` in place of Claude Code's, `--setting-sources ""` (no hooks, plugins, skills, allow-rules or `CLAUDE.md` from either the runner or the repo), `--strict-mcp-config` with no config (no MCP servers), and an explicit `--disallowed-tools` list. A session then exposes one tool, `StructuredOutput`, and no MCP servers. Removing the agent preamble and the tool definitions is also what makes this path cheap. +Ticket text is written by strangers, so the CLI is invoked with as little around it as possible: our own `--system-prompt` in place of Claude Code's, `--setting-sources ""` (no hooks, plugins, skills, allow-rules or `CLAUDE.md` from either your machine or this repo), `--strict-mcp-config` with no config (no MCP servers), and an explicit `--disallowed-tools` list. A session then exposes one tool, `StructuredOutput`, and no MCP servers. Removing the agent preamble and the tool definitions is also what makes this path cheap. Three findings from `v2.1.218` that explain why it's written that way — all worth re-testing after a CLI upgrade: diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 8c8ccf3..0a7f209 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -21,11 +21,11 @@ | question | feature_request | other Two ways to reach Claude, chosen with --backend: - api the Anthropic API, with an ANTHROPIC_API_KEY (org-owned credential, - per-token billing, structured outputs enforced by the API) - claude-cli the local `claude` CLI, authenticating as Claude Code — your own - login locally, or a CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token` - in CI. No API key, but it draws on that subscription's usage limits. + api the Anthropic API, with an ANTHROPIC_API_KEY. What CI runs: an + org-owned credential, per-token billing, schema enforced by the API. + claude-cli the local `claude` CLI, authenticating as your own Claude Code + login. The default for local runs — no API key needed, but it + spends your subscription's usage rather than being billed. Config (env vars, or CLI flags for local runs): ZENDESK_SUBDOMAIN e.g. "mycompany" -> https://mycompany.zendesk.com From a1bb944fd52bb32147ef65c2d3d90f4836190b34 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 08:30:31 +0200 Subject: [PATCH 09/16] feat: pin the triage model to claude-opus-5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the `opus` alias with the id. An alias resolves to whatever the credential's newest Opus is, which would move severity calibration and cluster labels on someone else's release schedule — not what an unattended daily digest wants, since no one is watching a run to notice the shift. ZENDESK_TRIAGE_MODEL and --model stay as overrides and still accept aliases (sonnet for a large backfill), so API_MODEL_ALIASES stays too. The default lives in the script rather than in a repo variable so there is one place to change it; the workflow's vars.ZENDESK_TRIAGE_MODEL can stay unset. Documents why Opus and why pinned, with the cost figures that make the tier choice a non-question at this volume. Verified the CLI accepts the full id on a subscription login (modelUsage reports claude-opus-5), so the local default works too, and the test now asserts the invariant that matters: DEFAULT_MODEL must resolve to an API id, whether it's a pin or an alias. --- README.md | 13 +++++++++++-- zendesk_triage/test_triage.py | 9 ++++++--- zendesk_triage/triage.py | 29 ++++++++++++++++++----------- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e0714da..46f7c9f 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ If you go looking for that key and can't find one: an API key only exists inside | `--state` | flag | *(unset)* | Dedup state file. The workflow points this at the cached `.triage-state/seen.json` | | `--state-retention-days` | flag | `30` | Forget state entries older than N days | | `ZENDESK_QUERY` | env / `--query` | *(unset)* | Explicit Zendesk search query. Overrides `--window-hours` entirely | -| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `opus` | Model alias (`opus`, `sonnet`, `haiku`) or a full id. Set it to `sonnet` to reduce cost on large batches. On `--backend api` the alias is mapped to an id by `API_MODEL_ALIASES` | +| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `claude-opus-5` | Overrides the model. Takes a full id, or an alias (`opus`, `sonnet`, `haiku`) which `--backend api` maps to an id via `API_MODEL_ALIASES`. **Leave it unset for normal operation** — the default lives in the script so there's one place to change it | | `--backend` | flag | `claude-cli` (flag) / `api` (workflow) | Where classification happens: `claude-cli` for local runs, `api` for CI, or `file` to render findings classified elsewhere | | `--max-tickets` | workflow input / flag | `1000` (workflow) / `100` (flag) | Runaway guard on tickets analyzed per run, **not** a batch size. The workflow passes `1000`; a bare `python triage.py` uses the script's own `DEFAULT_MAX_TICKETS` of `100`. Zendesk's search API caps a query at 1000 results, so higher values don't fetch more | | `--batch-size` | flag | `400` | Split batches larger than this across multiple requests | @@ -157,7 +157,16 @@ If you go looking for that key and can't find one: an API key only exists inside | `--no-hydrate` | flag | off | Skip fetching comments for content-free tickets | | `--effort` | flag | `medium` | Claude reasoning effort (`low`–`max`) | -> **Why an alias and not a pinned model id:** the alias is resolved at run time against whatever the authenticated plan allows, so a new Opus release needs no edit here, and a plan without Opus access falls back rather than failing on an id it can't serve. Pin a full id (`claude-opus-4-8`) only when you need a specific version — for reproducing a past run, say. +#### Why this model, and why pinned + +**Opus**, because the hard part of this job isn't per-ticket classification — enum-constrained categories with prompt guidance is squarely mid-tier work. It's the two batch-wide fields: `cluster` has to spot that a German app-store review and an English bug report describe one root cause, and `priority_rank` has to stay consistent across the whole batch. Those need the model to hold ~45 heterogeneous tickets in mind at once. The exact-transcription requirement (a 66-character Session ID copied verbatim) points the same way. And the entire job costs **single-digit dollars a month** on any current model — roughly $10 on Opus 5 against $6 on Sonnet 5 and $2 on Haiku 4.5 — so trading classification quality for a few dollars would be optimising the wrong thing when the cost of a miss is an unseen abuse report. + +**Pinned to an id rather than the `opus` alias**, because this is an unattended digest. An alias resolves to the newest Opus the credential allows, so severity calibration and cluster labels would shift on someone else's release schedule, with no run in between to notice it. Bumping the pin is a deliberate one-line change in [triage.py](zendesk_triage/triage.py) (`DEFAULT_MODEL`). + +Two cases for overriding it: + +- **Large backfills.** A `reset_state` run at `--max-tickets 1000` chunks into 400-ticket requests, where Opus latency and spend actually show up and cross-chunk cluster fidelity is already reduced by design. `ZENDESK_TRIAGE_MODEL=sonnet` for those. +- **Never Fable 5.** It prices above Opus tier, targets long-horizon agentic reasoning, and requires 30-day data retention — all wrong for batch classification of support tickets. #### Batch size vs. ticket cap diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index cdc6123..62f6de8 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -830,9 +830,12 @@ def test_every_alias_maps_to_an_id(self): self.assertEqual(triage.resolve_api_model(alias), model_id) self.assertTrue(model_id.startswith("claude-"), model_id) - def test_the_default_model_is_mappable(self): - """DEFAULT_MODEL is an alias, so --backend api would 404 without an entry.""" - self.assertIn(triage.DEFAULT_MODEL, triage.API_MODEL_ALIASES) + def test_the_default_model_resolves_to_an_api_id(self): + """--backend api 404s on a Claude Code alias, so whatever DEFAULT_MODEL is — + a pinned id today, an alias if that ever changes — it has to resolve to one.""" + resolved = triage.resolve_api_model(triage.DEFAULT_MODEL) + self.assertNotIn(resolved, triage.API_MODEL_ALIASES) + self.assertTrue(resolved.startswith("claude-"), resolved) def test_a_full_id_passes_through(self): self.assertEqual(triage.resolve_api_model("claude-opus-4-8"), "claude-opus-4-8") diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 0a7f209..17d43af 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -34,7 +34,9 @@ ANTHROPIC_API_KEY Claude API key, for --backend api (read by the SDK itself) DISCORD_WEBHOOK_URL Discord incoming webhook ZENDESK_QUERY (optional) Zendesk search query; see DEFAULT_QUERY - ZENDESK_TRIAGE_MODEL (optional) Claude model alias or id; defaults to `opus` + ZENDESK_TRIAGE_MODEL (optional) Claude model id or alias; defaults to + claude-opus-5. Set it to override, e.g. `sonnet` for a + large backfill. Usage: # real run (CI): reads everything from the environment @@ -105,14 +107,18 @@ def window_label(hours): days = hours // 24 return f"created in the past {days} day{'s' if days > 1 else ''}" return f"created in the past {hours}h" -# An alias, not a pinned id: the CLI resolves `opus` to the newest Opus the -# authenticated plan allows, so a model release needs no edit here and a plan -# without Opus access degrades instead of 404-ing on a dead id. -DEFAULT_MODEL = "opus" -# The Anthropic API takes model ids, not Claude Code aliases, so the same alias has -# to be mapped for --backend api. Each entry is the newest model in its family, -# which is what the CLI's own alias resolution lands on — pass a full id to pin a -# specific version instead. +# A pinned id rather than the `opus` alias, deliberately. This is an unattended +# digest a human skims: the batch-wide fields (`cluster`, `priority_rank`) and the +# severity calibration shift when the model underneath changes, and an alias would +# move them on someone else's release schedule. Opus rather than a cheaper tier +# because clustering asks the model to recognise one root cause across 45 tickets in +# several languages, and the whole job costs single-digit dollars a month either way. +# Bumping this is a one-line, deliberate change; both backends accept a full id. +DEFAULT_MODEL = "claude-opus-5" +# Aliases still work as an override (ZENDESK_TRIAGE_MODEL=sonnet for a big backfill), +# and the Anthropic API takes ids only — so --backend api maps them here. Each entry +# is the newest model in its family, which is where the CLI's own alias resolution +# lands; a full id passes through untouched. API_MODEL_ALIASES = { "opus": "claude-opus-5", "sonnet": "claude-sonnet-5", @@ -1125,8 +1131,9 @@ def main(): parser.add_argument("--window-hours", type=int, metavar="N", help="Only analyze unsolved tickets created in the last N hours. " "The scheduled daily run uses 48.") - parser.add_argument("--model", help="Claude model alias (opus, sonnet, haiku) or full " - "id (else ZENDESK_TRIAGE_MODEL, else opus).") + parser.add_argument("--model", help=f"Claude model id, or an alias (opus, sonnet, " + f"haiku) which --backend api maps to an id " + f"(else ZENDESK_TRIAGE_MODEL, else {DEFAULT_MODEL}).") parser.add_argument("--effort", default="medium", choices=["low", "medium", "high", "xhigh", "max"], help="Claude reasoning effort (default: medium).") parser.add_argument("--max-tickets", type=int, default=DEFAULT_MAX_TICKETS, From 58dd974bed691f8f6b8cffcf4815f9ec46e1b101 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 08:46:44 +0200 Subject: [PATCH 10/16] refactor: drop the claude-cli classification path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One way to reach Claude instead of two. The CLI path existed because there was no API key; now that there is one, keeping it meant maintaining a second classification path, its lockdown flags, and a deny list that needs re-checking on every CLI upgrade — for a path nothing runs. Gone with it: analyze_via_claude_cli, findings_from_cli_envelope, the CLI_ISOLATION_ARGS/CLI_DENIED_TOOLS pair, extract_json_object (only the CLI path parsed JSON out of prose), the subprocess import, and 15 tests. --backend keeps `api` (now the default, so the workflow stops passing it) and `file` for rendering findings classified elsewhere. The model shorthands stay: ZENDESK_TRIAGE_MODEL=sonnet is documented for backfills and would otherwise 404 as a bare alias. Verified with a local dry run against the API on real tickets, and 158 tests pass. --- .github/workflows/zendesk_triage.yml | 5 - README.md | 37 ++---- zendesk_triage/test_triage.py | 90 -------------- zendesk_triage/triage.py | 169 ++++----------------------- 4 files changed, 29 insertions(+), 272 deletions(-) diff --git a/.github/workflows/zendesk_triage.yml b/.github/workflows/zendesk_triage.yml index a7d30ac..eccd46b 100644 --- a/.github/workflows/zendesk_triage.yml +++ b/.github/workflows/zendesk_triage.yml @@ -97,10 +97,6 @@ jobs: fi echo "Window: ${window}h, max tickets: ${max}" - # Classification goes through the Anthropic API here, never the `claude` CLI: - # CI gets an org-owned credential that doesn't draw on an individual's - # subscription quota, and the runner needs no 270MB CLI download. The - # claude-cli backend stays in the script for local runs — see the README. - name: Run triage env: ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }} @@ -113,7 +109,6 @@ jobs: run: | mkdir -p .triage-state python zendesk_triage/triage.py \ - --backend api \ --window-hours "${{ steps.cfg.outputs.window }}" \ --max-tickets "${{ steps.cfg.outputs.max }}" \ --state .triage-state/seen.json diff --git a/README.md b/README.md index 46f7c9f..5766103 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Two caveats worth knowing: ### Claude Authentication -The workflow classifies through the Anthropic API (`--backend api`) with an `ANTHROPIC_API_KEY`. That keeps CI on an organization-owned credential that doesn't draw on any individual's subscription quota, and keeps the runner free of a ~270MB Claude Code download. The `claude-cli` backend stays in the script for [local runs](#local-testing) and is not used in CI. +Classification goes through the Anthropic API with an `ANTHROPIC_API_KEY`, in CI and locally alike — an organization-owned credential that doesn't draw on any individual's subscription quota. If you go looking for that key and can't find one: an API key only exists inside a **Claude Console organization** (`platform.claude.com`), which is a separate organization from a claude.ai Pro/Max/Team/Enterprise subscription, with its own membership and billing. A claude.ai admin console has no API keys in it at all, so the usual answer is that no Console organization exists yet rather than that you're missing a permission. @@ -149,7 +149,7 @@ If you go looking for that key and can't find one: an API key only exists inside | `--state-retention-days` | flag | `30` | Forget state entries older than N days | | `ZENDESK_QUERY` | env / `--query` | *(unset)* | Explicit Zendesk search query. Overrides `--window-hours` entirely | | `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `claude-opus-5` | Overrides the model. Takes a full id, or an alias (`opus`, `sonnet`, `haiku`) which `--backend api` maps to an id via `API_MODEL_ALIASES`. **Leave it unset for normal operation** — the default lives in the script so there's one place to change it | -| `--backend` | flag | `claude-cli` (flag) / `api` (workflow) | Where classification happens: `claude-cli` for local runs, `api` for CI, or `file` to render findings classified elsewhere | +| `--backend` | flag | `api` | `file` instead renders a findings JSON classified elsewhere, skipping the model — pair with `--findings` | | `--max-tickets` | workflow input / flag | `1000` (workflow) / `100` (flag) | Runaway guard on tickets analyzed per run, **not** a batch size. The workflow passes `1000`; a bare `python triage.py` uses the script's own `DEFAULT_MAX_TICKETS` of `100`. Zendesk's search API caps a query at 1000 results, so higher values don't fetch more | | `--batch-size` | flag | `400` | Split batches larger than this across multiple requests | | `--review-star-floor` | flag | `3` | Classify app-store reviews at or below N stars; count the rest | @@ -227,43 +227,24 @@ Offline tests covering the window arithmetic, dedup partitioning, state round-tr ### Local Testing -Locally the default backend is `claude-cli`, which reuses your own Claude Code login — so no Claude credential is needed, only the Zendesk ones: +Local runs use the same Anthropic API path as CI, so they need an `ANTHROPIC_API_KEY` alongside the Zendesk credentials. `--dry-run` prints the Discord payload instead of posting, so no webhook is needed: ``` pip install -r zendesk_triage/requirements.txt -export ZENDESK_SUBDOMAIN=... ZENDESK_EMAIL=... ZENDESK_API_TOKEN=... +export ZENDESK_SUBDOMAIN=... ZENDESK_EMAIL=... ZENDESK_API_TOKEN=... ANTHROPIC_API_KEY=... -# fetch + classify through your Claude Code login, print the payload, post nothing +# what CI runs, minus the Discord post and the state file python zendesk_triage/triage.py --window-hours 48 --dry-run -# exercise exactly what CI runs -ANTHROPIC_API_KEY=... python zendesk_triage/triage.py --backend api --window-hours 48 --dry-run +# keep it cheap while iterating on the rendering +python zendesk_triage/triage.py --window-hours 12 --max-tickets 5 --dry-run -# or dump the batch, classify it by hand, and feed the findings back +# or take the model out of the loop: dump the batch, classify it by hand, +# and feed the findings back in to render python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 python zendesk_triage/triage.py --backend file --findings /tmp/findings.json --dry-run ``` -Two things about the local `claude-cli` path: - -- **It spends your own subscription usage**, shared with your interactive Claude Code and chat usage — one invocation per chunk, so a 48h window is a single call (~$0.015 of equivalent usage on a small batch). -- **An exported `ANTHROPIC_API_KEY` silently takes over.** It [outranks your login](https://code.claude.com/docs/en/authentication#authentication-precedence) in Claude Code's credential precedence, and in `-p` mode a key that is present is always used — so with one exported in your shell, `--backend claude-cli` bills the API rather than using your subscription. `unset ANTHROPIC_API_KEY` if you want the subscription path. -- **It needs Claude Code v2.1.205 or newer** for `--json-schema`. On an older CLI the run exits with "no structured_output" naming that version; check `claude --version`. - -#### How the `claude-cli` invocation is locked down - -Ticket text is written by strangers, so the CLI is invoked with as little around it as possible: our own `--system-prompt` in place of Claude Code's, `--setting-sources ""` (no hooks, plugins, skills, allow-rules or `CLAUDE.md` from either your machine or this repo), `--strict-mcp-config` with no config (no MCP servers), and an explicit `--disallowed-tools` list. A session then exposes one tool, `StructuredOutput`, and no MCP servers. Removing the agent preamble and the tool definitions is also what makes this path cheap. - -Three findings from `v2.1.218` that explain why it's written that way — all worth re-testing after a CLI upgrade: - -- **`--disallowed-tools "*"` can't be used**, tempting as it is. It empties the surface, but `--json-schema` is itself implemented as a `StructuredOutput` tool, so the wildcard denies that too and the run returns prose with no `structured_output`. Allow-listing `StructuredOutput` alongside the wildcard leaves the tool present but still doesn't produce structured output. -- **`--permission-mode dontAsk` is not a boundary.** A session with no allow rules still ran `Bash(echo …)`, because the mode permits a read-only command set. It's kept as a backstop, not as the control. -- **The deny list is therefore by name, and will go stale** as tools are added. Naming only the obvious ones (`Bash`, `Read`, `Write`, …) left 19 others live, including several with outward side effects. To see what a session really exposes, read the `init` event: `echo hi | claude -p --output-format stream-json --verbose [flags] | grep '"subtype":"init"'`. - -One more, on the flag not used: - -- **Do not add `--bare`.** It's otherwise the right flag for a scripted call (it skips hook, plugin, MCP and `CLAUDE.md` discovery, so the runner behaves the same as your laptop), but bare mode reads `ANTHROPIC_API_KEY` or an `apiKeyHelper` **only** — it never touches OAuth credentials, which is exactly what both CI and your local login use. See [bare mode](https://code.claude.com/docs/en/headless#start-faster-with-bare-mode); the docs say it will become the default for `-p` in a future release, so this is worth re-checking on CLI upgrades. - ## Workflow Failure Notificaiton If a workflow fails and is in the list of workflows monitored by the failure notificaiton workflow, the failure notificaiton workflow will send a message to a discord webhook. diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index 62f6de8..7c8a12a 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -791,37 +791,6 @@ def test_a_non_object_entry_exits(self): triage.tickets_from_payload({"tickets": [["not", "an", "object"]]}, "x") -class TestCliIsolation(unittest.TestCase): - """The claude-cli invocation is locked down because ticket text is untrusted and - the runner has a checkout. Each of these fails silently if broken: a wrong deny - list returns prose instead of findings, a missing flag loads the repo's config.""" - - def test_structured_output_is_never_denied(self): - """--json-schema is implemented as the StructuredOutput tool, so denying it — - or passing a `*` wildcard — makes the run return prose and no findings.""" - denied = triage.CLI_DENIED_TOOLS.split() - self.assertNotIn("StructuredOutput", denied) - self.assertNotIn("*", denied) - - def test_tools_with_side_effects_are_denied(self): - denied = triage.CLI_DENIED_TOOLS.split() - for tool in ("Bash", "Write", "Edit", "WebFetch", "WebSearch", "Task"): - self.assertIn(tool, denied) - - def test_the_system_prompt_travels_as_a_flag(self): - """Not on stdin with the tickets: stdin is untrusted input, the prompt isn't.""" - self.assertIn("--system-prompt", triage.CLI_ISOLATION_ARGS) - self.assertIn(triage.SYSTEM_PROMPT, triage.CLI_ISOLATION_ARGS) - - def test_no_setting_sources_are_loaded(self): - """An empty value is what keeps hooks, plugins, skills and CLAUDE.md out.""" - args = triage.CLI_ISOLATION_ARGS - self.assertEqual(args[args.index("--setting-sources") + 1], "") - - def test_mcp_config_is_strict(self): - self.assertIn("--strict-mcp-config", triage.CLI_ISOLATION_ARGS) - - class TestResolveApiModel(unittest.TestCase): """The CLI resolves aliases itself; the API takes ids, so only that path maps.""" @@ -845,65 +814,6 @@ def test_an_unknown_value_passes_through(self): self.assertEqual(triage.resolve_api_model("claude-future-9"), "claude-future-9") -class TestFindingsFromCliEnvelope(unittest.TestCase): - def envelope(self, **overrides): - base = { - "subtype": "success", - "is_error": False, - "structured_output": {"tickets": [finding(1)]}, - } - base.update(overrides) - return base - - def test_reads_structured_output(self): - self.assertEqual(triage.findings_from_cli_envelope(self.envelope()), [finding(1)]) - - def test_missing_structured_output_exits(self): - """A CLI too old for --json-schema returns prose in `result` and no - structured_output; without this check the digest comes out silently empty.""" - stale = self.envelope(result='{"tickets": []}') - del stale["structured_output"] - with self.assertRaises(SystemExit): - triage.findings_from_cli_envelope(stale) - - def test_a_reported_cli_error_exits(self): - for envelope in (self.envelope(is_error=True), - self.envelope(subtype="error_max_turns")): - with self.assertRaises(SystemExit): - triage.findings_from_cli_envelope(envelope) - - def test_a_cost_field_is_reported_not_fatal(self): - result = triage.findings_from_cli_envelope(self.envelope(total_cost_usd=0.42)) - self.assertEqual(result, [finding(1)]) - - -class TestExtractJsonObject(unittest.TestCase): - def test_bare_object(self): - self.assertEqual(triage.extract_json_object('{"a": 1}'), {"a": 1}) - - def test_object_inside_a_markdown_fence(self): - self.assertEqual(triage.extract_json_object('```json\n{"a": 1}\n```'), {"a": 1}) - - def test_object_surrounded_by_prose(self): - self.assertEqual( - triage.extract_json_object('Sure! Here you go:\n{"a": 1}\nHope that helps.'), {"a": 1} - ) - - def test_nested_braces_survive(self): - self.assertEqual( - triage.extract_json_object('{"t": [{"id": 1}, {"id": 2}]}'), - {"t": [{"id": 1}, {"id": 2}]}, - ) - - def test_no_object_exits(self): - with self.assertRaises(SystemExit): - triage.extract_json_object("no json here") - - def test_malformed_object_exits(self): - with self.assertRaises(SystemExit): - triage.extract_json_object('{"a": }') - - class TestAnalyzeInChunks(unittest.TestCase): """A batch of 2000 would need ~204K output tokens, past the 128K ceiling, so oversized batches must split rather than truncate.""" diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 17d43af..4e39bd1 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -20,19 +20,15 @@ bug_report | low_star_review | legal_request | security_or_legislation | question | feature_request | other -Two ways to reach Claude, chosen with --backend: - api the Anthropic API, with an ANTHROPIC_API_KEY. What CI runs: an - org-owned credential, per-token billing, schema enforced by the API. - claude-cli the local `claude` CLI, authenticating as your own Claude Code - login. The default for local runs — no API key needed, but it - spends your subscription's usage rather than being billed. - -Config (env vars, or CLI flags for local runs): +Classification goes through the Anthropic API, with structured outputs enforcing +SCHEMA. --backend file skips it entirely and renders findings produced elsewhere. + +Config (env vars, or flags for local runs): ZENDESK_SUBDOMAIN e.g. "mycompany" -> https://mycompany.zendesk.com ZENDESK_EMAIL agent email for API token auth ZENDESK_API_TOKEN Zendesk API token - ANTHROPIC_API_KEY Claude API key, for --backend api (read by the SDK itself) - DISCORD_WEBHOOK_URL Discord incoming webhook + ANTHROPIC_API_KEY Claude API key (read by the SDK itself) + DISCORD_WEBHOOK_URL Discord incoming webhook (not needed with --dry-run) ZENDESK_QUERY (optional) Zendesk search query; see DEFAULT_QUERY ZENDESK_TRIAGE_MODEL (optional) Claude model id or alias; defaults to claude-opus-5. Set it to override, e.g. `sonnet` for a @@ -51,9 +47,6 @@ # or an explicit query, which overrides --window-hours python triage.py --query "type:ticket status:open tags:bug" --max-tickets 50 - # classify through the Anthropic API instead of Claude Code - python triage.py --backend api --dry-run - # split classification out entirely: dump the batch, classify it by hand, # feed the findings back in to render python triage.py --dump-batch /tmp/batch.json --max-tickets 20 @@ -64,7 +57,6 @@ import math import os import re -import subprocess import sys import textwrap import time @@ -115,10 +107,10 @@ def window_label(hours): # several languages, and the whole job costs single-digit dollars a month either way. # Bumping this is a one-line, deliberate change; both backends accept a full id. DEFAULT_MODEL = "claude-opus-5" -# Aliases still work as an override (ZENDESK_TRIAGE_MODEL=sonnet for a big backfill), -# and the Anthropic API takes ids only — so --backend api maps them here. Each entry -# is the newest model in its family, which is where the CLI's own alias resolution -# lands; a full id passes through untouched. +# Shorthands for the override, so ZENDESK_TRIAGE_MODEL=sonnet works for a big +# backfill without anyone looking up an id. The API takes ids only, so they are +# mapped here; each is the newest model in its family, and a full id passes through +# untouched. API_MODEL_ALIASES = { "opus": "claude-opus-5", "sonnet": "claude-sonnet-5", @@ -636,9 +628,9 @@ def compact_ticket(ticket): def resolve_api_model(model): - """Map a Claude Code model alias onto the id the Anthropic API expects. + """Map a shorthand model name onto the id the Anthropic API expects. - Anything that isn't a known alias passes through untouched, so a pinned id + Anything that isn't a known shorthand passes through untouched, so a pinned id (`claude-opus-4-8`) or a model newer than this table still works. """ return API_MODEL_ALIASES.get(model, model) @@ -652,18 +644,6 @@ def build_analysis_prompt(compact_tickets): ) -def extract_json_object(text): - """Pull the outermost JSON object out of model prose (tolerates code fences).""" - start = text.find("{") - end = text.rfind("}") - if start == -1 or end <= start: - sys.exit(f"No JSON object found in the model output:\n{text[:500]}") - try: - return json.loads(text[start : end + 1]) - except json.JSONDecodeError as exc: - sys.exit(f"Model output was not valid JSON ({exc}):\n{text[start : start + 500]}") - - REQUIRED_FINDING_KEYS = ("id", "category", "severity") @@ -687,9 +667,9 @@ def validate_findings(findings, label): def tickets_from_payload(payload, source): """Pull the `tickets` list out of a classification payload, or exit clearly. - Structured outputs guarantee the key and the item shape on the API path, but the - CLI path has neither — a bare KeyError mid-render is a confusing way to learn - that, so both paths are validated here before anything renders them. + Structured outputs guarantee the key and the item shape, so on a normal run this + never fires; it is the guard for --backend file, whose findings nothing validates, + and a bare KeyError mid-render is a confusing way to learn a key is missing. """ found = payload.get("tickets") if isinstance(payload, dict) else None if not isinstance(found, list): @@ -698,111 +678,6 @@ def tickets_from_payload(payload, source): return validate_findings(found, source) -# Ticket text is untrusted input written by strangers, and the runner has a checkout -# of this repo, so the CLI invocation is stripped to just "classify this text": -# --system-prompt ours replaces Claude Code's, so there is no agent preamble -# and no per-machine section (cwd, env, git status) either -# --setting-sources "" loads no user/project/local settings, so no hooks, plugins, -# skills, allow-rules or CLAUDE.md from the runner or the repo -# --strict-mcp-config with no --mcp-config, that means no MCP servers at all -# --disallowed-tools the tools below, by name — see CLI_DENIED_TOOLS -# --permission-mode dontAsk, as a backstop rather than the boundary -# Everything Claude needs is in the prompt, so a ticket that tries to talk its way -# into running something should have nothing to reach for. -# --bare would give the same isolation and a faster start, but it reads -# ANTHROPIC_API_KEY only and never OAuth credentials, which is what this path uses. -# -# Two things measured on v2.1.218 that constrain how the tool surface is closed: -# - `--disallowed-tools "*"` does empty the surface, but it also denies -# StructuredOutput, which is how --json-schema is implemented — the run then -# returns prose and no structured_output. So the tools have to be named. -# - `--permission-mode dontAsk` is not a boundary on its own: a session with no -# allow rules still executed `Bash(echo …)`, because the mode permits a -# read-only command set. It stays as a backstop, not as the control. -# A named list goes stale as tools are added, so re-check what a session actually -# exposes after a CLI upgrade — the `init` event lists it: -# echo hi | claude -p --output-format stream-json --verbose [flags] \ -# | grep '"subtype":"init"' -CLI_DENIED_TOOLS = " ".join([ - # filesystem and execution - "Bash", "BashOutput", "KillShell", "Read", "Write", "Edit", "NotebookEdit", - "Glob", "Grep", - # network - "WebFetch", "WebSearch", - # delegation and session control - "Task", "TaskOutput", "TaskStop", "Workflow", "Skill", "SlashCommand", - "ToolSearch", "TodoWrite", "EnterWorktree", "ExitWorktree", "Monitor", - "ScheduleWakeup", - # anything that reaches outside the run - "Artifact", "SendMessage", "PushNotification", "RemoteTrigger", "DesignSync", - "CronCreate", "CronDelete", "CronList", "ShareOnboardingGuide", "ReportFindings", -]) -CLI_ISOLATION_ARGS = [ - "--system-prompt", SYSTEM_PROMPT, - "--setting-sources", "", - "--strict-mcp-config", - "--permission-mode", "dontAsk", - "--disallowed-tools", CLI_DENIED_TOOLS, -] - - -def findings_from_cli_envelope(envelope): - """Pull the findings out of a `claude -p --output-format json` envelope. - - With --json-schema the shape lands in `structured_output`, already validated - against SCHEMA. An envelope without that key would otherwise surface as an empty - digest, so it exits naming both causes: a CLI too old for the flag, or a reply - that ran out of output tokens before the JSON closed. - """ - if envelope.get("is_error") or envelope.get("subtype") != "success": - sys.exit(f"`claude` reported an error: {envelope.get('result') or envelope}") - cost = envelope.get("total_cost_usd") - if cost is not None: - print(f"claude CLI reported ${cost:.4f} for this batch.") - payload = envelope.get("structured_output") - if not isinstance(payload, dict): - sys.exit("`claude` returned no structured_output. Either the CLI predates " - "--json-schema (needs v2.1.205 or newer — check `claude --version`) " - f"or the batch outgrew the output ceiling: lower --batch-size " - f"(currently splitting at {DEFAULT_BATCH_SIZE}).") - return tickets_from_payload(payload, "`claude` CLI") - - -def analyze_via_claude_cli(model, effort, compact_tickets, timeout=1800): - """Classify the batch with the local `claude` CLI instead of the Anthropic API. - - It authenticates as Claude Code, so no ANTHROPIC_API_KEY is needed, and - --json-schema enforces the same SCHEMA the API path uses — so the response needs - no prose parsing and no hand-maintained field list in the prompt. - """ - cmd = ["claude", "-p", "--output-format", "json", - "--json-schema", json.dumps(SCHEMA), *CLI_ISOLATION_ARGS] - if model: - cmd += ["--model", model] - if effort: - cmd += ["--effort", effort] - try: - # Only the ticket payload goes over stdin — the instructions are the system - # prompt above. It goes over stdin rather than argv because a full batch can - # exceed the argv size limit. - proc = subprocess.run( - cmd, - input=build_analysis_prompt(compact_tickets), - capture_output=True, - text=True, - timeout=timeout, - ) - except FileNotFoundError: - sys.exit("`claude` not found on PATH. Install Claude Code " - "(https://code.claude.com/docs/en/setup), or use --backend api.") - except subprocess.TimeoutExpired: - sys.exit(f"`claude` timed out after {timeout}s. Try a smaller --max-tickets.") - if proc.returncode != 0: - sys.exit(f"`claude` failed ({proc.returncode}): {proc.stderr[:500]}") - - return findings_from_cli_envelope(extract_json_object(proc.stdout)) - - def dump_batch(path, compact_tickets, model): """Write the batch to disk so it can be classified by hand. Contains ticket text.""" payload = { @@ -1144,10 +1019,9 @@ def main(): f"(default: {DEFAULT_BATCH_SIZE}).") parser.add_argument("--dry-run", action="store_true", help="Fetch and analyze, then print the Discord payload instead of posting.") - parser.add_argument("--backend", default="claude-cli", choices=["claude-cli", "api", "file"], - help="Where classification happens: Claude Code (default, no API " - "key needed), the Anthropic API (needs ANTHROPIC_API_KEY), or " - "a findings file classified elsewhere.") + parser.add_argument("--backend", default="api", choices=["api", "file"], + help="Where classification happens: the Anthropic API (default), " + "or a findings file classified elsewhere.") parser.add_argument("--findings", help="Findings JSON to render instead of classifying (--backend file).") parser.add_argument("--review-star-floor", type=int, default=DEFAULT_REVIEW_STAR_FLOOR, @@ -1266,11 +1140,8 @@ def main(): print("Classify it, then: --backend file --findings --dry-run") return - if args.backend == "api": - client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY - analyzer = partial(analyze, client, resolve_api_model(model), args.effort) - else: - analyzer = partial(analyze_via_claude_cli, model, args.effort) + client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY + analyzer = partial(analyze, client, resolve_api_model(model), args.effort) findings = analyze_in_chunks(analyzer, compact, args.batch_size) # Keep only findings whose id maps to a fetched ticket, in case of drift. From 43c2590fff6b94c4cbfd677e15b8d8fface3c08b Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 08:53:19 +0200 Subject: [PATCH 11/16] refactor: --findings replaces the --backend flag --backend only had two values left, and one of them was inseparable from --findings: `--backend file --findings X` had to be passed together, with a validation branch to catch half of it. `--findings X` now says the whole thing on its own, and the flag it replaced is gone along with that check. Verified both branches: --findings renders a hand-written file with no Zendesk or Anthropic credentials in the environment, and a live dry run still fetches and classifies. --- README.md | 8 ++++---- zendesk_triage/test_triage.py | 4 ++-- zendesk_triage/triage.py | 25 ++++++++++--------------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 5766103..0863213 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Detection uses the Zendesk `via.channel`, which identified reviews with no false Twitter DM tickets arrive with `description` identical to `subject` — both just `"Conversation with "` — which is 15% of non-review tickets and unclassifiable as fetched. For those only, `hydrate_descriptions` fetches a page of up to 10 comments and joins every body that differs from the subject into the description; later replies often carry the actual detail. Hydration is an enrichment, so an HTTP error or an unreachable endpoint leaves the ticket as-is rather than failing the run (`--no-hydrate` to skip it entirely). -The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time window, classifies the whole batch in one schema-enforced request through the `claude` CLI, and posts Discord embeds: a summary embed plus one embed per highlighted ticket (linking to the ticket in Zendesk). +The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time window, classifies the whole batch in one schema-enforced request to the Anthropic API, and posts Discord embeds: a summary embed plus one embed per highlighted ticket (linking to the ticket in Zendesk). The summary embed accounts for the batch in full, so nothing is dropped silently: @@ -148,8 +148,8 @@ If you go looking for that key and can't find one: an API key only exists inside | `--state` | flag | *(unset)* | Dedup state file. The workflow points this at the cached `.triage-state/seen.json` | | `--state-retention-days` | flag | `30` | Forget state entries older than N days | | `ZENDESK_QUERY` | env / `--query` | *(unset)* | Explicit Zendesk search query. Overrides `--window-hours` entirely | -| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `claude-opus-5` | Overrides the model. Takes a full id, or an alias (`opus`, `sonnet`, `haiku`) which `--backend api` maps to an id via `API_MODEL_ALIASES`. **Leave it unset for normal operation** — the default lives in the script so there's one place to change it | -| `--backend` | flag | `api` | `file` instead renders a findings JSON classified elsewhere, skipping the model — pair with `--findings` | +| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `claude-opus-5` | Overrides the model. Takes a full id, or a shorthand (`opus`, `sonnet`, `haiku`) mapped to an id via `API_MODEL_ALIASES`. **Leave it unset for normal operation** — the default lives in the script so there's one place to change it | +| `--findings` | flag | *(unset)* | Render a findings JSON classified elsewhere, skipping Zendesk and Claude entirely. Pairs with `--dump-batch` | | `--max-tickets` | workflow input / flag | `1000` (workflow) / `100` (flag) | Runaway guard on tickets analyzed per run, **not** a batch size. The workflow passes `1000`; a bare `python triage.py` uses the script's own `DEFAULT_MAX_TICKETS` of `100`. Zendesk's search API caps a query at 1000 results, so higher values don't fetch more | | `--batch-size` | flag | `400` | Split batches larger than this across multiple requests | | `--review-star-floor` | flag | `3` | Classify app-store reviews at or below N stars; count the rest | @@ -242,7 +242,7 @@ python zendesk_triage/triage.py --window-hours 12 --max-tickets 5 --dry-run # or take the model out of the loop: dump the batch, classify it by hand, # and feed the findings back in to render python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 -python zendesk_triage/triage.py --backend file --findings /tmp/findings.json --dry-run +python zendesk_triage/triage.py --findings /tmp/findings.json --dry-run ``` ## Workflow Failure Notificaiton diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index 7c8a12a..41bc8ae 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -779,7 +779,7 @@ def test_an_empty_list_is_valid(self): self.assertEqual(triage.tickets_from_payload({"tickets": []}, "x"), []) def test_a_finding_missing_renderer_keys_exits(self): - """A hand-edited --backend file findings list has nothing enforcing its shape, + """A hand-edited --findings list has nothing enforcing its shape, so an entry without category/severity would KeyError in build_summary_embed.""" for entry in ({"id": 1}, {"id": 1, "category": "bug_report"}, {"category": "bug_report", "severity": "major"}): @@ -800,7 +800,7 @@ def test_every_alias_maps_to_an_id(self): self.assertTrue(model_id.startswith("claude-"), model_id) def test_the_default_model_resolves_to_an_api_id(self): - """--backend api 404s on a Claude Code alias, so whatever DEFAULT_MODEL is — + """The API 404s on a bare shorthand, so whatever DEFAULT_MODEL is — a pinned id today, an alias if that ever changes — it has to resolve to one.""" resolved = triage.resolve_api_model(triage.DEFAULT_MODEL) self.assertNotIn(resolved, triage.API_MODEL_ALIASES) diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 4e39bd1..9e3bf24 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -21,7 +21,7 @@ | question | feature_request | other Classification goes through the Anthropic API, with structured outputs enforcing -SCHEMA. --backend file skips it entirely and renders findings produced elsewhere. +SCHEMA. --findings skips it entirely and renders findings produced elsewhere. Config (env vars, or flags for local runs): ZENDESK_SUBDOMAIN e.g. "mycompany" -> https://mycompany.zendesk.com @@ -50,7 +50,7 @@ # split classification out entirely: dump the batch, classify it by hand, # feed the findings back in to render python triage.py --dump-batch /tmp/batch.json --max-tickets 20 - python triage.py --backend file --findings /tmp/findings.json --dry-run + python triage.py --findings /tmp/findings.json --dry-run """ import argparse import json @@ -105,7 +105,7 @@ def window_label(hours): # move them on someone else's release schedule. Opus rather than a cheaper tier # because clustering asks the model to recognise one root cause across 45 tickets in # several languages, and the whole job costs single-digit dollars a month either way. -# Bumping this is a one-line, deliberate change; both backends accept a full id. +# Bumping this is a one-line, deliberate change. DEFAULT_MODEL = "claude-opus-5" # Shorthands for the override, so ZENDESK_TRIAGE_MODEL=sonnet works for a big # backfill without anyone looking up an id. The API takes ids only, so they are @@ -668,7 +668,7 @@ def tickets_from_payload(payload, source): """Pull the `tickets` list out of a classification payload, or exit clearly. Structured outputs guarantee the key and the item shape, so on a normal run this - never fires; it is the guard for --backend file, whose findings nothing validates, + never fires; it is the guard for --findings, whose contents nothing validates, and a bare KeyError mid-render is a confusing way to learn a key is missing. """ found = payload.get("tickets") if isinstance(payload, dict) else None @@ -1007,7 +1007,7 @@ def main(): help="Only analyze unsolved tickets created in the last N hours. " "The scheduled daily run uses 48.") parser.add_argument("--model", help=f"Claude model id, or an alias (opus, sonnet, " - f"haiku) which --backend api maps to an id " + f"haiku) mapped to an id " f"(else ZENDESK_TRIAGE_MODEL, else {DEFAULT_MODEL}).") parser.add_argument("--effort", default="medium", choices=["low", "medium", "high", "xhigh", "max"], help="Claude reasoning effort (default: medium).") @@ -1019,11 +1019,9 @@ def main(): f"(default: {DEFAULT_BATCH_SIZE}).") parser.add_argument("--dry-run", action="store_true", help="Fetch and analyze, then print the Discord payload instead of posting.") - parser.add_argument("--backend", default="api", choices=["api", "file"], - help="Where classification happens: the Anthropic API (default), " - "or a findings file classified elsewhere.") - parser.add_argument("--findings", - help="Findings JSON to render instead of classifying (--backend file).") + parser.add_argument("--findings", metavar="PATH", + help="Render findings classified elsewhere, skipping Zendesk and " + "Claude entirely. Pairs with --dump-batch.") parser.add_argument("--review-star-floor", type=int, default=DEFAULT_REVIEW_STAR_FLOOR, metavar="N", help=f"Classify app-store reviews of N stars or fewer; count the rest " @@ -1045,9 +1043,6 @@ def main(): "hand-classification. WARNING: writes ticket content to disk.") args = parser.parse_args() - if args.backend == "file" and not args.findings: - sys.exit("--backend file requires --findings PATH.") - # Subdomain is always needed: it builds the ticket links in the Discord payload. subdomain = get_env("ZENDESK_SUBDOMAIN", args.subdomain) # A dump exits before rendering anything, so it never needs the webhook either. @@ -1060,7 +1055,7 @@ def main(): classified = [] updated_ids = set() - if args.backend == "file": + if args.findings: # Findings already exist, so neither Zendesk nor a model is involved. findings = load_findings(args.findings) print(f"Loaded {len(findings)} findings from {args.findings}.") @@ -1137,7 +1132,7 @@ def main(): dump_batch(args.dump_batch, compact, model) print(f"Wrote {len(compact)} tickets to {args.dump_batch} — this file contains " f"ticket content, so keep it out of the repo.") - print("Classify it, then: --backend file --findings --dry-run") + print("Classify it, then: --findings --dry-run") return client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY From ba88631b45c2c4b91c674e3681501df66d65a028 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 09:28:36 +0200 Subject: [PATCH 12/16] feat: replace Discord embeds with one line per ticket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The digest is read by skimming, and an embed gave every field its own labelled box — sixteen of those is a wall, and Discord's 10-embeds-per- message cap fanned a busy day out over several posts for no reason. Each ticket is now a line: 🔥 | 🐞 | #27605 · Notifications only appear after opening the app | Likely cause: Background push service not waking client leading with a severity marker (🚨 for the urgent categories, which the model rates not_applicable because they aren't bugs), then the category emoji, a masked link on the id, the summary, and the root-cause guess. An abuse report keeps carrying its reported Session ID, which is the one field on the old embed worth the characters. The header keeps the coverage accounting that stops a truncated run from reading as a quiet day, now with a compact category tally and a duplicate- cluster line. Chunking moves from Discord's 6,000-char embed budget to the 2,000-char content limit, counting the newlines that join lines; per-message ticket-id coverage is unchanged, so partial-failure recovery still works. Category colours went with the embeds: CATEGORY_SPECS now carries an `urgent` flag where the colour was, and the emoji is derived from the label so the table stays the one place a category is described. --- README.md | 25 +++- zendesk_triage/test_triage.py | 238 ++++++++++++++++-------------- zendesk_triage/triage.py | 268 ++++++++++++++++------------------ 3 files changed, 271 insertions(+), 260 deletions(-) diff --git a/README.md b/README.md index 0863213..25d72b0 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Claude reviews recently-created unsolved Zendesk tickets fetched from the Zendes ### Categories -`CATEGORY_SPECS` in [triage.py](zendesk_triage/triage.py) is the single source of truth — the schema enum, the Discord labels, the urgency colours, and the prompt guidance are all derived from it, so adding a category is one edit. +`CATEGORY_SPECS` in [triage.py](zendesk_triage/triage.py) is the single source of truth — the schema enum, the Discord labels and emoji, which categories count as urgent, and the prompt guidance are all derived from it, so adding a category is one edit. | Category | Notes | | --- | --- | @@ -81,7 +81,7 @@ Claude reviews recently-created unsolved Zendesk tickets fetched from the Zendes | `positive_review` | 4-5★ review, no actionable content | | `feature_request`, `question`, `spam_or_solicitation`, `other` | | -The first three are **urgent categories**: they are not bugs, so the model rates their severity `not_applicable`. Colouring by severity alone painted them the calmest blue and sorted them last, so category urgency wins — they render dark red, sort ahead of everything else, and cannot be pushed out of the digest by the display cap. +The first three are **urgent categories**: they are not bugs, so the model rates their severity `not_applicable`. Marking by severity alone gave them the calmest marker and sorted them last, so category urgency wins — they lead their line with 🚨, sort ahead of everything else, and cannot be pushed out of the digest by the display cap. ### App-store review filtering @@ -93,16 +93,25 @@ Detection uses the Zendesk `via.channel`, which identified reviews with no false Twitter DM tickets arrive with `description` identical to `subject` — both just `"Conversation with "` — which is 15% of non-review tickets and unclassifiable as fetched. For those only, `hydrate_descriptions` fetches a page of up to 10 comments and joins every body that differs from the subject into the description; later replies often carry the actual detail. Hydration is an enrichment, so an HTTP error or an unreachable endpoint leaves the ticket as-is rather than failing the run (`--no-hydrate` to skip it entirely). -The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time window, classifies the whole batch in one schema-enforced request to the Anthropic API, and posts Discord embeds: a summary embed plus one embed per highlighted ticket (linking to the ticket in Zendesk). +The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time window, classifies the whole batch in one schema-enforced request to the Anthropic API, and posts a Discord digest: a short header, then one line per ticket worth looking into. -The summary embed accounts for the batch in full, so nothing is dropped silently: +The digest is plain message text rather than embeds — its job is to be skimmed, and a labelled box per field reads as a wall at 16 tickets a day. Each line leads with a severity marker and a category emoji, links the ticket id, and carries the model's one-line summary plus its root-cause guess: ``` -Analyzed **2** of **47** tickets in the window (created in the past 2 days). Skipped **45** already reported and unchanged. +🗂️ **Zendesk triage** — analyzed **5** of **47** tickets in the window (created in the past 2 days). Skipped **31** positive app-store review(s). Backlog: **5,609** unsolved tickets in total (not triaged). -**1** worth looking into. 🔄 **1** changed since last reported. +**4** worth looking into, including **1** crash/data-loss. 🔄 **1** changed since last reported. +🐞 **2** · ⚖️ **1** · 🚨 **1** · ❓ **1** +Likely duplicates: **push-wake** ×2 (#27605, #27610) +🚨 | ⚖️ | #27612 · GDPR request to delete all account data +🔥 | 🐞 | #27605 · Notifications only appear after manually opening the app | Likely cause: Background push service not waking client +🟠 | 🐞 | 🔄 #27610 · Keine Benachrichtigungen bis die App geöffnet wird | Likely cause: Same push wake issue ``` +The header accounts for the batch in full, so nothing is dropped silently. Severity markers are 🔥 crash · 💥 data loss · 🟠 major · 🟡 minor · ⚪ cosmetic · ▫️ not applicable, with 🚨 replacing them on the urgent categories. An abuse report also carries the reported Session ID on its line, since that is the actionable part and it saves opening the ticket. + +Discord caps one message's content at 2,000 characters, so lines are clipped (`SUMMARY_CHARS`, `ROOT_CAUSE_CHARS`) and chunked across messages; each message records which ticket ids it accounts for, which is what makes a partial post failure recoverable. + > **Scope:** the window covers tickets *created* recently, so the long tail of older unsolved tickets is counted in the backlog line but not triaged. That is deliberate — the job is a new-ticket digest, not a backlog sweep. ### Deduplication @@ -113,7 +122,7 @@ The daily window is 48h, so consecutive runs overlap. A state file (`--state`) r | ------ | ------- | | Not seen before | Analyzed and reported | | Seen, `updated_at` unchanged | **Skipped before the model call** — costs no tokens | -| Seen, `updated_at` moved | Re-analyzed, reported, and flagged 🔄 in the embed title | +| Seen, `updated_at` moved | Re-analyzed, reported, and flagged 🔄 on its line | State is written only on a real run, and only for tickets covered by messages Discord **accepted**. Each message carries the ticket ids it accounts for, so a partial failure records exactly what landed: already-posted messages aren't repeated next run, and undelivered tickets stay eligible. The run then exits non-zero. `--dry-run` never writes state. @@ -223,7 +232,7 @@ If you outgrow the cache's guarantees, the next step up is a private store (a pr python -m unittest discover -s zendesk_triage -v ``` -Offline tests covering the window arithmetic, dedup partitioning, state round-trip and pruning, corrupt-state degradation, Discord embed rendering and chunking, defensive JSON parsing, and the retry/pagination behaviour with a stub session. No secrets or network access needed. They run in CI on any push or PR touching `zendesk_triage/`. +Offline tests covering the window arithmetic, dedup partitioning, state round-trip and pruning, corrupt-state degradation, Discord line rendering and message chunking, defensive JSON parsing, and the retry/pagination behaviour with a stub session. No secrets or network access needed. They run in CI on any push or PR touching `zendesk_triage/`. ### Local Testing diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index 41bc8ae..dc0b4b6 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -319,108 +319,146 @@ def test_state_written_by_save_state_round_trips_the_version(self): # ---- Discord rendering ----------------------------------------------------- -class TestSummaryEmbed(unittest.TestCase): - def description(self, findings, stats): +class TestHeader(unittest.TestCase): + def header(self, findings, stats): highlights = [f for f in findings if f.get("worth_looking_into")] - return triage.build_summary_embed(findings, highlights, "acme", stats)["description"] + return triage.build_header(findings, highlights, stats) def test_reports_analyzed_against_matched(self): - text = self.description([finding(1)], {"matched": 47}) - self.assertIn("Analyzed **1** of **47** tickets in the window", text) + text = self.header([finding(1)], {"matched": 47}) + self.assertIn("analyzed **1** of **47** tickets in the window", text) def test_names_the_window(self): - text = self.description([finding(1)], {"matched": 5, "scope": "created in the past 2 days"}) + text = self.header([finding(1)], {"matched": 5, "scope": "created in the past 2 days"}) self.assertIn("(created in the past 2 days)", text) def test_reports_skipped_unchanged_tickets(self): - text = self.description([finding(1)], {"matched": 47, "skipped_unchanged": 45}) + text = self.header([finding(1)], {"matched": 47, "skipped_unchanged": 45}) self.assertIn("Skipped **45** already reported and unchanged", text) def test_omits_the_skip_line_when_nothing_was_skipped(self): - self.assertNotIn("Skipped", self.description([finding(1)], {"skipped_unchanged": 0})) + self.assertNotIn("Skipped", self.header([finding(1)], {"skipped_unchanged": 0})) def test_reports_the_untriaged_backlog_with_thousands_separators(self): - text = self.description([finding(1)], {"total_unsolved": 5609}) + text = self.header([finding(1)], {"total_unsolved": 5609}) self.assertIn("Backlog: **5,609** unsolved tickets in total", text) def test_omits_the_backlog_line_when_the_count_is_unavailable(self): - self.assertNotIn("Backlog", self.description([finding(1)], {"total_unsolved": None})) + self.assertNotIn("Backlog", self.header([finding(1)], {"total_unsolved": None})) def test_flags_how_many_were_re_reports(self): - text = self.description([finding(1)], {"updated_count": 3}) + text = self.header([finding(1)], {"updated_count": 3}) self.assertIn("🔄 **3** changed since last reported", text) def test_counts_crash_and_data_loss_as_serious(self): findings = [finding(1, severity="crash"), finding(2, severity="data_loss")] - self.assertIn("**2** crash/data-loss", self.description(findings, {})) + self.assertIn("**2** crash/data-loss", self.header(findings, {})) def test_works_with_no_stats_at_all(self): - text = self.description([finding(1)], None) - self.assertIn("Analyzed **1**", text) + text = self.header([finding(1)], None) + self.assertIn("analyzed **1**", text) self.assertNotIn("of **", text) + def test_tallies_categories_by_emoji(self): + findings = [finding(1), finding(2), finding(3, category="question")] + text = self.header(findings, {}) + self.assertIn(f"{triage.CATEGORY_EMOJI['bug_report']} **2**", text) + self.assertIn(f"{triage.CATEGORY_EMOJI['question']} **1**", text) + def test_groups_repeated_clusters(self): findings = [finding(1, cluster="push"), finding(2, cluster="push"), finding(3, cluster="solo")] - embed = triage.build_summary_embed(findings, findings, "acme", {}) - names = [f["name"] for f in embed["fields"]] - self.assertIn("Likely duplicate clusters", names) - clusters = next(f for f in embed["fields"] if f["name"] == "Likely duplicate clusters") - self.assertIn("push", clusters["value"]) - self.assertNotIn("solo", clusters["value"]) # a single ticket is not a cluster + text = triage.build_header(findings, findings, {}) + self.assertIn("Likely duplicates:", text) + self.assertIn("push", text) + self.assertNotIn("solo", text) # a single ticket is not a cluster + +class TestTicketLine(unittest.TestCase): + def test_reads_as_markers_then_id_then_summary(self): + line = triage.build_ticket_line( + finding(27605, severity="crash", summary="Notifications only appear after opening"), + "acme", + ) + self.assertTrue(line.startswith(f"🔥 | {triage.CATEGORY_EMOJI['bug_report']} | ")) + self.assertIn("[#27605](https://acme.zendesk.com/agent/tickets/27605)", line) + self.assertIn("· Notifications only appear after opening", line) + self.assertIn("| Likely cause: cause", line) -class TestHighlightEmbed(unittest.TestCase): def test_update_marker_only_appears_for_re_reports(self): - fresh = triage.build_highlight_embed(finding(1), "acme", is_update=False) - repeat = triage.build_highlight_embed(finding(1), "acme", is_update=True) - self.assertFalse(fresh["title"].startswith("🔄")) - self.assertTrue(repeat["title"].startswith("🔄")) + fresh = triage.build_ticket_line(finding(1), "acme", is_update=False) + repeat = triage.build_ticket_line(finding(1), "acme", is_update=True) + self.assertNotIn("🔄", fresh) + self.assertIn("🔄 [#1]", repeat) + + def test_omits_the_cause_segment_when_there_is_none(self): + line = triage.build_ticket_line(finding(1, likely_root_cause=""), "acme") + self.assertNotIn("Likely cause", line) + + def test_carries_the_reported_account_for_abuse_reports(self): + """The one field worth the characters: it is what an abuse report is for.""" + line = triage.build_ticket_line( + finding(1, category="abuse_report", reported_session_id="05" + "a" * 64), "acme" + ) + self.assertIn("Reported: `05" + "a" * 64 + "`", line) - def test_links_back_to_the_ticket(self): - embed = triage.build_highlight_embed(finding(42), "acme") - self.assertEqual(embed["url"], "https://acme.zendesk.com/agent/tickets/42") + def test_an_urgent_category_outranks_a_benign_severity(self): + line = triage.build_ticket_line( + finding(1, category="legal_or_data_request", severity="not_applicable"), "acme" + ) + self.assertTrue(line.startswith(triage.URGENT_MARKER)) - def test_title_stays_within_the_discord_limit(self): - embed = triage.build_highlight_embed(finding(1, summary="x" * 500), "acme", is_update=True) - self.assertLessEqual(len(embed["title"]), 256) + def test_a_long_summary_is_clipped(self): + line = triage.build_ticket_line(finding(1, summary="x" * 500), "acme") + self.assertIn("…", line) + self.assertLess(len(line), 500) class TestBuildMessages(unittest.TestCase): - def test_only_tickets_worth_looking_into_get_their_own_embed(self): + def lines(self, messages): + return "\n".join(m["content"] for m in messages).splitlines() + + def test_only_tickets_worth_looking_into_get_a_line(self): findings = [finding(1), finding(2, worth_looking_into=False)] - messages = build_messages(findings, "acme") - self.assertEqual(len(messages[0]["embeds"]), 2) # summary + one highlight + lines = self.lines(build_messages(findings, "acme")) + self.assertEqual(sum(1 for line in lines if "[#" in line), 1) + self.assertIn("[#1]", "\n".join(lines)) def test_highlights_are_ordered_by_priority_rank(self): findings = [finding(1, priority_rank=3), finding(2, priority_rank=1)] - embeds = build_messages(findings, "acme")[0]["embeds"] - self.assertIn("#2", embeds[1]["title"]) - self.assertIn("#1", embeds[2]["title"]) + ticket_lines = [line for line in self.lines(build_messages(findings, "acme")) if "[#" in line] + self.assertIn("[#2]", ticket_lines[0]) + self.assertIn("[#1]", ticket_lines[1]) - def test_updated_ids_reach_the_right_embed(self): + def test_updated_ids_mark_the_right_line(self): findings = [finding(1), finding(2)] - embeds = build_messages(findings, "acme", {}, updated_ids={2})[0]["embeds"] - titles = {e["title"].lstrip("🔄 ").split(" ")[0]: e["title"] for e in embeds[1:]} - self.assertFalse(titles["#1"].startswith("🔄")) - self.assertTrue(titles["#2"].startswith("🔄")) + text = "\n".join(m["content"] for m in build_messages(findings, "acme", {}, updated_ids={2})) + self.assertIn("🔄 [#2]", text) + self.assertNotIn("🔄 [#1]", text) - def test_embeds_are_chunked_to_the_discord_per_message_limit(self): + def test_the_header_leads_the_first_message(self): + messages = build_messages([finding(1)], "acme", {"matched": 3}) + self.assertTrue(messages[0]["content"].startswith("🗂️ **Zendesk triage**")) + + def test_every_highlight_reaches_a_message(self): findings = [finding(i, priority_rank=i) for i in range(triage.MAX_HIGHLIGHTS)] messages = build_messages(findings, "acme") - for message in messages: - self.assertLessEqual(len(message["embeds"]), triage.MAX_EMBEDS_PER_MESSAGE) - total = sum(len(m["embeds"]) for m in messages) - self.assertEqual(total, triage.MAX_HIGHLIGHTS + 1) # + the summary + text = "\n".join(m["content"] for m in messages) + for i in range(triage.MAX_HIGHLIGHTS): + self.assertIn(f"[#{i}]", text) def test_highlights_beyond_the_cap_are_dropped_but_announced(self): over = triage.MAX_HIGHLIGHTS + 5 findings = [finding(i, priority_rank=i) for i in range(over)] messages = build_messages(findings, "acme") - self.assertIn(f"top {triage.MAX_HIGHLIGHTS} of {over}", messages[0]["content"]) + self.assertIn(f"top **{triage.MAX_HIGHLIGHTS}** of **{over}**", messages[0]["content"]) - def test_no_content_line_when_nothing_was_dropped(self): + def test_no_truncation_notice_when_nothing_was_dropped(self): messages = build_messages([finding(1)], "acme") - self.assertNotIn("content", messages[0]) + self.assertNotIn("Showing the top", messages[0]["content"]) + + def test_messages_carry_no_embeds(self): + for message in build_messages([finding(1)], "acme"): + self.assertEqual(set(message), {"content"}) # ---- Parsing helpers ------------------------------------------------------- @@ -461,37 +499,24 @@ def test_platform_enum_is_wired_into_the_schema(self): class TestUrgency(unittest.TestCase): def test_urgent_category_beats_a_benign_severity(self): - """An abuse report is not a bug, so severity is not_applicable — which used to - paint the most serious ticket in the digest the calmest colour.""" + """An abuse report is not a bug, so the model rates it not_applicable — the + calmest marker on the most serious ticket in the digest is backwards.""" abuse = finding(1, category="abuse_report", severity="not_applicable") - self.assertEqual(triage.embed_color(abuse), triage.CATEGORY_COLOR["abuse_report"]) - self.assertNotEqual(triage.embed_color(abuse), - triage.SEVERITY_COLOR["not_applicable"]) + self.assertEqual(triage.severity_marker(abuse), triage.URGENT_MARKER) + self.assertNotEqual(triage.severity_marker(abuse), + triage.SEVERITY_EMOJI["not_applicable"]) def test_non_urgent_category_still_uses_severity(self): - self.assertEqual(triage.embed_color(finding(1, category="bug_report", severity="crash")), - triage.SEVERITY_COLOR["crash"]) - - def test_unknown_severity_falls_back_to_grey(self): - self.assertEqual(triage.embed_color({"category": "other", "severity": "???"}), 0x95A5A6) - - def test_urgent_tickets_are_highlighted_even_if_not_flagged(self): - abuse = finding(1, category="abuse_report", worth_looking_into=False) - shown, _ = triage.select_highlights([abuse]) - self.assertEqual([f["id"] for f in shown], [1]) + self.assertEqual( + triage.severity_marker(finding(1, category="bug_report", severity="crash")), + triage.SEVERITY_EMOJI["crash"]) - def test_urgent_tickets_sort_ahead_of_better_ranked_ordinary_ones(self): - ordinary = finding(1, category="bug_report", priority_rank=1) - abuse = finding(2, category="abuse_report", priority_rank=99) - shown, _ = triage.select_highlights([ordinary, abuse]) - self.assertEqual([f["id"] for f in shown], [2, 1]) + def test_unknown_severity_falls_back_to_a_neutral_marker(self): + self.assertEqual(triage.severity_marker({"category": "other", "severity": "???"}), "▫️") - def test_urgent_tickets_cannot_be_pushed_out_by_the_display_cap(self): - ordinary = [finding(i, priority_rank=i) for i in range(triage.MAX_HIGHLIGHTS + 5)] - abuse = finding(9999, category="abuse_report", priority_rank=9999) - shown, omitted = triage.select_highlights(ordinary + [abuse]) - self.assertIn(9999, [f["id"] for f in shown]) - self.assertNotIn(9999, [f["id"] for f in omitted]) + def test_every_severity_has_a_marker(self): + missing = [sev for sev in triage.SEVERITIES if sev not in triage.SEVERITY_EMOJI] + self.assertEqual(missing, []) class TestReviewFiltering(unittest.TestCase): @@ -637,46 +662,39 @@ def test_hydration_leaves_the_ticket_alone_when_no_comment_adds_anything(self): self.assertEqual(triage.hydrate_descriptions(session, "acme", [row]), 0) -class TestEmbedCharLimit(unittest.TestCase): - """Discord caps a message at 10 embeds *and* 6,000 chars across them; chunking on - count alone can build a payload Discord rejects.""" +class TestMessageCharLimit(unittest.TestCase): + """Discord caps one message's content at 2,000 characters. Every line is + pre-clipped, and chunking has to account for the newlines that join them.""" def fat(self, ticket_id): - # ~1,300 chars of field text: 10 of these would be ~13,000, over the limit. - return finding(ticket_id, summary="s" * 200, likely_root_cause="r" * 300, - affected_component="c" * 100, language="l" * 40) + return finding(ticket_id, summary="s" * 400, likely_root_cause="r" * 400) - def test_every_message_respects_both_limits(self): + def test_every_message_stays_within_the_limit(self): findings = [self.fat(i) for i in range(triage.MAX_HIGHLIGHTS)] - for message in build_messages(findings, "acme"): - self.assertLessEqual(len(message["embeds"]), triage.MAX_EMBEDS_PER_MESSAGE) - total = sum(triage.embed_char_count(e) for e in message["embeds"]) - self.assertLessEqual(total, triage.MAX_EMBED_CHARS_PER_MESSAGE) - - def test_char_limit_splits_where_the_count_limit_would_not(self): - """9 fat highlights + summary = 10 embeds: within the count limit, over 6,000 chars.""" - messages = build_messages([self.fat(i) for i in range(9)], "acme") - embeds = sum(len(m["embeds"]) for m in messages) - self.assertLessEqual(embeds, triage.MAX_EMBEDS_PER_MESSAGE) # count alone: 1 message - self.assertGreater(len(messages), 1) # chars forced the split - - def test_lean_embeds_are_not_split_early(self): - """The char limit must not fragment ordinary digests.""" - messages = build_messages([finding(i, priority_rank=i) for i in range(9)], "acme") - self.assertEqual(len(messages), 1) - - def test_no_embed_is_dropped_while_chunking(self): - findings = [self.fat(i) for i in range(15)] messages = build_messages(findings, "acme") - self.assertEqual(sum(len(m["embeds"]) for m in messages), 16) # 15 + summary - - def test_char_count_covers_titles_descriptions_and_fields(self): - embed = {"title": "abc", "description": "de", - "fields": [{"name": "fg", "value": "hij"}]} - self.assertEqual(triage.embed_char_count(embed), 3 + 2 + 2 + 3) + for message in messages: + self.assertLessEqual(len(message["content"]), triage.MAX_MESSAGE_CHARS) + self.assertGreater(len(messages), 1) # fat lines must actually split - def test_char_count_tolerates_missing_keys(self): - self.assertEqual(triage.embed_char_count({}), 0) + def test_no_line_is_dropped_while_chunking(self): + findings = [self.fat(i) for i in range(triage.MAX_HIGHLIGHTS)] + text = "\n".join(m["content"] for m in build_messages(findings, "acme")) + for i in range(triage.MAX_HIGHLIGHTS): + self.assertIn(f"[#{i}]", text) + + def test_lean_lines_are_not_split_early(self): + findings = [finding(i, summary="s", likely_root_cause="") for i in range(5)] + self.assertEqual(len(build_messages(findings, "acme")), 1) + + def test_chunking_counts_the_joining_newlines(self): + """Two 1,200-char lines are 2,401 joined — over the cap only if the newline + counts, which is the off-by-one this guards.""" + entries = [("x" * 1200, {1}), ("y" * 1200, {2})] + self.assertEqual(len(triage.chunk_entries(entries)), 2) + + def test_an_oversized_entry_still_gets_a_message(self): + chunks = triage.chunk_entries([("x" * (triage.MAX_MESSAGE_CHARS + 50), {1})]) + self.assertEqual(len(chunks), 1) class TestCoverage(unittest.TestCase): diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 9e3bf24..11df9aa 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -132,47 +132,49 @@ def window_label(hours): # prompt never explains. # # Percentages come from a 3,662-ticket sample of the 13 months to 2026-08. -# Columns: (name, Discord label, urgency colour or None, guidance for the model) +# Columns: (name, Discord label, urgent, guidance for the model) CATEGORY_SPECS = ( - ("abuse_report", "🚨 Abuse report", 0xC0392B, + ("abuse_report", "🚨 Abuse report", True, "One user reporting another account for illegal or abusive content (CSAM, " "harassment, drugs, impersonation). Usually quotes the offending Session ID. " "~11% of non-review tickets. Always set worth_looking_into."), - ("security_report", "🔒 Security report", 0xC0392B, + ("security_report", "🔒 Security report", True, "A vulnerability, exploit, or account-compromise disclosure. Not the same as a " "policy question. Always set worth_looking_into."), - ("legal_or_data_request", "⚖️ Legal / data request", 0xC0392B, + ("legal_or_data_request", "⚖️ Legal / data request", True, "GDPR or data-deletion request, subpoena, law-enforcement or court order. " "Always set worth_looking_into."), - ("bug_report", "🐞 Bug report", None, + ("bug_report", "🐞 Bug report", False, "Something in the app is broken or misbehaving."), - ("account_access", "🔑 Account access", None, + ("account_access", "🔑 Account access", False, "Lost recovery phrase, locked out, or asking to restore an account. Usually " "irreversible by design, but track the volume."), - ("policy_question", "📜 Policy question", None, + ("policy_question", "📜 Policy question", False, "Questions about law, regulation, or policy — 'Chat Control', encryption " "backdoors, whether Session complies with something."), - ("low_star_review", "⭐ Low-star review", None, + ("low_star_review", "⭐ Low-star review", False, "An app-store review of 3 stars or fewer. These often hide a real bug — put " "the underlying problem in `summary`."), - ("positive_review", "👍 Positive review", None, + ("positive_review", "👍 Positive review", False, "An app-store review of 4-5 stars with no actionable content."), - ("feature_request", "💡 Feature request", None, + ("feature_request", "💡 Feature request", False, "Asking for something the app does not do yet."), - ("question", "❓ Question", None, + ("question", "❓ Question", False, "A how-do-I or usage question that is not a bug."), - ("spam_or_solicitation", "🗑️ Spam / solicitation", None, + ("spam_or_solicitation", "🗑️ Spam / solicitation", False, "Marketing, token or OTC investment offers, partnership pitches, listing spam."), - ("other", "• Other", None, + ("other", "• Other", False, "Genuinely none of the above. Prefer a specific category wherever one fits."), ) CATEGORIES = [name for name, _, _, _ in CATEGORY_SPECS] CATEGORY_LABEL = {name: label for name, label, _, _ in CATEGORY_SPECS} # Categories whose urgency `severity` cannot express. They are not bugs, so the model -# rates them not_applicable — which would otherwise paint the most serious ticket in -# the batch the calmest colour and sort it last. -CATEGORY_COLOR = {name: color for name, _, color, _ in CATEGORY_SPECS if color} -URGENT_CATEGORIES = frozenset(CATEGORY_COLOR) +# rates them not_applicable — which would otherwise give the most serious ticket in +# the batch the calmest marker and sort it last. +# The emoji on its own, for the per-ticket lines. Derived from the label so the +# table stays the single place a category is described. +CATEGORY_EMOJI = {name: label.split(" ", 1)[0] for name, label, _, _ in CATEGORY_SPECS} +URGENT_CATEGORIES = frozenset(name for name, _, urgent, _ in CATEGORY_SPECS if urgent) CATEGORY_GUIDANCE = "\n".join(f"- {name}: {desc}" for name, _, _, desc in CATEGORY_SPECS) SEVERITIES = ["crash", "data_loss", "major", "minor", "cosmetic", "not_applicable"] @@ -650,8 +652,8 @@ def build_analysis_prompt(compact_tickets): def validate_findings(findings, label): """Exit unless every entry is an object carrying the keys the renderer indexes. - build_summary_embed does f["category"] / f["severity"] and build_highlight_embed - does f["id"], so a missing key surfaces as a KeyError halfway through building a + build_header does f["category"] / f["severity"] and build_ticket_line does + f["id"], so a missing key surfaces as a KeyError halfway through building a Discord payload. Failing here names the offending entry instead. """ for position, entry in enumerate(findings): @@ -756,18 +758,27 @@ def analyze(client, model, effort, compact_tickets): # ---- Discord rendering ----------------------------------------------------- - -SEVERITY_COLOR = { - "crash": 0xE74C3C, # red - "data_loss": 0xC0392B, # dark red - "major": 0xE67E22, # orange - "minor": 0xF1C40F, # yellow - "cosmetic": 0x95A5A6, # grey - "not_applicable": 0x3498DB, # blue +# +# The digest is plain message text: a short header, then one line per ticket. Its job +# is to be skimmed, and a labelled box per field reads as a wall at 16 tickets a day. + +# Leads each line so severity is scannable straight down the left edge. Urgent +# categories get URGENT_MARKER instead: they are not bugs, so the model rates them +# not_applicable, and the calmest marker on the most serious ticket is backwards. +SEVERITY_EMOJI = { + "crash": "🔥", + "data_loss": "💥", + "major": "🟠", + "minor": "🟡", + "cosmetic": "⚪", + "not_applicable": "▫️", } -MAX_EMBEDS_PER_MESSAGE = 10 -MAX_EMBED_CHARS_PER_MESSAGE = 6000 # Discord's aggregate limit across one message -MAX_HIGHLIGHTS = 27 # 3 messages of ~9 highlights + a summary embed +URGENT_MARKER = "🚨" +# Discord's cap on one message's content. Lines are clipped and chunked against it. +MAX_MESSAGE_CHARS = 2000 +SUMMARY_CHARS = 160 +ROOT_CAUSE_CHARS = 140 +MAX_HIGHLIGHTS = 27 # ~3 messages' worth of lines, plus the header def ticket_url(subdomain, ticket_id): @@ -779,7 +790,53 @@ def clip(text, limit): return text if len(text) <= limit else text[: limit - 1] + "…" -def build_summary_embed(findings, highlights, subdomain, stats=None): +def is_urgent(finding): + return finding.get("category") in URGENT_CATEGORIES + + +def severity_marker(finding): + """The leading emoji: category urgency first, then severity.""" + if is_urgent(finding): + return URGENT_MARKER + return SEVERITY_EMOJI.get(finding.get("severity", "not_applicable"), "▫️") + + +def build_ticket_line(finding, subdomain, is_update=False): + """One skimmable line per ticket: + + 🔥 | 🐞 | #27605 · Notifications only arrive… | Likely cause: push service… + + The id is a masked link, so the ticket stays one click away without spending the + character budget on a visible URL. + """ + tid = finding["id"] + # 🔄 marks a ticket already shown that has since changed, so the reader knows it + # is a follow-up rather than a duplicate post. + marker = "🔄 " if is_update else "" + parts = [ + severity_marker(finding), + CATEGORY_EMOJI.get(finding.get("category"), "•"), + f"{marker}[#{tid}]({ticket_url(subdomain, tid)}) · " + f"{clip(finding.get('summary'), SUMMARY_CHARS) or '(no summary)'}", + ] + root = clip(finding.get("likely_root_cause"), ROOT_CAUSE_CHARS) + if root: + parts.append(f"Likely cause: {root}") + # The reported account is the actionable part of an abuse report — carrying it on + # the line saves opening the ticket to copy it. + reported = clip(finding.get("reported_session_id"), 70) + if reported: + parts.append(f"Reported: `{reported}`") + return " | ".join(parts) + + +def build_header(findings, highlights, stats=None): + """The lead lines: what was looked at, the category tally, duplicate clusters. + + Accounts for the batch honestly — how much of the window was analyzed, what was + skipped and why, how big the untriaged backlog behind it is — so a short digest + never reads as a quiet day when it was really a truncated one. + """ by_category = {} by_severity = {} clusters = {} @@ -790,32 +847,16 @@ def build_summary_embed(findings, highlights, subdomain, stats=None): if label: clusters.setdefault(label, []).append(f["id"]) - cat_lines = "\n".join( - f"{CATEGORY_LABEL.get(cat, cat)}: **{count}**" - for cat, count in sorted(by_category.items(), key=lambda kv: -kv[1]) - ) - serious = by_severity.get("crash", 0) + by_severity.get("data_loss", 0) - dup_clusters = {k: v for k, v in clusters.items() if len(v) > 1} - fields = [{"name": "By category", "value": cat_lines or "—", "inline": False}] - if dup_clusters: - cluster_lines = "\n".join( - f"**{clip(label, 40)}** — {len(ids)} tickets (#{', #'.join(str(i) for i in ids[:6])})" - for label, ids in sorted(dup_clusters.items(), key=lambda kv: -len(kv[1]))[:6] - ) - fields.append({"name": "Likely duplicate clusters", "value": clip(cluster_lines, 1024), "inline": False}) - - # Account for the batch honestly: how many of the window we looked at, how many - # we skipped as unchanged, and how big the untriaged backlog is behind it. stats = stats or {} matched = stats.get("matched") skipped = stats.get("skipped_unchanged") or 0 updated = stats.get("updated_count") or 0 backlog = stats.get("total_unsolved") - window = f"Analyzed **{len(findings)}**" + window = f"🗂️ **Zendesk triage** — analyzed **{len(findings)}**" if matched is not None: window += f" of **{matched}**" - window += f" tickets in the window" + window += " tickets in the window" if stats.get("scope"): window += f" ({stats['scope']})" window += "." @@ -829,99 +870,46 @@ def build_summary_embed(findings, highlights, subdomain, stats=None): if backlog is not None: lines.append(f"Backlog: **{backlog:,}** unsolved tickets in total (not triaged).") + serious = by_severity.get("crash", 0) + by_severity.get("data_loss", 0) tail = f"**{len(highlights)}** worth looking into" tail += f", including **{serious}** crash/data-loss." if serious else "." if updated: tail += f" 🔄 **{updated}** changed since last reported." lines.append(tail) - return { - "title": "🗂️ Zendesk triage", - "description": "\n".join(lines), - "color": 0xE67E22 if highlights else 0x2ECC71, - "fields": fields, - } - - -def is_urgent(finding): - return finding.get("category") in URGENT_CATEGORIES - - -def embed_color(finding): - """Colour by category urgency first, then severity. - - An abuse or legal report is not a bug, so the model rates it not_applicable — - which maps to the calmest blue. Category has to win, or the most serious ticket - in the digest looks the most benign. - """ - urgent = CATEGORY_COLOR.get(finding.get("category")) - if urgent: - return urgent - return SEVERITY_COLOR.get(finding.get("severity", "not_applicable"), 0x95A5A6) - - -def build_highlight_embed(finding, subdomain, is_update=False): - tid = finding["id"] - sev = finding.get("severity", "not_applicable") - cat = CATEGORY_LABEL.get(finding.get("category"), finding.get("category", "")) - # 🔄 marks a ticket we already showed that has since changed, so the reader - # knows it is a follow-up rather than a duplicate post. - marker = "🔄 " if is_update else "" - title = f"{marker}#{tid} · {clip(finding.get('summary'), 200) or '(no summary)'}" - fields = [ - {"name": "Category", "value": clip(cat, 60) or "—", "inline": True}, - {"name": "Severity", "value": sev, "inline": True}, - {"name": "Language", "value": clip(finding.get("language"), 40) or "—", "inline": True}, - ] - platform = finding.get("platform") - if platform and platform != "unknown": - fields.append({"name": "Platform", "value": clip(platform, 40), "inline": True}) - component = clip(finding.get("affected_component"), 100) - if component: - fields.append({"name": "Component", "value": component, "inline": True}) - version = clip(finding.get("app_version"), 40) - if version: - fields.append({"name": "Version", "value": version, "inline": True}) - # The reported account is the actionable part of an abuse report — surfacing it - # here saves opening the ticket to copy it. - reported = clip(finding.get("reported_session_id"), 100) - if reported: - fields.append({"name": "Reported account", "value": f"`{reported}`", "inline": False}) - root = clip(finding.get("likely_root_cause"), 300) - description = f"Likely cause: {root}" if root else "" - return { - "title": clip(title, 256), - "url": ticket_url(subdomain, tid), - "description": description, - "color": embed_color(finding), - "fields": fields, - } + if by_category: + lines.append(clip(" · ".join( + f"{CATEGORY_EMOJI.get(cat, '•')} **{count}**" + for cat, count in sorted(by_category.items(), key=lambda kv: -kv[1]) + ), 300)) + dup_clusters = {k: v for k, v in clusters.items() if len(v) > 1} + if dup_clusters: + cluster_lines = " · ".join( + f"**{clip(label, 40)}** ×{len(ids)} (#{', #'.join(str(i) for i in ids[:4])})" + for label, ids in sorted(dup_clusters.items(), key=lambda kv: -len(kv[1]))[:4] + ) + lines.append(f"Likely duplicates: {clip(cluster_lines, 400)}") -def embed_char_count(embed): - """Characters Discord counts against the per-message embed budget.""" - total = len(embed.get("title") or "") + len(embed.get("description") or "") - for field in embed.get("fields") or []: - total += len(field.get("name") or "") + len(field.get("value") or "") - return total + return "\n".join(lines) def chunk_entries(entries): - """Group (embed, ticket_ids) pairs into messages within both Discord limits. + """Group (line, ticket_ids) pairs into messages within MAX_MESSAGE_CHARS. - Discord caps a message at 10 embeds *and* 6,000 characters summed across them; - chunking on count alone can produce a payload that is rejected as too large. + Lines are joined with a newline, so each one after the first costs a character + more than its own length. An entry longer than the cap still gets its own message + rather than being dropped; the pieces are pre-clipped so that shouldn't arise. """ chunks, current, current_chars = [], [], 0 - for embed, ids in entries: - size = embed_char_count(embed) - too_many = len(current) >= MAX_EMBEDS_PER_MESSAGE - too_long = current_chars + size > MAX_EMBED_CHARS_PER_MESSAGE - if current and (too_many or too_long): + for text, ids in entries: + projected = current_chars + len(text) + (1 if current else 0) + if current and projected > MAX_MESSAGE_CHARS: chunks.append(current) current, current_chars = [], 0 - current.append((embed, ids)) - current_chars += size + projected = len(text) + current.append((text, ids)) + current_chars = projected if current: chunks.append(current) return chunks @@ -951,27 +939,23 @@ def build_messages(findings, subdomain, stats=None, updated_ids=None): shown_ids = {f.get("id") for f in shown} omitted_ids = {f.get("id") for f in omitted} - # The summary embed accounts for every classified ticket except the highlights - # that didn't fit; those are covered by no message and stay eligible. - summary_ids = {f.get("id") for f in findings} - shown_ids - omitted_ids - entries = [(build_summary_embed(findings, shown + omitted, subdomain, stats), summary_ids)] + # The header accounts for every classified ticket except the highlights that + # didn't fit; those are covered by no message and stay eligible next run. + header_ids = {f.get("id") for f in findings} - shown_ids - omitted_ids + header = build_header(findings, shown + omitted, stats) + if omitted: + header += (f"\nShowing the top **{len(shown)}** of " + f"**{len(shown) + len(omitted)}** worth looking into.") + entries = [(header, header_ids)] entries += [ - (build_highlight_embed(f, subdomain, is_update=f.get("id") in updated_ids), + (build_ticket_line(f, subdomain, is_update=f.get("id") in updated_ids), {f.get("id")}) for f in shown ] - content = None - if omitted: - content = (f"Showing the top {len(shown)} of {len(shown) + len(omitted)} " - f"tickets worth looking into.") - messages, coverage = [], [] - for index, chunk in enumerate(chunk_entries(entries)): - payload = {"embeds": [embed for embed, _ in chunk]} - if index == 0 and content: - payload["content"] = content - messages.append(payload) + for chunk in chunk_entries(entries): + messages.append({"content": "\n".join(text for text, _ in chunk)}) covered = set() for _, ids in chunk: covered |= ids @@ -1151,7 +1135,7 @@ def main(): print(f"Note: {len(analyzed) - len(classified)} ticket(s) came back without a " f"classification; they stay eligible for the next run.") - # Same selection the embeds use, so the console count can't disagree with the + # Same selection the digest uses, so the console count can't disagree with the # digest — worth_looking_into alone would miss urgent categories the model # failed to flag. shown, omitted = select_highlights(findings) From 3207c5bcff8516482557e5c0a316f87a7b188027 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 09:35:15 +0200 Subject: [PATCH 13/16] feat: carry the digest in one embed, add a platform icon per line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes to the same line. The lines now travel inside a single embed description rather than message content, purely for the character budget: content caps at 2,000, a description at 4,096. A typical day is ~16 lines at ~180 characters — 55 of which is the masked link on the id — so that is the difference between two messages and one. No `fields` are used; the embed is a bigger text box with a coloured border, orange when something is worth looking into and green when nothing is. Chunking now counts against 4,096, and the first embed carries the title so the rest read as continuations. Each line also gets a platform icon between the category and the id, so "is this mine?" is answerable without reading the summary: 🤖 Android, 🍎 iOS, 🖥️ desktop, 🌐 multiple, ❔ unknown. The three desktop platforms share an icon because the distinction rarely changes who picks a ticket up, and a test asserts every PLATFORMS value has an entry. --- README.md | 26 ++++++---- zendesk_triage/test_triage.py | 91 ++++++++++++++++++++++------------- zendesk_triage/triage.py | 55 ++++++++++++++++----- 3 files changed, 116 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 25d72b0..55d5f38 100644 --- a/README.md +++ b/README.md @@ -95,24 +95,30 @@ Twitter DM tickets arrive with `description` identical to `subject` — both jus The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time window, classifies the whole batch in one schema-enforced request to the Anthropic API, and posts a Discord digest: a short header, then one line per ticket worth looking into. -The digest is plain message text rather than embeds — its job is to be skimmed, and a labelled box per field reads as a wall at 16 tickets a day. Each line leads with a severity marker and a category emoji, links the ticket id, and carries the model's one-line summary plus its root-cause guess: +Each line leads with a severity marker, a category emoji and a platform icon, links the ticket id, and carries the model's one-line summary plus its root-cause guess: ``` -🗂️ **Zendesk triage** — analyzed **5** of **47** tickets in the window (created in the past 2 days). Skipped **31** positive app-store review(s). +🗂️ Zendesk triage +Analyzed **5** of **47** tickets in the window (created in the past 2 days). Skipped **31** positive app-store review(s). Backlog: **5,609** unsolved tickets in total (not triaged). -**4** worth looking into, including **1** crash/data-loss. 🔄 **1** changed since last reported. -🐞 **2** · ⚖️ **1** · 🚨 **1** · ❓ **1** +**5** worth looking into, including **1** crash/data-loss. 🔄 **1** changed since last reported. +🐞 **3** · ⚖️ **1** · 🚨 **1** Likely duplicates: **push-wake** ×2 (#27605, #27610) -🚨 | ⚖️ | #27612 · GDPR request to delete all account data -🔥 | 🐞 | #27605 · Notifications only appear after manually opening the app | Likely cause: Background push service not waking client -🟠 | 🐞 | 🔄 #27610 · Keine Benachrichtigungen bis die App geöffnet wird | Likely cause: Same push wake issue +🚨 | ⚖️ | ❔ | #27612 · GDPR request to delete all account data +🔥 | 🐞 | 🤖 | #27605 · Notifications only appear after manually opening the app | Likely cause: Background push service not waking client +🟠 | 🐞 | 🍎 | 🔄 #27610 · Attachments fail to download on cellular | Likely cause: Same push wake issue +🟡 | 🐞 | 🖥️ | #27611 · Window does not restore after minimise to tray ``` -The header accounts for the batch in full, so nothing is dropped silently. Severity markers are 🔥 crash · 💥 data loss · 🟠 major · 🟡 minor · ⚪ cosmetic · ▫️ not applicable, with 🚨 replacing them on the urgent categories. An abuse report also carries the reported Session ID on its line, since that is the actionable part and it saves opening the ticket. +| Column | Values | +| --- | --- | +| Severity | 🔥 crash · 💥 data loss · 🟠 major · 🟡 minor · ⚪ cosmetic · ▫️ not applicable — replaced by 🚨 on the urgent categories | +| Category | The emoji from `CATEGORY_SPECS`, so it matches the tally line | +| Platform | 🤖 Android · 🍎 iOS · 🖥️ desktop (all three) · 🌐 multiple · ❔ unknown | -Discord caps one message's content at 2,000 characters, so lines are clipped (`SUMMARY_CHARS`, `ROOT_CAUSE_CHARS`) and chunked across messages; each message records which ticket ids it accounts for, which is what makes a partial post failure recoverable. +The header accounts for the batch in full, so nothing is dropped silently, and the embed's left border is orange when something is worth looking into and green when nothing is — a glance answers "does today need me?". An abuse report also carries the reported Session ID on its line, since that is the actionable part and it saves opening the ticket. -> **Scope:** the window covers tickets *created* recently, so the long tail of older unsolved tickets is counted in the backlog line but not triaged. That is deliberate — the job is a new-ticket digest, not a backlog sweep. +**Why an embed for plain lines.** Discord caps message content at 2,000 characters but an embed description at 4,096, and no `fields` are used — the embed is only a bigger text box with a coloured border. A typical day is ~16 lines at ~180 characters (55 of which is the masked link on the id), which is two messages as plain text and one inside an embed. Lines are clipped (`SUMMARY_CHARS`, `ROOT_CAUSE_CHARS`) and chunked against 4,096, counting the newlines that join them; each message records which ticket ids it accounts for, which is what makes a partial post failure recoverable. ### Deduplication diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index dc0b4b6..a258328 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -64,6 +64,11 @@ def build_messages(*args, **kwargs): return triage.build_messages(*args, **kwargs)[0] +def digest_text(messages): + """The digest as one string: every embed description, in order.""" + return "\n".join(e["description"] for m in messages for e in m["embeds"]) + + class FakeResponse: # retry-after: 0 keeps the retry tests instant instead of sleeping through # the real backoff, and exercises the header-honoring path while it's at it. @@ -326,7 +331,7 @@ def header(self, findings, stats): def test_reports_analyzed_against_matched(self): text = self.header([finding(1)], {"matched": 47}) - self.assertIn("analyzed **1** of **47** tickets in the window", text) + self.assertIn("Analyzed **1** of **47** tickets in the window", text) def test_names_the_window(self): text = self.header([finding(1)], {"matched": 5, "scope": "created in the past 2 days"}) @@ -356,7 +361,7 @@ def test_counts_crash_and_data_loss_as_serious(self): def test_works_with_no_stats_at_all(self): text = self.header([finding(1)], None) - self.assertIn("analyzed **1**", text) + self.assertIn("Analyzed **1**", text) self.assertNotIn("of **", text) def test_tallies_categories_by_emoji(self): @@ -414,51 +419,64 @@ def test_a_long_summary_is_clipped(self): class TestBuildMessages(unittest.TestCase): - def lines(self, messages): - return "\n".join(m["content"] for m in messages).splitlines() - def test_only_tickets_worth_looking_into_get_a_line(self): findings = [finding(1), finding(2, worth_looking_into=False)] - lines = self.lines(build_messages(findings, "acme")) - self.assertEqual(sum(1 for line in lines if "[#" in line), 1) - self.assertIn("[#1]", "\n".join(lines)) + text = digest_text(build_messages(findings, "acme")) + self.assertEqual(sum(1 for line in text.splitlines() if "[#" in line), 1) + self.assertIn("[#1]", text) def test_highlights_are_ordered_by_priority_rank(self): findings = [finding(1, priority_rank=3), finding(2, priority_rank=1)] - ticket_lines = [line for line in self.lines(build_messages(findings, "acme")) if "[#" in line] - self.assertIn("[#2]", ticket_lines[0]) - self.assertIn("[#1]", ticket_lines[1]) + lines = [l for l in digest_text(build_messages(findings, "acme")).splitlines() if "[#" in l] + self.assertIn("[#2]", lines[0]) + self.assertIn("[#1]", lines[1]) def test_updated_ids_mark_the_right_line(self): findings = [finding(1), finding(2)] - text = "\n".join(m["content"] for m in build_messages(findings, "acme", {}, updated_ids={2})) + text = digest_text(build_messages(findings, "acme", {}, updated_ids={2})) self.assertIn("🔄 [#2]", text) self.assertNotIn("🔄 [#1]", text) - def test_the_header_leads_the_first_message(self): + def test_the_first_embed_is_titled_and_the_rest_are_continuations(self): + findings = [finding(i, priority_rank=i, summary="s" * 400, likely_root_cause="r" * 400) + for i in range(triage.MAX_HIGHLIGHTS)] + messages = build_messages(findings, "acme") + self.assertGreater(len(messages), 1) + self.assertEqual(messages[0]["embeds"][0]["title"], "🗂️ Zendesk triage") + for message in messages[1:]: + self.assertNotIn("title", message["embeds"][0]) + + def test_the_header_leads_the_first_embed(self): messages = build_messages([finding(1)], "acme", {"matched": 3}) - self.assertTrue(messages[0]["content"].startswith("🗂️ **Zendesk triage**")) + self.assertTrue(messages[0]["embeds"][0]["description"].startswith("Analyzed **1**")) + + def test_the_border_flags_whether_anything_needs_attention(self): + flagged = build_messages([finding(1)], "acme") + quiet = build_messages([finding(1, worth_looking_into=False)], "acme") + self.assertEqual(flagged[0]["embeds"][0]["color"], triage.COLOR_ATTENTION) + self.assertEqual(quiet[0]["embeds"][0]["color"], triage.COLOR_CLEAR) def test_every_highlight_reaches_a_message(self): findings = [finding(i, priority_rank=i) for i in range(triage.MAX_HIGHLIGHTS)] - messages = build_messages(findings, "acme") - text = "\n".join(m["content"] for m in messages) + text = digest_text(build_messages(findings, "acme")) for i in range(triage.MAX_HIGHLIGHTS): self.assertIn(f"[#{i}]", text) def test_highlights_beyond_the_cap_are_dropped_but_announced(self): over = triage.MAX_HIGHLIGHTS + 5 findings = [finding(i, priority_rank=i) for i in range(over)] - messages = build_messages(findings, "acme") - self.assertIn(f"top **{triage.MAX_HIGHLIGHTS}** of **{over}**", messages[0]["content"]) + text = digest_text(build_messages(findings, "acme")) + self.assertIn(f"top **{triage.MAX_HIGHLIGHTS}** of **{over}**", text) def test_no_truncation_notice_when_nothing_was_dropped(self): - messages = build_messages([finding(1)], "acme") - self.assertNotIn("Showing the top", messages[0]["content"]) + self.assertNotIn("Showing the top", digest_text(build_messages([finding(1)], "acme"))) - def test_messages_carry_no_embeds(self): + def test_messages_use_one_embed_and_no_fields(self): + """Fields are what made this a wall; the embed is only a bigger text box.""" for message in build_messages([finding(1)], "acme"): - self.assertEqual(set(message), {"content"}) + self.assertEqual(set(message), {"embeds"}) + self.assertEqual(len(message["embeds"]), 1) + self.assertNotIn("fields", message["embeds"][0]) # ---- Parsing helpers ------------------------------------------------------- @@ -663,37 +681,44 @@ def test_hydration_leaves_the_ticket_alone_when_no_comment_adds_anything(self): class TestMessageCharLimit(unittest.TestCase): - """Discord caps one message's content at 2,000 characters. Every line is - pre-clipped, and chunking has to account for the newlines that join them.""" + """An embed description caps at 4,096 characters. Every line is pre-clipped, and + chunking has to account for the newlines that join them.""" def fat(self, ticket_id): return finding(ticket_id, summary="s" * 400, likely_root_cause="r" * 400) - def test_every_message_stays_within_the_limit(self): + def test_every_description_stays_within_the_limit(self): findings = [self.fat(i) for i in range(triage.MAX_HIGHLIGHTS)] messages = build_messages(findings, "acme") for message in messages: - self.assertLessEqual(len(message["content"]), triage.MAX_MESSAGE_CHARS) + self.assertLessEqual(len(message["embeds"][0]["description"]), + triage.MAX_DESCRIPTION_CHARS) self.assertGreater(len(messages), 1) # fat lines must actually split def test_no_line_is_dropped_while_chunking(self): findings = [self.fat(i) for i in range(triage.MAX_HIGHLIGHTS)] - text = "\n".join(m["content"] for m in build_messages(findings, "acme")) + text = digest_text(build_messages(findings, "acme")) for i in range(triage.MAX_HIGHLIGHTS): self.assertIn(f"[#{i}]", text) - def test_lean_lines_are_not_split_early(self): - findings = [finding(i, summary="s", likely_root_cause="") for i in range(5)] - self.assertEqual(len(build_messages(findings, "acme")), 1) + def test_a_typical_day_fits_one_message(self): + """The point of the embed. A real day is ~16 highlights at ~180 chars a line + (55 of which is the masked link): over 2,000 and so two messages under the + old plain-text cap, comfortably one inside a 4,096-char description.""" + findings = [finding(i, summary="s" * 60, likely_root_cause="r" * 40) + for i in range(16)] + messages = build_messages(findings, "acme") + self.assertEqual(len(messages), 1) + self.assertGreater(len(messages[0]["embeds"][0]["description"]), 2000) def test_chunking_counts_the_joining_newlines(self): - """Two 1,200-char lines are 2,401 joined — over the cap only if the newline + """Two 2,048-char lines are 4,097 joined — over the cap only if the newline counts, which is the off-by-one this guards.""" - entries = [("x" * 1200, {1}), ("y" * 1200, {2})] + entries = [("x" * 2048, {1}), ("y" * 2048, {2})] self.assertEqual(len(triage.chunk_entries(entries)), 2) def test_an_oversized_entry_still_gets_a_message(self): - chunks = triage.chunk_entries([("x" * (triage.MAX_MESSAGE_CHARS + 50), {1})]) + chunks = triage.chunk_entries([("x" * (triage.MAX_DESCRIPTION_CHARS + 50), {1})]) self.assertEqual(len(chunks), 1) diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 11df9aa..3be7057 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -126,8 +126,9 @@ def window_label(hours): # ---- Taxonomy -------------------------------------------------------------- # -# Single source of truth. The schema enum, the Discord labels, the urgency colours, -# and the system-prompt guidance are all derived from this table, so adding a +# Single source of truth. The schema enum, the Discord labels and emoji, which +# categories are urgent, and the system-prompt guidance are all derived from this +# table, so adding a # category is one edit and the model can never be given an enum value that the # prompt never explains. # @@ -759,8 +760,11 @@ def analyze(client, model, effort, compact_tickets): # ---- Discord rendering ----------------------------------------------------- # -# The digest is plain message text: a short header, then one line per ticket. Its job -# is to be skimmed, and a labelled box per field reads as a wall at 16 tickets a day. +# A short header, then one line per ticket — the digest is read by skimming, so the +# lines are the layout. They travel inside a single embed per message purely for the +# character budget: message content caps at 2,000 while an embed description gets +# 4,096, which is the difference between one message and two on a busy day. Fields +# are deliberately unused; a labelled box per attribute is what made this a wall. # Leads each line so severity is scannable straight down the left edge. Urgent # categories get URGENT_MARKER instead: they are not bugs, so the model rates them @@ -774,11 +778,28 @@ def analyze(client, model, effort, compact_tickets): "not_applicable": "▫️", } URGENT_MARKER = "🚨" -# Discord's cap on one message's content. Lines are clipped and chunked against it. -MAX_MESSAGE_CHARS = 2000 +# Second marker column, so "is this mine?" is answerable without reading the summary. +# Every PLATFORMS value needs an entry; the three desktops share one icon because the +# distinction rarely changes who picks the ticket up. +PLATFORM_EMOJI = { + "android": "🤖", + "ios": "🍎", + "desktop_windows": "🖥️", + "desktop_macos": "🖥️", + "desktop_linux": "🖥️", + "multiple": "🌐", + "unknown": "❔", +} +# An embed description caps at 4,096; lines are clipped and chunked against that. +# One embed per message, so Discord's 6,000-across-all-embeds budget never binds. +MAX_DESCRIPTION_CHARS = 4096 SUMMARY_CHARS = 160 ROOT_CAUSE_CHARS = 140 -MAX_HIGHLIGHTS = 27 # ~3 messages' worth of lines, plus the header +MAX_HIGHLIGHTS = 27 +# The left border is the only colour left: one glance says whether today needs +# attention, without reading a word of it. +COLOR_ATTENTION = 0xE67E22 # orange — something is worth looking into +COLOR_CLEAR = 0x2ECC71 # green — nothing flagged def ticket_url(subdomain, ticket_id): @@ -804,7 +825,7 @@ def severity_marker(finding): def build_ticket_line(finding, subdomain, is_update=False): """One skimmable line per ticket: - 🔥 | 🐞 | #27605 · Notifications only arrive… | Likely cause: push service… + 🔥 | 🐞 | 🤖 | #27605 · Notifications only arrive… | Likely cause: push service… The id is a masked link, so the ticket stays one click away without spending the character budget on a visible URL. @@ -816,6 +837,7 @@ def build_ticket_line(finding, subdomain, is_update=False): parts = [ severity_marker(finding), CATEGORY_EMOJI.get(finding.get("category"), "•"), + PLATFORM_EMOJI.get(finding.get("platform"), PLATFORM_EMOJI["unknown"]), f"{marker}[#{tid}]({ticket_url(subdomain, tid)}) · " f"{clip(finding.get('summary'), SUMMARY_CHARS) or '(no summary)'}", ] @@ -853,7 +875,7 @@ def build_header(findings, highlights, stats=None): updated = stats.get("updated_count") or 0 backlog = stats.get("total_unsolved") - window = f"🗂️ **Zendesk triage** — analyzed **{len(findings)}**" + window = f"Analyzed **{len(findings)}**" if matched is not None: window += f" of **{matched}**" window += " tickets in the window" @@ -895,7 +917,7 @@ def build_header(findings, highlights, stats=None): def chunk_entries(entries): - """Group (line, ticket_ids) pairs into messages within MAX_MESSAGE_CHARS. + """Group (line, ticket_ids) pairs into messages within MAX_DESCRIPTION_CHARS. Lines are joined with a newline, so each one after the first costs a character more than its own length. An entry longer than the cap still gets its own message @@ -904,7 +926,7 @@ def chunk_entries(entries): chunks, current, current_chars = [], [], 0 for text, ids in entries: projected = current_chars + len(text) + (1 if current else 0) - if current and projected > MAX_MESSAGE_CHARS: + if current and projected > MAX_DESCRIPTION_CHARS: chunks.append(current) current, current_chars = [], 0 projected = len(text) @@ -954,8 +976,15 @@ def build_messages(findings, subdomain, stats=None, updated_ids=None): ] messages, coverage = [], [] - for chunk in chunk_entries(entries): - messages.append({"content": "\n".join(text for text, _ in chunk)}) + for index, chunk in enumerate(chunk_entries(entries)): + embed = { + "description": "\n".join(text for text, _ in chunk), + "color": COLOR_ATTENTION if shown or omitted else COLOR_CLEAR, + } + # Only the first embed is titled; the rest are continuations of one digest. + if index == 0: + embed["title"] = "🗂️ Zendesk triage" + messages.append({"embeds": [embed]}) covered = set() for _, ids in chunk: covered |= ids From 75a727b02f1554b6e72505ef52322dd48fa33bb7 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 09:47:28 +0200 Subject: [PATCH 14/16] build: move the cache actions to v6 for the Node 24 runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit actions/cache/{restore,save}@v4 target Node 20, which the runners now force onto Node 24 with a deprecation warning on every run. v5 was the node24 migration and v6 an internal ESM change; neither alters the inputs this workflow passes, and GitHub-hosted runners are well past the 2.327.1 minimum v5+ requires. These were the last Node 20 actions in the repo — everything else is already on a current major (checkout@v7, setup-python@v6, and so on). --- .github/workflows/zendesk_triage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/zendesk_triage.yml b/.github/workflows/zendesk_triage.yml index eccd46b..4c831e9 100644 --- a/.github/workflows/zendesk_triage.yml +++ b/.github/workflows/zendesk_triage.yml @@ -65,7 +65,7 @@ jobs: # partially-delivered tickets even when a later Discord POST fails, and the # combined action would discard that on a failed job. - name: Restore triage state - uses: actions/cache/restore@v4 + uses: actions/cache/restore@v6 with: path: .triage-state key: zendesk-triage-state-${{ github.run_id }}-${{ github.run_attempt }} @@ -117,7 +117,7 @@ jobs: # later message fails, and that must survive the job's non-zero exit. - name: Save triage state if: always() - uses: actions/cache/save@v4 + uses: actions/cache/save@v6 with: path: .triage-state key: zendesk-triage-state-${{ github.run_id }}-${{ github.run_attempt }} From 855372a74e2350e196338dee32d939c217c12739 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 09:49:40 +0200 Subject: [PATCH 15/16] feat: post the digest as plain messages, no embeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lines carry their own structure, so the embed was contributing a border and a title. Back to message content, with the 🗂️ header line leading the first message instead of an embed title, and no colour. The trade-off is the character budget, and it is real: content caps at 2,000 where an embed description got 4,096, and a masked link on a ticket id spends 54 characters the reader never sees. Rebuilding last night's run — 16 analyzed, 9 worth looking into — comes to ~2,400 characters, so it arrives as two messages (1,781 + 641) where the embed held it in one. Chunking, clipping and per-message ticket coverage are otherwise unchanged. Platform icons stay. --- README.md | 23 +++++++------ zendesk_triage/test_triage.py | 61 +++++++++++------------------------ zendesk_triage/triage.py | 39 ++++++++-------------- 3 files changed, 42 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index 55d5f38..e509a05 100644 --- a/README.md +++ b/README.md @@ -98,16 +98,15 @@ The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time wi Each line leads with a severity marker, a category emoji and a platform icon, links the ticket id, and carries the model's one-line summary plus its root-cause guess: ``` -🗂️ Zendesk triage -Analyzed **5** of **47** tickets in the window (created in the past 2 days). Skipped **31** positive app-store review(s). -Backlog: **5,609** unsolved tickets in total (not triaged). -**5** worth looking into, including **1** crash/data-loss. 🔄 **1** changed since last reported. -🐞 **3** · ⚖️ **1** · 🚨 **1** -Likely duplicates: **push-wake** ×2 (#27605, #27610) -🚨 | ⚖️ | ❔ | #27612 · GDPR request to delete all account data -🔥 | 🐞 | 🤖 | #27605 · Notifications only appear after manually opening the app | Likely cause: Background push service not waking client -🟠 | 🐞 | 🍎 | 🔄 #27610 · Attachments fail to download on cellular | Likely cause: Same push wake issue -🟡 | 🐞 | 🖥️ | #27611 · Window does not restore after minimise to tray +🗂️ **Zendesk triage** — analyzed **16** of **46** tickets in the window (created in the past 2 days). Skipped **30** positive app-store review(s). +Backlog: **5,680** unsolved tickets in total (not triaged). +**9** worth looking into. +⭐ **6** · 🐞 **3** · ❓ **2** · 🔑 **1** · ⚖️ **1** · 🔒 **1** +Likely duplicates: **push-notifications-not-delivered** ×5 (#27637, #27610, #27606, #27605) +🚨 | ⚖️ | ❔ | #27632 · Police summons demanding user details for a Session ID +🚨 | 🔒 | 🤖 | #27603 · Exported component lets another app obtain internal SharedPreferences | Likely cause: Improperly exported provider allowing external apps to trigger file sharing +🟠 | ⭐ | 🍎 | #27610 · Messages not delivered for days; nothing shows even after opening | Likely cause: Push notification delivery / message retrieval failure +🟠 | 🐞 | 🤖 | 🔄 #27605 · Message and call notifications only appear when the app is opened | Likely cause: Push notification service failure on Android ``` | Column | Values | @@ -116,9 +115,9 @@ Likely duplicates: **push-wake** ×2 (#27605, #27610) | Category | The emoji from `CATEGORY_SPECS`, so it matches the tally line | | Platform | 🤖 Android · 🍎 iOS · 🖥️ desktop (all three) · 🌐 multiple · ❔ unknown | -The header accounts for the batch in full, so nothing is dropped silently, and the embed's left border is orange when something is worth looking into and green when nothing is — a glance answers "does today need me?". An abuse report also carries the reported Session ID on its line, since that is the actionable part and it saves opening the ticket. +The header accounts for the batch in full, so nothing is dropped silently. An abuse report also carries the reported Session ID on its line, since that is the actionable part and it saves opening the ticket. -**Why an embed for plain lines.** Discord caps message content at 2,000 characters but an embed description at 4,096, and no `fields` are used — the embed is only a bigger text box with a coloured border. A typical day is ~16 lines at ~180 characters (55 of which is the masked link on the id), which is two messages as plain text and one inside an embed. Lines are clipped (`SUMMARY_CHARS`, `ROOT_CAUSE_CHARS`) and chunked against 4,096, counting the newlines that join them; each message records which ticket ids it accounts for, which is what makes a partial post failure recoverable. +**Plain message content, no embeds.** The lines carry their own structure, so an embed added a border and nothing else. The cost is the character budget: Discord caps message content at 2,000 against an embed description's 4,096, and a masked link on the id spends 54 characters that the reader never sees. A real 9-highlight day comes to ~2,400 characters, so it arrives as two messages. Lines are clipped (`SUMMARY_CHARS`, `ROOT_CAUSE_CHARS`) and chunked against 2,000, counting the newlines that join them; each message records which ticket ids it accounts for, which is what makes a partial post failure recoverable. ### Deduplication diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py index a258328..b50aee1 100644 --- a/zendesk_triage/test_triage.py +++ b/zendesk_triage/test_triage.py @@ -65,8 +65,8 @@ def build_messages(*args, **kwargs): def digest_text(messages): - """The digest as one string: every embed description, in order.""" - return "\n".join(e["description"] for m in messages for e in m["embeds"]) + """The digest as one string: every message's content, in order.""" + return "\n".join(m["content"] for m in messages) class FakeResponse: @@ -331,7 +331,7 @@ def header(self, findings, stats): def test_reports_analyzed_against_matched(self): text = self.header([finding(1)], {"matched": 47}) - self.assertIn("Analyzed **1** of **47** tickets in the window", text) + self.assertIn("analyzed **1** of **47** tickets in the window", text) def test_names_the_window(self): text = self.header([finding(1)], {"matched": 5, "scope": "created in the past 2 days"}) @@ -361,7 +361,7 @@ def test_counts_crash_and_data_loss_as_serious(self): def test_works_with_no_stats_at_all(self): text = self.header([finding(1)], None) - self.assertIn("Analyzed **1**", text) + self.assertIn("analyzed **1**", text) self.assertNotIn("of **", text) def test_tallies_categories_by_emoji(self): @@ -437,24 +437,9 @@ def test_updated_ids_mark_the_right_line(self): self.assertIn("🔄 [#2]", text) self.assertNotIn("🔄 [#1]", text) - def test_the_first_embed_is_titled_and_the_rest_are_continuations(self): - findings = [finding(i, priority_rank=i, summary="s" * 400, likely_root_cause="r" * 400) - for i in range(triage.MAX_HIGHLIGHTS)] - messages = build_messages(findings, "acme") - self.assertGreater(len(messages), 1) - self.assertEqual(messages[0]["embeds"][0]["title"], "🗂️ Zendesk triage") - for message in messages[1:]: - self.assertNotIn("title", message["embeds"][0]) - - def test_the_header_leads_the_first_embed(self): + def test_the_header_leads_the_first_message(self): messages = build_messages([finding(1)], "acme", {"matched": 3}) - self.assertTrue(messages[0]["embeds"][0]["description"].startswith("Analyzed **1**")) - - def test_the_border_flags_whether_anything_needs_attention(self): - flagged = build_messages([finding(1)], "acme") - quiet = build_messages([finding(1, worth_looking_into=False)], "acme") - self.assertEqual(flagged[0]["embeds"][0]["color"], triage.COLOR_ATTENTION) - self.assertEqual(quiet[0]["embeds"][0]["color"], triage.COLOR_CLEAR) + self.assertTrue(messages[0]["content"].startswith("🗂️ **Zendesk triage**")) def test_every_highlight_reaches_a_message(self): findings = [finding(i, priority_rank=i) for i in range(triage.MAX_HIGHLIGHTS)] @@ -471,12 +456,9 @@ def test_highlights_beyond_the_cap_are_dropped_but_announced(self): def test_no_truncation_notice_when_nothing_was_dropped(self): self.assertNotIn("Showing the top", digest_text(build_messages([finding(1)], "acme"))) - def test_messages_use_one_embed_and_no_fields(self): - """Fields are what made this a wall; the embed is only a bigger text box.""" + def test_messages_are_plain_content(self): for message in build_messages([finding(1)], "acme"): - self.assertEqual(set(message), {"embeds"}) - self.assertEqual(len(message["embeds"]), 1) - self.assertNotIn("fields", message["embeds"][0]) + self.assertEqual(set(message), {"content"}) # ---- Parsing helpers ------------------------------------------------------- @@ -681,18 +663,17 @@ def test_hydration_leaves_the_ticket_alone_when_no_comment_adds_anything(self): class TestMessageCharLimit(unittest.TestCase): - """An embed description caps at 4,096 characters. Every line is pre-clipped, and - chunking has to account for the newlines that join them.""" + """Discord caps one message's content at 2,000 characters. Every line is + pre-clipped, and chunking has to account for the newlines that join them.""" def fat(self, ticket_id): return finding(ticket_id, summary="s" * 400, likely_root_cause="r" * 400) - def test_every_description_stays_within_the_limit(self): + def test_every_message_stays_within_the_limit(self): findings = [self.fat(i) for i in range(triage.MAX_HIGHLIGHTS)] messages = build_messages(findings, "acme") for message in messages: - self.assertLessEqual(len(message["embeds"][0]["description"]), - triage.MAX_DESCRIPTION_CHARS) + self.assertLessEqual(len(message["content"]), triage.MAX_MESSAGE_CHARS) self.assertGreater(len(messages), 1) # fat lines must actually split def test_no_line_is_dropped_while_chunking(self): @@ -701,24 +682,18 @@ def test_no_line_is_dropped_while_chunking(self): for i in range(triage.MAX_HIGHLIGHTS): self.assertIn(f"[#{i}]", text) - def test_a_typical_day_fits_one_message(self): - """The point of the embed. A real day is ~16 highlights at ~180 chars a line - (55 of which is the masked link): over 2,000 and so two messages under the - old plain-text cap, comfortably one inside a 4,096-char description.""" - findings = [finding(i, summary="s" * 60, likely_root_cause="r" * 40) - for i in range(16)] - messages = build_messages(findings, "acme") - self.assertEqual(len(messages), 1) - self.assertGreater(len(messages[0]["embeds"][0]["description"]), 2000) + def test_lean_lines_are_not_split_early(self): + findings = [finding(i, summary="s", likely_root_cause="") for i in range(5)] + self.assertEqual(len(build_messages(findings, "acme")), 1) def test_chunking_counts_the_joining_newlines(self): - """Two 2,048-char lines are 4,097 joined — over the cap only if the newline + """Two 1,000-char lines are 2,001 joined — over the cap only if the newline counts, which is the off-by-one this guards.""" - entries = [("x" * 2048, {1}), ("y" * 2048, {2})] + entries = [("x" * 1000, {1}), ("y" * 1000, {2})] self.assertEqual(len(triage.chunk_entries(entries)), 2) def test_an_oversized_entry_still_gets_a_message(self): - chunks = triage.chunk_entries([("x" * (triage.MAX_DESCRIPTION_CHARS + 50), {1})]) + chunks = triage.chunk_entries([("x" * (triage.MAX_MESSAGE_CHARS + 50), {1})]) self.assertEqual(len(chunks), 1) diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 3be7057..1f9cc69 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -126,9 +126,8 @@ def window_label(hours): # ---- Taxonomy -------------------------------------------------------------- # -# Single source of truth. The schema enum, the Discord labels and emoji, which -# categories are urgent, and the system-prompt guidance are all derived from this -# table, so adding a +# Single source of truth. The schema enum, the Discord labels, the urgency colours, +# and the system-prompt guidance are all derived from this table, so adding a # category is one edit and the model can never be given an enum value that the # prompt never explains. # @@ -761,10 +760,10 @@ def analyze(client, model, effort, compact_tickets): # ---- Discord rendering ----------------------------------------------------- # # A short header, then one line per ticket — the digest is read by skimming, so the -# lines are the layout. They travel inside a single embed per message purely for the -# character budget: message content caps at 2,000 while an embed description gets -# 4,096, which is the difference between one message and two on a busy day. Fields -# are deliberately unused; a labelled box per attribute is what made this a wall. +# lines are the layout. Plain message content, no embeds: the lines carry their own +# structure, so the box added nothing but a border. The cost is the character budget +# (2,000 for content against 4,096 for an embed description), which a busy day can +# spill into a second message — see chunk_entries. # Leads each line so severity is scannable straight down the left edge. Urgent # categories get URGENT_MARKER instead: they are not bugs, so the model rates them @@ -790,16 +789,11 @@ def analyze(client, model, effort, compact_tickets): "multiple": "🌐", "unknown": "❔", } -# An embed description caps at 4,096; lines are clipped and chunked against that. -# One embed per message, so Discord's 6,000-across-all-embeds budget never binds. -MAX_DESCRIPTION_CHARS = 4096 +# Discord's cap on one message's content. Lines are clipped and chunked against it. +MAX_MESSAGE_CHARS = 2000 SUMMARY_CHARS = 160 ROOT_CAUSE_CHARS = 140 MAX_HIGHLIGHTS = 27 -# The left border is the only colour left: one glance says whether today needs -# attention, without reading a word of it. -COLOR_ATTENTION = 0xE67E22 # orange — something is worth looking into -COLOR_CLEAR = 0x2ECC71 # green — nothing flagged def ticket_url(subdomain, ticket_id): @@ -875,7 +869,7 @@ def build_header(findings, highlights, stats=None): updated = stats.get("updated_count") or 0 backlog = stats.get("total_unsolved") - window = f"Analyzed **{len(findings)}**" + window = f"🗂️ **Zendesk triage** — analyzed **{len(findings)}**" if matched is not None: window += f" of **{matched}**" window += " tickets in the window" @@ -917,7 +911,7 @@ def build_header(findings, highlights, stats=None): def chunk_entries(entries): - """Group (line, ticket_ids) pairs into messages within MAX_DESCRIPTION_CHARS. + """Group (line, ticket_ids) pairs into messages within MAX_MESSAGE_CHARS. Lines are joined with a newline, so each one after the first costs a character more than its own length. An entry longer than the cap still gets its own message @@ -926,7 +920,7 @@ def chunk_entries(entries): chunks, current, current_chars = [], [], 0 for text, ids in entries: projected = current_chars + len(text) + (1 if current else 0) - if current and projected > MAX_DESCRIPTION_CHARS: + if current and projected > MAX_MESSAGE_CHARS: chunks.append(current) current, current_chars = [], 0 projected = len(text) @@ -976,15 +970,8 @@ def build_messages(findings, subdomain, stats=None, updated_ids=None): ] messages, coverage = [], [] - for index, chunk in enumerate(chunk_entries(entries)): - embed = { - "description": "\n".join(text for text, _ in chunk), - "color": COLOR_ATTENTION if shown or omitted else COLOR_CLEAR, - } - # Only the first embed is titled; the rest are continuations of one digest. - if index == 0: - embed["title"] = "🗂️ Zendesk triage" - messages.append({"embeds": [embed]}) + for chunk in chunk_entries(entries): + messages.append({"content": "\n".join(text for text, _ in chunk)}) covered = set() for _, ids in chunk: covered |= ids From 0035eecb17fdef1f7d3512dbc80244414fcf3cbc Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 7 Aug 2026 10:03:53 +0200 Subject: [PATCH 16/16] fix: replaced lady_beetle emoji with :bug: --- README.md | 4 ++-- zendesk_triage/triage.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e509a05..f9efbae 100644 --- a/README.md +++ b/README.md @@ -101,12 +101,12 @@ Each line leads with a severity marker, a category emoji and a platform icon, li 🗂️ **Zendesk triage** — analyzed **16** of **46** tickets in the window (created in the past 2 days). Skipped **30** positive app-store review(s). Backlog: **5,680** unsolved tickets in total (not triaged). **9** worth looking into. -⭐ **6** · 🐞 **3** · ❓ **2** · 🔑 **1** · ⚖️ **1** · 🔒 **1** +⭐ **6** · 🐛 **3** · ❓ **2** · 🔑 **1** · ⚖️ **1** · 🔒 **1** Likely duplicates: **push-notifications-not-delivered** ×5 (#27637, #27610, #27606, #27605) 🚨 | ⚖️ | ❔ | #27632 · Police summons demanding user details for a Session ID 🚨 | 🔒 | 🤖 | #27603 · Exported component lets another app obtain internal SharedPreferences | Likely cause: Improperly exported provider allowing external apps to trigger file sharing 🟠 | ⭐ | 🍎 | #27610 · Messages not delivered for days; nothing shows even after opening | Likely cause: Push notification delivery / message retrieval failure -🟠 | 🐞 | 🤖 | 🔄 #27605 · Message and call notifications only appear when the app is opened | Likely cause: Push notification service failure on Android +🟠 | 🐛 | 🤖 | 🔄 #27605 · Message and call notifications only appear when the app is opened | Likely cause: Push notification service failure on Android ``` | Column | Values | diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py index 1f9cc69..f2c74d9 100644 --- a/zendesk_triage/triage.py +++ b/zendesk_triage/triage.py @@ -144,7 +144,7 @@ def window_label(hours): ("legal_or_data_request", "⚖️ Legal / data request", True, "GDPR or data-deletion request, subpoena, law-enforcement or court order. " "Always set worth_looking_into."), - ("bug_report", "🐞 Bug report", False, + ("bug_report", "🐛 Bug report", False, "Something in the app is broken or misbehaving."), ("account_access", "🔑 Account access", False, "Lost recovery phrase, locked out, or asking to restore an account. Usually " @@ -819,7 +819,7 @@ def severity_marker(finding): def build_ticket_line(finding, subdomain, is_update=False): """One skimmable line per ticket: - 🔥 | 🐞 | 🤖 | #27605 · Notifications only arrive… | Likely cause: push service… + 🔥 | 🐛 | 🤖 | #27605 · Notifications only arrive… | Likely cause: push service… The id is a masked link, so the ticket stays one click away without spending the character budget on a visible URL.