From eaf8aca15cd76d79a7a4472dfb37a37e61a422ea Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 13:11:59 -0800 Subject: [PATCH 01/14] Add Investigation Engine SDK and investigate CLI group Co-Authored-By: Claude Sonnet 5.5 --- limacharlie/cli.py | 1 + limacharlie/client.py | 4 +- limacharlie/commands/investigate.py | 1141 ++++++++++++++++++++++++++ limacharlie/discovery.py | 17 + limacharlie/errors.py | 39 +- limacharlie/sdk/investigation.py | 1145 +++++++++++++++++++++++++++ 6 files changed, 2331 insertions(+), 16 deletions(-) create mode 100644 limacharlie/commands/investigate.py create mode 100644 limacharlie/sdk/investigation.py diff --git a/limacharlie/cli.py b/limacharlie/cli.py index 3f6c4c65..6505bcfe 100644 --- a/limacharlie/cli.py +++ b/limacharlie/cli.py @@ -147,6 +147,7 @@ def _wants_help() -> bool: "hive": ("hive", "group"), "ingestion-key": ("ingestion_key", "group"), "installation-key": ("installation_key", "group"), + "investigate": ("investigate", "group"), "integrity": ("integrity", "group"), "ioc": ("ioc", "group"), "job": ("job", "group"), diff --git a/limacharlie/client.py b/limacharlie/client.py index 5118b56d..a1c02ffc 100644 --- a/limacharlie/client.py +++ b/limacharlie/client.py @@ -872,11 +872,13 @@ def request(self, verb: str, url: str, params: dict[str, Any] | None = None, alt self._debug(f"Rate limited, waiting {wait}s before retry...") time.sleep(wait) continue - raise RateLimitError( + rate_err = RateLimitError( f"Rate limit exceeded: {data}", retry_after=parse_retry_after(error_headers.get("retry-after")), code=code, ) + rate_err.response_body = data + raise rate_err if code == HTTP_SERVICE_UNAVAILABLE: if retry_service_unavailable and not has_unavailable_retried and retries < max_retries: diff --git a/limacharlie/commands/investigate.py b/limacharlie/commands/investigate.py new file mode 100644 index 00000000..bc92083a --- /dev/null +++ b/limacharlie/commands/investigate.py @@ -0,0 +1,1141 @@ +"""Investigation Engine commands for LimaCharlie CLI v2. + +``limacharlie investigate ...`` builds, tests and tunes investigation +templates and runs investigations with the Investigation Engine, so a person +or an AI agent can drive the whole loop from a terminal: + + kit -> draft -> validate -> test on past alerts -> refine -> save a DRAFT + -> a person promotes it. + +Output is the engine's own JSON by default when piped (``--output`` selects +another format). Failures carry the engine's coded issues verbatim. +""" + +from __future__ import annotations + +import functools +import json +import sys +from typing import Any, Callable + +import click + +from ..cli import pass_context +from ..client import Client +from ..discovery import register_explain +from ..output import detect_output_format, format_output +from ..sdk.investigation import ( + KIT_SECTIONS, + MAX_POLL_WAIT, + Investigation, + InvestigationError, +) +from ..sdk.organization import Organization +from ._input_helpers import load_file +from ._output_helpers import command_output, note + +# Exit code of a result that is well-formed but says "no" (invalid template, +# refused save): the same code as the CLI's validation errors. +EXIT_REFUSED = 4 + +_STRUCTURED = ("json", "yaml", "toon", "jsonl") + + +# --------------------------------------------------------------------------- +# Explain texts +# --------------------------------------------------------------------------- + +_EXPLAIN_GROUP = """\ +Build, test and tune investigation templates, and run investigations, with +the Investigation Engine (experimental; it must be enabled for your +deployment and organization). + +AI agent workflow for a new template (a person promotes it at the end): + + 1. limacharlie investigate kit --section guide --section schema --section codes + 2. look at real data (detection get, search run, investigate wave ...) + 3. write the template YAML + 4. limacharlie investigate templates validate --file plan.yaml + 5. limacharlie investigate templates test --file plan.yaml --detect-cat --last 10 + 6. refine from the disagreements and unknowns, repeat 4-5 + 7. limacharlie investigate templates save my-plan --file plan.yaml --comment "" + (always a DRAFT; a draft runs only through a test or a run a person starts) + 8. a person reviews it and runs: limacharlie investigate templates promote my-plan --confirm + +Questions without a template: use `investigate packs` then `investigate wave`. + +Permissions: reads need investigation.get + insight.evt.get, writes (start, +wave, test, cancel, decide) investigation.set + insight.evt.get, pause/resume +org.conf.set, template records investigation_plan.get/.set/.del. +""" + +_EXPLAIN_KIT = """\ +Fetch the authoring kit: template JSON Schema, the authoring guide, the +org's typed operator catalogue, examples, effective policy and budgets, every +validation code with its fix, the fact packs, and (when the engine has it) the +default template library. + +The whole kit is about 33k tokens. Fetch it in parts with --section +(repeatable): start with guide, policy, codes, then schema and library. The +`size` entry of the answer says what was returned. + +Examples: + limacharlie investigate kit --section guide --section policy --section codes + limacharlie investigate kit --section library +""" + +_EXPLAIN_OPERATORS = """\ +List the operators a template can call, with typed arguments and the shape of +the fact each produces, and whether this org may use it. --brief keeps only +name, class, description and availability; then ask again with --name for the +full rows of the operators you will use. + +Examples: + limacharlie investigate operators --brief + limacharlie investigate operators --name ioc.prevalence --name entity.get + limacharlie investigate operators --match search +""" + +_EXPLAIN_PACKS = """\ +List the fact packs a template or a wave can `use`: named, tested bundles of +facts (host context, hash spread, ...) with their parameters and exports. +--brief is the compact index (name, brief, params, exports); it is enough to +write waves. + +Examples: + limacharlie investigate packs --brief + limacharlie investigate packs --name host_context +""" + +_EXPLAIN_TEMPLATES_LIST = """\ +List the templates the engine serves for this org (its investigation_plan +records): name, version, description, applies_to and status (draft or +active). Use it to see what exists and to pick a free name. + +Example: + limacharlie investigate templates list --status draft +""" + +_EXPLAIN_TEMPLATES_GET = """\ +Read a stored template record. By default the record is returned as a +document with status, etag, tags and comment; --yaml prints only the template +source, ready to edit and validate again. + +Examples: + limacharlie investigate templates get my-plan + limacharlie investigate templates get my-plan --yaml > plan.yaml +""" + +_EXPLAIN_TEMPLATES_VALIDATE = """\ +Check a template (YAML file, or - for stdin) without running anything: +structure, operator names and arguments, every $reference and expression +against the operators' typed outputs, dependency cycles and the org's budgets. + +The report has `valid`, `errors` and `warnings`, each {code, path, message, +suggestion?}, the worst-case step count and an `estimate`. Fix every error and +read every warning. Exit code is 0 when valid and 4 when not. + +Example: + limacharlie investigate templates validate --file plan.yaml +""" + +_EXPLAIN_TEMPLATES_TEST = """\ +Dry-run a template on REAL past alerts and report how it did. The plan's +read-only operators run with your own token on each sample; nothing is written +(no run, case, label or action). Response steps that would run are only listed +in `would_propose`. + +Samples (at most 25 after expansion): --detection-id ID, --case-id ID, +--detect-cat CATEGORY with --last N (newest N alerts of the category), or +--sample '{"inputs": {...}, "expected": "false_positive"}'. A sample whose case +is closed is judged against the case's classification: `agreement` is agree, +disagree, undecided or none, and summary.agreement_rate is the share that +agreed. Use 5-10 samples of the category, with known false AND true positives. + +Read the result in this order: `validation` (state `invalid`: fix the +template), `summary`, each `disagree`, then `undecided` samples (`unknowns` says +which fact could not be resolved and why). Steps that succeeded are left out +unless --include-steps. + +The call waits for the test (default up to 600 s, --timeout). A test that is +still running prints its test_id: continue with --test-id ID. A test counts +against the org's concurrent run cap and may bill LLM steps. + +Examples: + limacharlie investigate templates test --file plan.yaml --detect-cat my-category --last 10 + limacharlie investigate templates test my-plan --detection-id --case-id + limacharlie investigate templates test --test-id test-173...-ab12 --timeout 120 +""" + +_EXPLAIN_TEMPLATES_SAVE = """\ +Validate a template and write it as the investigation_plan record NAME. + +The record is ALWAYS a draft unless a person passes --status active. A source +that says `status: active` is refused. Nothing is written when validation has +errors, when the record exists and --overwrite is missing, or when the record +is active and the new one is a draft (a draft never replaces an active +template: save under a new name). A draft never runs on its own: it runs only +through `templates test` or a run a person starts, with autonomy capped at +`suggest`. Put what you tested in --comment so the reviewer sees it. + +Examples: + limacharlie investigate templates save my-plan --file plan.yaml --comment "10 samples, 9 agree, 0 unknown" + limacharlie investigate templates save my-plan --file plan.yaml --overwrite +""" + +_EXPLAIN_TEMPLATES_PROMOTE = """\ +Make a stored draft template ACTIVE, so D&R rules and cases can run it on +their own. This is a person's decision: review the draft and its test +results first. AI agents must not run it. The stored template is validated +again and the write is conditional on the version that was read. Requires +--confirm. + +Example: + limacharlie investigate templates promote my-plan --confirm +""" + +_EXPLAIN_TEMPLATES_DELETE = """\ +Delete a template record. Runs already started keep running on the version +they began with. Requires --confirm. + +Example: + limacharlie investigate templates delete my-plan --confirm +""" + +_EXPLAIN_TEMPLATES_INSTALL = """\ +Write the engine's default template library to this org's investigation_plan +records. An org has only the templates it has installed. + +Existing records are NEVER overwritten silently: they are skipped (with the +installed and library versions, so you can see an update is available) unless +--overwrite is given, and an active record is replaced only with +--status active --overwrite. Library templates are installed as DRAFTS by +default whatever status they are written with; a person promotes them or +re-runs with --status active. The result lists what was written, skipped and +failed. + +Needs a engine version whose authoring kit has the `library` section. + +Examples: + limacharlie investigate templates install-defaults + limacharlie investigate templates install-defaults --name mailsec-suspicious-message +""" + +_EXPLAIN_RUNS_START = """\ +Start an investigation run: the engine gathers facts with its operators, +weighs them and reaches a verdict (true_positive, false_positive or +inconclusive) with a proof. + +Pick the plan with --template NAME, --file plan.yaml (an inline draft) or +--auto (the active template whose applies_to matches the alert). Say what to +investigate with --detection-id, --case-id, --entity-id or --question. A +draft template runs with autonomy capped at suggest; an active one acts as +the org's policy says (response actions may wait for a person's approval). +--idempotency-key makes a repeated start return the same run. + +With --wait N the call blocks up to N seconds for the verdict; --follow then +streams the run's events until it ends. + +Examples: + limacharlie investigate runs start --template my-plan --detection-id + limacharlie investigate runs start --auto --case-id 42 --wait 30 +""" + +_EXPLAIN_RUNS_GET = """\ +Read a run: state, phase, verdict, actions and usage. + +Example: + limacharlie investigate runs get +""" + +_EXPLAIN_RUNS_LIST = """\ +List the org's runs, newest first. Filters stack as AND. One page is returned +with --limit; continue with --cursor from the previous page's next_cursor, or +use --all to follow every page. + +Examples: + limacharlie investigate runs list --state running + limacharlie investigate runs list --template my-plan --verdict false_positive --limit 50 +""" + +_EXPLAIN_RUNS_EVENTS = """\ +Read a run's events (structure only, never customer values). Without --follow +one poll is made after --after. With --follow the CLI keeps polling until the +run is over, writing one JSON event per line (or one text line in table +mode). It ends with exit code 0 when the run finished and 1 when --timeout +elapsed first; resume with --after N (printed on stderr). + +Examples: + limacharlie investigate runs events --follow + limacharlie investigate runs events --after 12 --wait 10 +""" + +_EXPLAIN_RUNS_STEPS = """\ +Read a run's step log, plan graph, usage and belief curve. --values includes +customer values (default: structure only). +""" + +_EXPLAIN_RUNS_CASE = """\ +Read the case document a run wrote (404 until the run has one). +""" + +_EXPLAIN_RUNS_CANCEL = """\ +Cancel a run. --revert-leases also reverts the leases of its active actions. + +Example: + limacharlie investigate runs cancel +""" + +_EXPLAIN_WAVE = """\ +Hand the engine a batch of questions at once and get compact, typed answers. +The engine runs every ready step in parallel with YOUR token (no template, no +verdict, nothing stored), reuses what it already knows, and can stop as soon +as the answer is known. + +The file (YAML or JSON) holds the wave: + + steps: + - {id: host, use: host_context, args: {sid: ""}} + - {id: prev, op: ioc.prevalence, args: {file_hash: ""}, select: [hosts_30d]} + stop_when: ["prev.hosts_30d >= 50"] + deadline_ms: 20000 + +Each step has an id and exactly one of `op` (an operator) or `use` (a fact +pack, see `investigate packs`). `select` names the fields to return, `limit` +caps list elements. A string argument that is a whole reference +("$host__resolve.entity_id") reads another step. Flags override the file: +--stop-when (repeatable), --deadline-ms, --full, --validate-only, --input. + +A step that is `unknown` or not `complete` established nothing: it is never an +absence. --validate-only checks the wave and runs nothing. An invalid wave +prints the engine's coded errors on stdout and exits 4. + +Examples: + limacharlie investigate wave --file wave.yaml --validate-only + limacharlie investigate wave --file wave.yaml +""" + +_EXPLAIN_ACTIONS_LIST = """\ +List the org's actions across all runs: the approvals inbox. What needs a +person (awaiting_approval, proposed, uncertain) comes first. --state takes a +state or a comma separated list. + +Examples: + limacharlie investigate actions list --state awaiting_approval +""" + +_EXPLAIN_ACTIONS_APPROVE = """\ +Approve an action a run proposed. You also need the permission of the +action's class (for example sensor.task for a containment action). Approving +executes it. + +Example: + limacharlie investigate actions approve +""" + +_EXPLAIN_ACTIONS_DENY = """\ +Deny an action a run proposed, with an optional note. + +Example: + limacharlie investigate actions deny --note "known admin activity" +""" + +_EXPLAIN_POLICY_GET = """\ +Explain the effective policy: budgets, the operators and extension requests +that are denied, and the autonomy mode of each action class, optionally for +an alert shape (--category, --priority, --severity, --target-tag, --class). +""" + +_EXPLAIN_POLICY_STATUS = """\ +Whether automatic actions are paused for the org, and who last flipped the switch. +""" + +_EXPLAIN_POLICY_PAUSE = """\ +Pause automatic actions for the whole org (the kill switch). Needs org.conf.set. +""" + +_EXPLAIN_POLICY_RESUME = """\ +Resume automatic actions for the org. Needs org.conf.set. +""" + +for _path, _text in { + "investigate": _EXPLAIN_GROUP, + "investigate.kit": _EXPLAIN_KIT, + "investigate.operators": _EXPLAIN_OPERATORS, + "investigate.packs": _EXPLAIN_PACKS, + "investigate.templates.list": _EXPLAIN_TEMPLATES_LIST, + "investigate.templates.get": _EXPLAIN_TEMPLATES_GET, + "investigate.templates.validate": _EXPLAIN_TEMPLATES_VALIDATE, + "investigate.templates.test": _EXPLAIN_TEMPLATES_TEST, + "investigate.templates.save": _EXPLAIN_TEMPLATES_SAVE, + "investigate.templates.promote": _EXPLAIN_TEMPLATES_PROMOTE, + "investigate.templates.delete": _EXPLAIN_TEMPLATES_DELETE, + "investigate.templates.install-defaults": _EXPLAIN_TEMPLATES_INSTALL, + "investigate.runs.start": _EXPLAIN_RUNS_START, + "investigate.runs.get": _EXPLAIN_RUNS_GET, + "investigate.runs.list": _EXPLAIN_RUNS_LIST, + "investigate.runs.events": _EXPLAIN_RUNS_EVENTS, + "investigate.runs.steps": _EXPLAIN_RUNS_STEPS, + "investigate.runs.case": _EXPLAIN_RUNS_CASE, + "investigate.runs.cancel": _EXPLAIN_RUNS_CANCEL, + "investigate.wave": _EXPLAIN_WAVE, + "investigate.actions.list": _EXPLAIN_ACTIONS_LIST, + "investigate.actions.approve": _EXPLAIN_ACTIONS_APPROVE, + "investigate.actions.deny": _EXPLAIN_ACTIONS_DENY, + "investigate.policy.get": _EXPLAIN_POLICY_GET, + "investigate.policy.status": _EXPLAIN_POLICY_STATUS, + "investigate.policy.pause": _EXPLAIN_POLICY_PAUSE, + "investigate.policy.resume": _EXPLAIN_POLICY_RESUME, +}.items(): + register_explain(_path, _text) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _get_investigation(ctx: click.Context) -> Investigation: + client = Client( + oid=ctx.obj.oid, + environment=ctx.obj.environment, + print_debug_fn=ctx.obj.debug_fn, + debug_full_response=ctx.obj.debug_full, + debug_curl=ctx.obj.debug_curl, + debug_verbose=ctx.obj.debug_verbose, + ) + return Investigation(Organization(client)) + + +def _fmt(ctx: click.Context) -> str: + return ctx.obj.output_format or detect_output_format() + + +def _engine_errors(fn: Callable[..., Any]) -> Callable[..., Any]: + """Report an engine failure as a document when the output is machine-read. + + In a structured output format the failure is written to stderr as the + engine's own answer (message, ``error_code``, coded ``errors`` and + ``warnings``) and the command exits with the error's exit code. In table + mode the error propagates and prints as text, issues included. + """ + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + try: + return fn(*args, **kwargs) + except InvestigationError as exc: + ctx = click.get_current_context() + if _fmt(ctx) not in _STRUCTURED: + raise + click.echo(format_output(exc.to_dict(), _fmt(ctx)), err=True) + ctx.exit(exc.exit_code or 1) + return wrapper + + +def _read_source(path: str) -> str: + """Read a template file, or stdin for ``-``.""" + if path == "-": + return sys.stdin.read() + try: + with open(path, "r", encoding="utf-8") as f: + return f.read() + except OSError as exc: + raise click.BadParameter(f"cannot read file: {exc}", param_hint="--file") + + +def _parse_kv(pairs: tuple[str, ...], hint: str) -> dict[str, str]: + out: dict[str, str] = {} + for pair in pairs: + if "=" not in pair: + raise click.BadParameter(f"expected KEY=VALUE, got {pair!r}", param_hint=hint) + key, value = pair.split("=", 1) + out[key] = value + return out + + +def _json_object(text: str | None, hint: str) -> dict[str, Any] | None: + if text is None: + return None + try: + value = json.loads(text) + except json.JSONDecodeError as exc: + raise click.BadParameter(f"invalid JSON: {exc}", param_hint=hint) + if not isinstance(value, dict): + raise click.BadParameter("must be a JSON object", param_hint=hint) + return value + + +def _label(verdict: Any) -> Any: + return verdict.get("label") if isinstance(verdict, dict) else verdict + + +def _emit_page(ctx: click.Context, key: str, items: list[Any], next_cursor: str, project: Callable[[Any], Any]) -> None: + """Print one page of a listing, the way hive listings are printed. + + Structured formats carry ``{key: [...], next_cursor}``. Table, CSV and + JSONL print the (projected) rows and put the cursor on stderr, since they + have nowhere to carry it. + """ + fmt = _fmt(ctx) + if fmt in ("json", "yaml", "toon"): + command_output(ctx, {key: items, "next_cursor": next_cursor or ""}) + return + command_output(ctx, [project(i) for i in items] if fmt in ("table", "csv") else items) + if next_cursor: + note(ctx, f"next_cursor: {next_cursor}") + + +def _run_row(run: dict[str, Any]) -> dict[str, Any]: + return { + "run_id": run.get("run_id"), + "state": run.get("state"), + "template": run.get("template"), + "verdict": _label(run.get("verdict")), + "urgency": run.get("urgency"), + "created": run.get("created"), + "started_by": (run.get("started_by") or {}).get("actor") if isinstance(run.get("started_by"), dict) else None, + } + + +def _action_row(action: dict[str, Any]) -> dict[str, Any]: + run = action.get("run") or {} + return { + "run_id": action.get("run_id"), + "id": action.get("id"), + "class": action.get("class"), + "state": action.get("state"), + "mode": action.get("mode"), + "template": run.get("template"), + "verdict": _label(run.get("verdict")), + } + + +# --------------------------------------------------------------------------- +# Group +# --------------------------------------------------------------------------- + +@click.group("investigate") +def group() -> None: + """Investigation Engine: build and test investigation templates, run investigations (experimental).""" + + +@group.command("kit") +@click.option("--section", "sections", multiple=True, type=click.Choice(KIT_SECTIONS), + help="Section to fetch (repeatable). Default: everything but 'library'; the whole kit is ~33k tokens.") +@pass_context +@_engine_errors +def kit_cmd(ctx, sections) -> None: + """Fetch the authoring kit: schema, guide, operators, examples, policy, codes, packs, library.""" + command_output(ctx, _get_investigation(ctx).get_authoring_kit(list(sections) or None)) + + +@group.command("operators") +@click.option("--match", "match", default=None, help="Keep operators whose name, class or description contains this text.") +@click.option("--name", "names", multiple=True, help="Keep only this operator (repeatable): its full row.") +@click.option("--brief", is_flag=True, default=False, help="Only name, class, description, available and why.") +@pass_context +@_engine_errors +def operators_cmd(ctx, match, names, brief) -> None: + """List the typed operator catalogue.""" + data = _get_investigation(ctx).list_operators() + rows = [o for o in data.get("operators") or [] if isinstance(o, dict)] + if names: + rows = [o for o in rows if o.get("name") in names] + if match: + needle = match.lower() + rows = [o for o in rows if any(needle in str(o.get(k, "")).lower() for k in ("name", "class", "description"))] + if brief: + rows = [{k: o.get(k) for k in ("name", "class", "description", "available", "why")} for o in rows] + command_output(ctx, rows if _fmt(ctx) in ("table", "csv", "jsonl") else {"operators": rows, "extension_requests": data.get("extension_requests", [])}) + + +@group.command("packs") +@click.option("--name", default=None, help="Only this pack.") +@click.option("--brief", is_flag=True, default=False, help="The compact index: name, brief, params, exports.") +@pass_context +@_engine_errors +def packs_cmd(ctx, name, brief) -> None: + """List the fact packs a template or a wave can use.""" + data = _get_investigation(ctx).list_packs(name) + packs = [p for p in data.get("packs") or [] if isinstance(p, dict)] + if brief: + packs = [ + { + "name": p.get("name"), + "brief": p.get("brief"), + "params": [{k: x.get(k) for k in ("name", "type", "required") if k in x} for x in p.get("params") or []], + "exports": [{k: x.get(k) for k in ("name", "type", "ref", "brief") if k in x} for x in p.get("exports") or []], + } + for p in packs + ] + command_output(ctx, packs if _fmt(ctx) in ("table", "csv", "jsonl") else {"packs": packs}) + + +# --------------------------------------------------------------------------- +# templates +# --------------------------------------------------------------------------- + +@group.group("templates") +def templates_group() -> None: + """Investigation templates: list, get, validate, test, save (draft), promote, delete.""" + + +@templates_group.command("list") +@click.option("--status", type=click.Choice(["draft", "active"]), default=None, help="Only templates with this status.") +@pass_context +@_engine_errors +def templates_list(ctx, status) -> None: + """List the templates the engine serves for this org.""" + data = _get_investigation(ctx).list_engine_templates() + rows = [t for t in data.get("templates") or [] if isinstance(t, dict)] + if status: + rows = [t for t in rows if (t.get("status") or "draft") == status] + command_output(ctx, rows if _fmt(ctx) in ("table", "csv", "jsonl") else {"templates": rows}) + + +@templates_group.command("get") +@click.argument("name") +@click.option("--yaml", "yaml_only", is_flag=True, default=False, help="Print only the template source.") +@pass_context +@_engine_errors +def templates_get(ctx, name, yaml_only) -> None: + """Read a stored template record.""" + view = _get_investigation(ctx).get_template(name) + if yaml_only: + click.echo(view["yaml"], nl=False) + return + command_output(ctx, view) + + +def _file_option(fn: Callable[..., Any]) -> Callable[..., Any]: + return click.option("--file", "-f", "file_path", default=None, help="Template YAML file ('-' for stdin).")(fn) + + +@templates_group.command("validate") +@_file_option +@pass_context +@_engine_errors +def templates_validate(ctx, file_path) -> None: + """Check a template without running anything (exit 4 when it has errors).""" + if not file_path: + raise click.UsageError("--file is required.") + report = _get_investigation(ctx).validate_plan(_read_source(file_path)) + command_output(ctx, report) + if report.get("valid") is not True or report.get("errors"): + ctx.exit(EXIT_REFUSED) + + +def _compact_test(test: dict[str, Any], include_steps: bool) -> dict[str, Any]: + """Drop the steps that succeeded from every sample, keeping the ones that did not.""" + if include_steps or not isinstance(test.get("results"), list): + return test + out = dict(test) + results = [] + for r in test["results"]: + if not isinstance(r, dict) or not isinstance(r.get("steps"), list): + results.append(r) + continue + r = dict(r) + steps = r.pop("steps") + r["problem_steps"] = [s for s in steps if isinstance(s, dict) and s.get("status") != "ok"] + r["steps_total"] = len(steps) + results.append(r) + out["results"] = results + return out + + +def _test_rows(test: dict[str, Any]) -> list[dict[str, Any]]: + rows = [] + for r in test.get("results") or []: + if not isinstance(r, dict): + continue + verdict = r.get("verdict") or {} + rows.append({ + "index": r.get("index"), + "sample": json.dumps(r.get("sample") or {}, separators=(",", ":")), + "state": r.get("state"), + "verdict": verdict.get("label") if isinstance(verdict, dict) else verdict, + "p": verdict.get("p") if isinstance(verdict, dict) else None, + "expected": (r.get("expected") or {}).get("label") if isinstance(r.get("expected"), dict) else None, + "agreement": r.get("agreement"), + "unknowns": len(r.get("unknowns") or []), + "wall_ms": r.get("wall_ms"), + }) + return rows + + +def _print_test(ctx: click.Context, test: dict[str, Any], include_steps: bool) -> None: + if _fmt(ctx) not in ("table", "csv"): + command_output(ctx, _compact_test(test, include_steps)) + return + header = f"test {test.get('test_id')}: {test.get('state')}" + if "samples_done" in test: + header += f" ({test.get('samples_done')}/{test.get('samples_total')} samples)" + click.echo(header) + for kind in ("errors", "warnings"): + for issue in (test.get("validation") or {}).get(kind) or []: + click.echo(f" {kind[:-1]}: [{issue.get('code')}] {issue.get('path')}: {issue.get('message')}" + + (f" (did you mean {issue['suggestion']})" if issue.get("suggestion") else "")) + rows = _test_rows(test) + if rows: + command_output(ctx, rows) + summary = test.get("summary") + if isinstance(summary, dict): + click.echo("summary: " + ", ".join(f"{k}={summary[k]}" for k in ( + "samples", "ran", "errors", "decided", "undecided", "true_positive", "false_positive", "inconclusive", + "with_expected", "agree", "disagree", "agreement_rate") if k in summary)) + if summary.get("unknown_reasons"): + click.echo(f"unknown reasons: {summary['unknown_reasons']}") + + +@templates_group.command("test") +@click.argument("name", required=False) +@_file_option +@click.option("--test-id", default=None, help="Keep waiting on a test started earlier (replaces NAME, --file and the samples).") +@click.option("--detection-id", "detection_ids", multiple=True, help="Sample: a stored detection (repeatable).") +@click.option("--case-id", "case_ids", multiple=True, help="Sample: a case; its classification is the expected label (repeatable).") +@click.option("--detect-cat", "detect_cats", multiple=True, help="Sample: the newest alerts of this detection category (repeatable; see --last).") +@click.option("--last", type=click.IntRange(1, 25), default=10, show_default=True, help="How many alerts per --detect-cat.") +@click.option("--sample", "samples_json", multiple=True, help="Sample as a JSON object, e.g. '{\"inputs\": {...}, \"expected\": \"false_positive\"}' (repeatable).") +@click.option("--samples-file", default=None, help="YAML/JSON file holding a list of samples.") +@click.option("--mode", type=click.Choice(["triage", "full"]), default=None, help="triage (default, the cheap pass) or full.") +@click.option("--strict", is_flag=True, default=False, help="Treat any unknown as a failure of the plan.") +@click.option("--overrides-json", default=None, help="Tighten-only budget overrides, e.g. '{\"budgets\": {\"max_wall_seconds\": 120}}'.") +@click.option("--wait", "wait_seconds", type=click.IntRange(0, MAX_POLL_WAIT), default=5, show_default=True, + help="Seconds the starting call waits for a fast test.") +@click.option("--timeout", type=click.FloatRange(min=0), default=600, show_default=True, help="Stop waiting for the test after this many seconds.") +@click.option("--no-wait", is_flag=True, default=False, help="Print the test as started (its test_id) and return.") +@click.option("--include-steps", is_flag=True, default=False, help="Keep every step of every sample (large). Default: only the steps that did not succeed.") +@pass_context +@_engine_errors +def templates_test(ctx, name, file_path, test_id, detection_ids, case_ids, detect_cats, last, samples_json, + samples_file, mode, strict, overrides_json, wait_seconds, timeout, no_wait, include_steps) -> None: + """Dry-run a template on real past alerts and report agreement.""" + inv = _get_investigation(ctx) + if test_id: + if name or file_path: + raise click.UsageError("--test-id replaces NAME and --file.") + test = inv.get_test_plan(test_id, wait=min(wait_seconds, MAX_POLL_WAIT)) + else: + if bool(name) == bool(file_path): + raise click.UsageError("Give exactly one of a stored template NAME or --file.") + samples: list[dict[str, Any]] = [] + samples += [{"detection_id": d} for d in detection_ids] + samples += [{"case_id": c} for c in case_ids] + samples += [{"detect_cat": c, "last": last} for c in detect_cats] + for text in samples_json: + samples.append(_json_object(text, "--sample") or {}) + if samples_file: + loaded = load_file(samples_file, "--samples-file") + if not isinstance(loaded, list) or not all(isinstance(s, dict) for s in loaded): + raise click.BadParameter("must be a list of sample objects", param_hint="--samples-file") + samples += loaded + if not samples: + raise click.UsageError("Give at least one sample: --detection-id, --case-id, --detect-cat, --sample or --samples-file.") + test = inv.test_plan( + samples, + template=name, + inline=_read_source(file_path) if file_path else None, + overrides=_json_object(overrides_json, "--overrides-json"), + strict=strict, + mode=mode, + wait_seconds=wait_seconds, + ) + if not no_wait and not test.get("done") and test.get("state") == "running": + test = inv.wait_for_test(test, timeout=timeout) + _print_test(ctx, test, include_steps) + state = test.get("state") + if state == "invalid": + ctx.exit(EXIT_REFUSED) + if state == "failed": + ctx.exit(1) + if state == "running": + note(ctx, f"test still running; continue with: limacharlie investigate templates test --test-id {test.get('test_id')}") + + +@templates_group.command("save") +@click.argument("name") +@_file_option +@click.option("--status", type=click.Choice(["draft", "active"]), default="draft", show_default=True, + help="'active' is for the person who reviewed the template; AI agents must leave it at draft.") +@click.option("--overwrite", is_flag=True, default=False, help="Replace an existing draft (an active one only with --status active).") +@click.option("--comment", default=None, help="What the template was tested on and how it did; stored as the record's comment.") +@click.option("--tag", "tags", multiple=True, help="Tag to add to the record (repeatable).") +@pass_context +@_engine_errors +def templates_save(ctx, name, file_path, status, overwrite, comment, tags) -> None: + """Validate a template and save it as a DRAFT record (exit 4 when refused).""" + if not file_path: + raise click.UsageError("--file is required.") + result = _get_investigation(ctx).save_template( + name, _read_source(file_path), status=status, overwrite=overwrite, comment=comment, tags=list(tags) or None, + ) + command_output(ctx, result) + if not result.get("saved"): + ctx.exit(EXIT_REFUSED) + + +@templates_group.command("promote") +@click.argument("name") +@click.option("--confirm", is_flag=True, default=False, help="Confirm that you reviewed the draft and want it active (required).") +@pass_context +@_engine_errors +def templates_promote(ctx, name, confirm) -> None: + """Make a reviewed draft ACTIVE (a person's decision).""" + if not confirm: + raise click.UsageError("Promoting a template makes it runnable on its own and requires --confirm.") + result = _get_investigation(ctx).promote_template(name) + command_output(ctx, result) + if not result.get("promoted"): + ctx.exit(EXIT_REFUSED) + + +@templates_group.command("delete") +@click.argument("name") +@click.option("--confirm", is_flag=True, default=False, help="Confirm deletion (required).") +@pass_context +@_engine_errors +def templates_delete(ctx, name, confirm) -> None: + """Delete a template record.""" + if not confirm: + raise click.UsageError("Destructive operation requires --confirm flag.") + result = _get_investigation(ctx).delete_template(name) + note(ctx, f"Template '{name}' deleted.") + command_output(ctx, result) + + +@templates_group.command("install-defaults") +@click.option("--name", "names", multiple=True, help="Install only this library template (repeatable).") +@click.option("--overwrite", is_flag=True, default=False, help="Replace existing records instead of skipping them.") +@click.option("--status", type=click.Choice(["draft", "active"]), default="draft", show_default=True, + help="Status the templates are written with, whatever the library says.") +@pass_context +@_engine_errors +def templates_install_defaults(ctx, names, overwrite, status) -> None: + """Write the engine's default template library to this org (never overwrites silently).""" + result = _get_investigation(ctx).install_default_templates(list(names) or None, overwrite=overwrite, status=status) + command_output(ctx, result) + if result["failed"]: + ctx.exit(1) + + +# --------------------------------------------------------------------------- +# runs +# --------------------------------------------------------------------------- + +@group.group("runs") +def runs_group() -> None: + """Investigation runs: start, get, list, events, steps, case, cancel.""" + + +@runs_group.command("start") +@click.option("--template", default=None, help="Stored template to run.") +@_file_option +@click.option("--auto", is_flag=True, default=False, help="Pick the active template whose applies_to matches the alert.") +@click.option("--detection-id", default=None, help="Detection to investigate.") +@click.option("--case-id", default=None, help="Case to investigate.") +@click.option("--entity-id", default=None, help="Entity to investigate.") +@click.option("--question", default=None, help="Free-text question, for a template that takes one.") +@click.option("--input", "inputs", multiple=True, help="Run input KEY=VALUE (repeatable; values are text).") +@click.option("--inputs-json", default=None, help="Run inputs as a JSON object (typed values).") +@click.option("--urgency", type=click.Choice(["low", "normal", "high"]), default=None, help="Run urgency.") +@click.option("--mode", type=click.Choice(["triage", "full", "scope"]), default=None, help="Run mode (engine default: triage).") +@click.option("--case-mode", "case_mode", default=None, help="create, none or attach: (default per the org's policy).") +@click.option("--idempotency-key", default=None, help="Starting twice with the same key returns the same run.") +@click.option("--wait", "wait_seconds", type=click.IntRange(0, 60), default=0, help="Block up to this many seconds (0..60) for the verdict.") +@click.option("--follow", is_flag=True, default=False, help="Then stream the run's events until it ends.") +@pass_context +@_engine_errors +def runs_start(ctx, template, file_path, auto, detection_id, case_id, entity_id, question, inputs, inputs_json, + urgency, mode, case_mode, idempotency_key, wait_seconds, follow) -> None: + """Start an investigation run on a detection, case, entity or question.""" + if sum(bool(x) for x in (template, file_path, auto)) != 1: + raise click.UsageError("Give exactly one of --template, --file or --auto.") + trigger = {k: v for k, v in (("detection_id", detection_id), ("case_id", case_id), + ("entity_id", entity_id), ("question", question)) if v} + if not trigger: + raise click.UsageError("Give one of --detection-id, --case-id, --entity-id or --question.") + plan: dict[str, Any] + if template: + plan = {"template": template} + elif file_path: + plan = {"inline": _read_source(file_path)} + else: + plan = {"auto": True} + request: dict[str, Any] = {"trigger": trigger, "plan": plan} + run_inputs: dict[str, Any] = _parse_kv(inputs, "--input") + run_inputs.update(_json_object(inputs_json, "--inputs-json") or {}) + if run_inputs: + request["inputs"] = run_inputs + for key, value in (("urgency", urgency), ("mode", mode), ("case", case_mode), ("idempotency_key", idempotency_key)): + if value: + request[key] = value + if wait_seconds: + request["wait_seconds"] = wait_seconds + inv = _get_investigation(ctx) + answer = inv.start_run(request) + command_output(ctx, answer) + if follow: + run_id = (answer.get("run") or {}).get("run_id") + if run_id: + _follow_events(ctx, inv, run_id, 0, MAX_POLL_WAIT, None, 900) + + +@runs_group.command("get") +@click.argument("run_id") +@pass_context +@_engine_errors +def runs_get(ctx, run_id) -> None: + """Read a run: state, phase, verdict, actions and usage.""" + command_output(ctx, _get_investigation(ctx).get_run(run_id)) + + +@runs_group.command("list") +@click.option("--state", default=None, help="Only runs in this state (queued, running, decided, completed, cancelled, failed, budget_exhausted).") +@click.option("--verdict", default=None, help="Only runs with this verdict (true_positive, false_positive, inconclusive).") +@click.option("--template", default=None, help="Only runs of this template.") +@click.option("--trigger", default=None, help="Only runs with this trigger source (api, dr-rule, case).") +@click.option("--entity", default=None, help="Only runs about this entity.") +@click.option("--started-by", default=None, help="Only runs started by this identity, e.g. user:.") +@click.option("--idempotency-key", default=None, help="Only the run started with this key.") +@click.option("--since", default=None, help="Only runs created at or after this RFC 3339 time.") +@click.option("--until", default=None, help="Only runs created before this RFC 3339 time.") +@click.option("--limit", type=click.IntRange(1, 200), default=None, help="Page size (engine default 50).") +@click.option("--cursor", default=None, help="Continue from the next_cursor of the previous page.") +@click.option("--all", "fetch_all", is_flag=True, default=False, help="Follow every page and print all runs.") +@pass_context +@_engine_errors +def runs_list(ctx, state, verdict, template, trigger, entity, started_by, idempotency_key, since, until, limit, cursor, fetch_all) -> None: + """List the org's runs, newest first.""" + inv = _get_investigation(ctx) + filters = dict(state=state, verdict=verdict, template=template, trigger=trigger, entity=entity, + started_by=started_by, idempotency_key=idempotency_key, since=since, until=until, + limit=limit, cursor=cursor) + if fetch_all: + _emit_page(ctx, "runs", list(inv.iter_runs(**filters)), "", _run_row) + return + page = inv.list_runs(**filters) + _emit_page(ctx, "runs", page.get("runs") or [], page.get("next_cursor") or "", _run_row) + + +def _event_line(event: dict[str, Any]) -> str: + data = event.get("data") + return f"{event.get('seq')}\t{event.get('at')}\t{event.get('type')}\t" + ( + json.dumps(data, separators=(",", ":"), default=str) if data else "") + + +def _follow_events(ctx: click.Context, inv: Investigation, run_id: str, after: int, wait: int, + limit: int | None, timeout: float | None) -> None: + """Stream a run's events until it is over, one per line, flushing as they arrive.""" + state: dict[str, Any] = {} + structured = _fmt(ctx) in _STRUCTURED + try: + for event in inv.iter_run_events(run_id, after=after, wait=wait, limit=limit, timeout=timeout, state=state): + if ctx.obj.quiet: + continue + click.echo(json.dumps(event, separators=(",", ":"), default=str) if structured else _event_line(event)) + sys.stdout.flush() + except KeyboardInterrupt: + note(ctx, f"interrupted; resume with --after {state.get('next', after)}") + ctx.exit(130) + if state.get("done"): + return + note(ctx, f"run not finished after {timeout:g}s; resume with --after {state.get('next', after)}") + ctx.exit(1) + + +@runs_group.command("events") +@click.argument("run_id") +@click.option("--after", type=click.IntRange(min=0), default=0, show_default=True, help="Return events after this sequence number.") +@click.option("--wait", type=click.IntRange(0, MAX_POLL_WAIT), default=None, help="Seconds a poll waits for a new event (default 0, or 10 with --follow).") +@click.option("--limit", type=click.IntRange(1, 500), default=None, help="Events per poll.") +@click.option("--follow", is_flag=True, default=False, help="Keep polling until the run is over, printing one event per line.") +@click.option("--timeout", type=click.FloatRange(min=1), default=900, show_default=True, help="With --follow: stop after this many seconds (exit 1).") +@pass_context +@_engine_errors +def runs_events(ctx, run_id, after, wait, limit, follow, timeout) -> None: + """Read a run's events, or follow them to the end.""" + inv = _get_investigation(ctx) + if follow: + _follow_events(ctx, inv, run_id, after, MAX_POLL_WAIT if wait is None else wait, limit, timeout) + return + page = inv.get_run_events(run_id, after=after, wait=wait or 0, limit=limit) + if _fmt(ctx) in ("table", "csv"): + for event in page.get("events") or []: + click.echo(_event_line(event)) + note(ctx, f"next: {page.get('next')} done: {page.get('done')}") + return + command_output(ctx, page) + + +@runs_group.command("steps") +@click.argument("run_id") +@click.option("--values", is_flag=True, default=False, help="Include customer values (default: structure only).") +@pass_context +@_engine_errors +def runs_steps(ctx, run_id, values) -> None: + """Read a run's step log, plan graph, usage and belief curve.""" + command_output(ctx, _get_investigation(ctx).get_run_steps(run_id, values=values)) + + +@runs_group.command("case") +@click.argument("run_id") +@pass_context +@_engine_errors +def runs_case(ctx, run_id) -> None: + """Read the case document a run wrote.""" + command_output(ctx, _get_investigation(ctx).get_run_case(run_id)) + + +@runs_group.command("cancel") +@click.argument("run_id") +@click.option("--revert-leases", is_flag=True, default=False, help="Also revert the leases of the run's active actions.") +@pass_context +@_engine_errors +def runs_cancel(ctx, run_id, revert_leases) -> None: + """Cancel a run.""" + command_output(ctx, _get_investigation(ctx).cancel_run(run_id, revert_leases=revert_leases)) + + +# --------------------------------------------------------------------------- +# wave +# --------------------------------------------------------------------------- + +@group.command("wave") +@click.option("--file", "-f", "file_path", required=True, help="Wave file (YAML or JSON): steps, stop_when, deadline_ms, inputs.") +@click.option("--stop-when", "stop_when", multiple=True, help="Add a stop_when expression (repeatable; at most 8 in all).") +@click.option("--deadline-ms", type=click.IntRange(min=1), default=None, help="Wave deadline in milliseconds (engine default 30000).") +@click.option("--input", "inputs", multiple=True, help="Add a named input KEY=VALUE, read by steps as $KEY (repeatable).") +@click.option("--full", is_flag=True, default=False, help="Return raw operator values instead of the compact answer (many more tokens).") +@click.option("--validate-only", is_flag=True, default=False, help="Validate the wave and run nothing.") +@pass_context +@_engine_errors +def wave_cmd(ctx, file_path, stop_when, deadline_ms, inputs, full, validate_only) -> None: + """Run a batch of operator and fact-pack questions in one call.""" + wave = load_file(file_path, "--file") + if isinstance(wave, list): + wave = {"steps": wave} + if not isinstance(wave, dict): + raise click.BadParameter("must hold a mapping (steps, stop_when, ...) or a list of steps", param_hint="--file") + wave = dict(wave) + if stop_when: + wave["stop_when"] = list(wave.get("stop_when") or []) + list(stop_when) + if deadline_ms: + wave["deadline_ms"] = deadline_ms + if inputs: + wave["inputs"] = {**(wave.get("inputs") or {}), **_parse_kv(inputs, "--input")} + if full: + wave["full"] = True + if validate_only: + wave["validate_only"] = True + try: + result = _get_investigation(ctx).run_wave(wave) + except InvestigationError as exc: + if not exc.issues: + raise + # A refused wave is a result: its coded errors are what the caller needs, on stdout. + command_output(ctx, {"valid": False, "error": exc.raw_message.split("\n")[0], "errors": exc.issues, "warnings": exc.warnings}) + ctx.exit(EXIT_REFUSED) + command_output(ctx, result) + if validate_only and result.get("valid") is False: + ctx.exit(EXIT_REFUSED) + + +# --------------------------------------------------------------------------- +# actions +# --------------------------------------------------------------------------- + +@group.group("actions") +def actions_group() -> None: + """The approvals inbox: list pending actions, approve or deny them.""" + + +@actions_group.command("list") +@click.option("--state", default=None, help="An action state or a comma separated list (awaiting_approval, proposed, executing, active, ...). Default: all.") +@click.option("--class", "action_class", default=None, help="Only this action class.") +@click.option("--run-id", default=None, help="Only the actions of this run.") +@click.option("--limit", type=click.IntRange(1, 200), default=None, help="Page size (engine default 50).") +@click.option("--cursor", default=None, help="Continue from the next_cursor of the previous page.") +@click.option("--all", "fetch_all", is_flag=True, default=False, help="Follow every page.") +@pass_context +@_engine_errors +def actions_list(ctx, state, action_class, run_id, limit, cursor, fetch_all) -> None: + """List the org's actions across runs; what needs a person comes first.""" + inv = _get_investigation(ctx) + items: list[Any] = [] + while True: + page = inv.list_actions(state=state, action_class=action_class, run_id=run_id, limit=limit, cursor=cursor) + items += page.get("actions") or [] + cursor = page.get("next_cursor") or "" + if not (fetch_all and cursor): + break + _emit_page(ctx, "actions", items, cursor, _action_row) + + +@actions_group.command("approve") +@click.argument("run_id") +@click.argument("action_id") +@pass_context +@_engine_errors +def actions_approve(ctx, run_id, action_id) -> None: + """Approve an action (also needs the permission of the action's class).""" + command_output(ctx, _get_investigation(ctx).decide_action(run_id, action_id, "approve")) + + +@actions_group.command("deny") +@click.argument("run_id") +@click.argument("action_id") +@click.option("--note", "reason", default=None, help="Why the action is denied.") +@pass_context +@_engine_errors +def actions_deny(ctx, run_id, action_id, reason) -> None: + """Deny an action.""" + command_output(ctx, _get_investigation(ctx).decide_action(run_id, action_id, "deny", note=reason)) + + +# --------------------------------------------------------------------------- +# policy +# --------------------------------------------------------------------------- + +@group.group("policy") +def policy_group() -> None: + """Effective policy and the pause switch for automatic actions.""" + + +@policy_group.command("get") +@click.option("--category", default=None, help="Detection category to match run-scoped policy records.") +@click.option("--priority", type=int, default=None, help="Detection priority.") +@click.option("--severity", default=None, help="Detection severity.") +@click.option("--target-tag", "target_tags", multiple=True, help="Tag of the target sensor (repeatable).") +@click.option("--class", "action_class", default=None, help="Restrict to one action class.") +@pass_context +@_engine_errors +def policy_get(ctx, category, priority, severity, target_tags, action_class) -> None: + """Explain the effective policy.""" + command_output(ctx, _get_investigation(ctx).get_effective_policy( + category=category, priority=priority, severity=severity, + target_tags=list(target_tags) or None, action_class=action_class, + )) + + +@policy_group.command("status") +@pass_context +@_engine_errors +def policy_status(ctx) -> None: + """Show whether automatic actions are paused.""" + command_output(ctx, _get_investigation(ctx).get_autonomy()) + + +@policy_group.command("pause") +@pass_context +@_engine_errors +def policy_pause(ctx) -> None: + """Pause automatic actions for the org.""" + command_output(ctx, _get_investigation(ctx).set_autonomy("pause")) + + +@policy_group.command("resume") +@pass_context +@_engine_errors +def policy_resume(ctx) -> None: + """Resume automatic actions for the org.""" + command_output(ctx, _get_investigation(ctx).set_autonomy("resume")) diff --git a/limacharlie/discovery.py b/limacharlie/discovery.py index 037486ab..ffe95c25 100644 --- a/limacharlie/discovery.py +++ b/limacharlie/discovery.py @@ -213,6 +213,23 @@ "artifact list", "artifact upload", "artifact download", ], }, + "investigation_engine": { + "description": "Investigation Engine: build, test and tune investigation templates, run investigations and waves, approve actions", + "commands": [ + "investigate kit", "investigate operators", "investigate packs", + "investigate templates list", "investigate templates get", + "investigate templates validate", "investigate templates test", + "investigate templates save", "investigate templates promote", + "investigate templates delete", "investigate templates install-defaults", + "investigate runs start", "investigate runs get", "investigate runs list", + "investigate runs events", "investigate runs steps", "investigate runs case", + "investigate runs cancel", + "investigate wave", + "investigate actions list", "investigate actions approve", "investigate actions deny", + "investigate policy get", "investigate policy status", + "investigate policy pause", "investigate policy resume", + ], + }, "cloud_security": { "description": "Cloud and Code Security: onboarding, posture, identity, findings, code-to-cloud risk, and remediation", "commands": [ diff --git a/limacharlie/errors.py b/limacharlie/errors.py index 4741cc9f..ff1c9512 100644 --- a/limacharlie/errors.py +++ b/limacharlie/errors.py @@ -319,6 +319,11 @@ def size_limit_suggestion(status_code: int, body: Any) -> str | None: def error_from_status_code(status_code: int, body: Any = None) -> LimaCharlieError: """Create the appropriate exception from an HTTP status code. + Every returned exception carries the parsed answer as ``response_body`` + (``ApiError`` always did; the typed 401/403/404/422/429 errors now do + too), so a caller that knows the route can read the API's own error code + and detail instead of parsing the message. + Args: status_code: HTTP status code. body: Response body (str or dict). @@ -328,34 +333,38 @@ def error_from_status_code(status_code: int, body: Any = None) -> LimaCharlieErr """ body_str = str(body) if body else "No response body" + err: LimaCharlieError if status_code == 401: - return AuthenticationError( + err = AuthenticationError( f"Authentication failed: {body_str}", code=status_code, ) - if status_code == 403: - return PermissionDeniedError( + elif status_code == 403: + err = PermissionDeniedError( f"Permission denied: {body_str}", code=status_code, ) - if status_code == 404: - return NotFoundError( + elif status_code == 404: + err = NotFoundError( f"Resource not found: {body_str}", code=status_code, ) - if status_code == 422: - return ValidationError( + elif status_code == 422: + err = ValidationError( f"Validation error: {body_str}", code=status_code, ) - if status_code == 429: - return RateLimitError( + elif status_code == 429: + err = RateLimitError( f"Rate limit exceeded: {body_str}", code=status_code, ) - return ApiError( - f"API error ({status_code}): {body_str}", - status_code=status_code, - response_body=body, - suggestion=size_limit_suggestion(status_code, body), - ) + else: + return ApiError( + f"API error ({status_code}): {body_str}", + status_code=status_code, + response_body=body, + suggestion=size_limit_suggestion(status_code, body), + ) + err.response_body = body + return err diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py new file mode 100644 index 00000000..316def91 --- /dev/null +++ b/limacharlie/sdk/investigation.py @@ -0,0 +1,1145 @@ +"""Investigation Engine SDK for LimaCharlie v2. + +Wraps the ``/investigate/{oid}/...`` REST routes served by the API gateway: +the authoring kit, the typed operator and fact-pack catalogues, plan +validation and dry runs on past alerts, investigation runs (start, read, +list, events, cancel), waves, the approvals inbox and the autonomy switch. + +The feature is experimental and gated per deployment. Where it is off, every +route is an unknown path (HTTP 404 with no engine error code); where the +organisation has no engine, the gateway answers 503. +:class:`InvestigationError` says which of those happened. + +Permissions (all of the listed ones are required): + +- reads (runs, templates, operators, packs, kit, policy, test results, the + approvals inbox): ``investigation.get`` + ``insight.evt.get`` +- writes (start a run, waves, cancel, test a plan, decide an action): + ``investigation.set`` + ``insight.evt.get``; deciding an action also needs + the permission of the action's own class (for example ``sensor.task``) +- pause / resume of automatic actions: ``org.conf.set`` +- the template records themselves (hive ``investigation_plan``): + ``investigation_plan.get`` / ``investigation_plan.set`` / + ``investigation_plan.del``. These are explicit grants, no default role has + them. + +Templates are plain Hive records. This module never writes an *active* +template unless the caller passes ``status="active"`` explicitly (the call a +person makes), never overwrites a record without ``overwrite=True`` and never +replaces an active template with a draft. A draft runs only through a dry run +(:meth:`Investigation.test_plan`) or a run started by a person, with response +actions capped at ``suggest``. +""" + +from __future__ import annotations + +import json +import re +import time +from typing import Any, Iterator, TYPE_CHECKING +from urllib.parse import quote as _quote + +import yaml + +from ..errors import ApiError, LimaCharlieError, ValidationError +from .hive import Hive, HiveRecord, is_record_not_found + +if TYPE_CHECKING: + from .organization import Organization + + +PLAN_HIVE = "investigation_plan" +STATUS_DRAFT = "draft" +STATUS_ACTIVE = "active" + +#: Sections of the authoring kit the engine can return. +KIT_SECTIONS = ("schema", "guide", "operators", "examples", "policy", "codes", "packs", "library") + +#: Decisions the gateway accepts on an action. +ACTION_VERBS = ("approve", "deny", "revert", "extend", "confirm") + +#: Run states after which a run no longer changes. +TERMINAL_RUN_STATES = ("completed", "cancelled", "failed", "budget_exhausted") + +#: Template (record) names this module will create. +TEMPLATE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") + +# What the gateway accepts as a run, action or test id; anything else is +# refused there. "." and ".." are refused as path traversal. +_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +# Sent as the etag of a write that must create a record. It matches no stored +# etag, so the hive refuses the write if the record exists by then. +_CREATE_ONLY_ETAG = "create-only: the record must not exist" + +# The engine's long polls (run events, test results) wait at most this long. +MAX_POLL_WAIT = 10 +MAX_EVENTS_LIMIT = 500 + +# Seconds added to a call's own wait to form its HTTP timeout. +_WAIT_MARGIN = 35 + + +class InvestigationError(ApiError): + """An Investigation Engine call failed. + + Carries what the engine said, verbatim, next to the HTTP status: the + engine's ``error_code`` (``bad_request``, ``not_found``, ``conflict``, + ``too_many_runs``, ...) and the coded ``errors`` / ``warnings`` of a plan + or wave that was refused (each ``{code, path, message, suggestion?}``). + + ``error_code`` (inherited) is the engine's code, or None when the answer + did not come from the engine itself (for example a gateway 404). + + Attributes: + issues: The coded errors of a refused plan or wave. + warnings: The coded warnings that came with them. + """ + + def __init__( + self, + message: str, + *, + status_code: int | None, + issues: list[dict[str, Any]] | None = None, + warnings: list[dict[str, Any]] | None = None, + response_body: Any = None, + suggestion: str | None = None, + exit_code: int = 1, + ) -> None: + self.issues = issues or [] + self.warnings = warnings or [] + super().__init__(message, status_code=status_code, response_body=response_body, suggestion=suggestion) + self.exit_code = exit_code + + def to_dict(self) -> dict[str, Any]: + """Render the failure as a machine-readable document. + + Returns: + dict: ``error`` (message), ``status`` (HTTP), ``error_code``, and + ``errors`` / ``warnings`` exactly as the engine sent them. + """ + out: dict[str, Any] = {"error": self.raw_message, "status": self.status_code} + if self.error_code: + out["error_code"] = self.error_code + if self.issues: + out["errors"] = self.issues + if self.warnings: + out["warnings"] = self.warnings + if self.suggestion: + out["suggestion"] = self.suggestion + return out + + +def _issue_line(issue: Any) -> str: + if not isinstance(issue, dict): + return str(issue) + line = f"[{issue.get('code', '?')}] {issue.get('path', '')}: {issue.get('message', '')}".replace(" : ", ": ") + if issue.get("suggestion"): + line += f" (did you mean {issue['suggestion']})" + return line + + +_PERMISSION_HINT = ( + "Reads need investigation.get + insight.evt.get, writes (start, wave, cancel, test, decide) need " + "investigation.set + insight.evt.get, pause/resume needs org.conf.set, and template records need " + "investigation_plan.get / investigation_plan.set. Deciding an action also needs the permission of its class." +) + +_DISABLED_HINT = ( + "The Investigation Engine is experimental and gated per deployment: this API does not serve /investigate " + "for this organization's region. Check you target the right environment, or ask for the feature to be enabled." +) + + +def _wrap_error(exc: LimaCharlieError) -> LimaCharlieError: + """Turn a transport error into an :class:`InvestigationError` that says what to do.""" + status = exc.status_code + if status is None or status == 401 or status < 400: + return exc + body = getattr(exc, "response_body", None) + message = "" + code: str | None = None + issues: list[dict[str, Any]] = [] + warnings: list[dict[str, Any]] = [] + if isinstance(body, dict): + message = str(body.get("error") or "") + code = body.get("error_code") if isinstance(body.get("error_code"), str) else None + issues = [i for i in (body.get("errors") or []) if i] if isinstance(body.get("errors"), list) else [] + warnings = [w for w in (body.get("warnings") or []) if w] if isinstance(body.get("warnings"), list) else [] + elif isinstance(body, str) and body.strip(): + message = body.strip() + if not message: + message = exc.raw_message + lines = [_issue_line(i) for i in issues] + if lines and not any(line in message for line in lines): + message += "\n" + "\n".join(f" {line}" for line in lines) + + suggestion: str | None = None + if status == 404 and code is None: + suggestion = _DISABLED_HINT + elif status == 403 and code is None: + suggestion = _PERMISSION_HINT + elif status == 503 and code != "unavailable": + suggestion = "No engine serves this organization right now (feature off or the region is unavailable); retry later." + elif status == 429 and code == "too_many_runs": + suggestion = "The organization's concurrent run cap is reached; wait for a run to finish and retry." + elif status == 501: + suggestion = "This feature is not enabled in the engine." + elif status in (502, 504): + suggestion = "The engine did not answer in time. Reads can be retried; check a run's state before starting it again." + return InvestigationError( + message, + status_code=status, + issues=issues, + warnings=warnings, + response_body=body, + suggestion=suggestion, + exit_code=exc.exit_code, + ) + + +def _check_id(kind: str, value: str) -> str: + if not isinstance(value, str) or value in (".", "..") or not _ID_RE.match(value): + raise ValidationError(f"invalid {kind} {value!r}: use letters, digits and . _ : - (at most 128 characters)") + return value + + +def _check_wait(wait: int) -> int: + if not isinstance(wait, int) or isinstance(wait, bool) or wait < 0 or wait > MAX_POLL_WAIT: + raise ValidationError(f"wait must be an integer between 0 and {MAX_POLL_WAIT} seconds") + return wait + + +class _TemplateLoader(yaml.SafeLoader): + """YAML loader for template sources. + + The engine reads templates with YAML 1.2 rules. PyYAML follows 1.1, which + would turn ``yes``/``no``/``on``/``off`` into booleans and an unquoted + date into a ``datetime``, so a template would be stored as something other + than what its author wrote. This loader keeps both as text and accepts + only ``true``/``false`` as booleans. + """ + + +_BOOL_TAG = "tag:yaml.org,2002:bool" +_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" +_TemplateLoader.yaml_implicit_resolvers = { + ch: [(tag, rx) for tag, rx in resolvers if tag not in (_BOOL_TAG, _TIMESTAMP_TAG)] + for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} +_TemplateLoader.add_implicit_resolver( + _BOOL_TAG, re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$"), list("tTfF"), +) + + +def parse_template_source(source: str) -> dict[str, Any]: + """Parse a template's YAML source into the mapping stored in the hive record. + + Args: + source: The template YAML. + + Returns: + dict: The template as plain JSON-compatible values. + + Raises: + ValidationError: If the source is empty, not YAML, not a mapping, or + uses YAML anchors or aliases (a template has no use for them). + """ + if not isinstance(source, str) or not source.strip(): + raise ValidationError("the template is empty") + try: + for event in yaml.parse(source, Loader=_TemplateLoader): + if isinstance(event, yaml.AliasEvent): + raise ValidationError( + "YAML anchors and aliases are not supported in a template; write the value out" + ) + doc = yaml.load(source, Loader=_TemplateLoader) # noqa: S506 - SafeLoader subclass + except yaml.YAMLError as exc: + raise ValidationError(f"the template is not valid YAML: {exc}") from exc + if not isinstance(doc, dict): + raise ValidationError("the template must be a YAML mapping (description, applies_to, facts, ...)") + return doc + + +def dump_template(data: dict[str, Any]) -> str: + """Render a template mapping as YAML, in the key order it was written. + + Args: + data: The template mapping. + + Returns: + str: YAML text the engine accepts. + """ + return yaml.safe_dump(data, sort_keys=False, default_flow_style=False, allow_unicode=True) + + +class Investigation: + """Investigation Engine client for LimaCharlie.""" + + def __init__(self, org: Organization) -> None: + self._org = org + + @property + def oid(self) -> str: + """The organization ID every call is made for.""" + return self._org.oid + + # ------------------------------------------------------------------ + # Low-level helpers + # ------------------------------------------------------------------ + + def _call( + self, + verb: str, + path: str, + *, + query: list[tuple[str, str]] | None = None, + body: dict[str, Any] | None = None, + write: bool = False, + timeout: int | None = None, + ) -> Any: + """Make one engine call, mapping failures to :class:`InvestigationError`. + + A write is never replayed after a gateway timeout: the engine may + already have started the work. + """ + kwargs: dict[str, Any] = {"query_params": query or None, "timeout": timeout} + if body is not None: + kwargs["raw_body"] = json.dumps(body).encode() + kwargs["content_type"] = "application/json" + if write: + kwargs["retry_gateway_timeout"] = False + try: + return self._org.client.request(verb, f"investigate/{self.oid}/{path}", **kwargs) + except LimaCharlieError as exc: + raise _wrap_error(exc) from exc + + def _get(self, path: str, query: list[tuple[str, str]] | None = None, timeout: int | None = None) -> Any: + return self._call("GET", path, query=query, timeout=timeout) + + @staticmethod + def _q(**params: Any) -> list[tuple[str, str]]: + """Build a query string list from the arguments that are set.""" + out: list[tuple[str, str]] = [] + for name, value in params.items(): + if value is None or value == "": + continue + if isinstance(value, bool): + value = "true" if value else "false" + out.append((name.rstrip("_"), str(value))) + return out + + # ------------------------------------------------------------------ + # Reference data + # ------------------------------------------------------------------ + + def get_authoring_kit(self, sections: list[str] | None = None) -> dict[str, Any]: + """Fetch the authoring kit, whole or in sections. + + The whole kit is large (tens of thousands of tokens); ask for the + sections you need. + + Args: + sections: Any of :data:`KIT_SECTIONS`. None asks for the engine's + default (everything except ``library``). + + Returns: + dict: The kit, with a ``size`` entry saying what was returned. + + Raises: + ValidationError: If a section name is not one the engine knows. + InvestigationError: On any API failure. + """ + if sections: + unknown = [s for s in sections if s not in KIT_SECTIONS] + if unknown: + raise ValidationError(f"unknown kit section(s) {', '.join(unknown)}; choose from {', '.join(KIT_SECTIONS)}") + query = self._q(sections=",".join(sections) if sections else None) + return self._get("authoring-kit", query) + + def list_operators(self) -> dict[str, Any]: + """List the operator catalogue with typed arguments and outputs. + + Returns: + dict: ``operators`` (per operator: class, args, output, summary, + availability for this org) and ``extension_requests``. + """ + return self._get("operators") + + def list_packs(self, name: str | None = None) -> dict[str, Any]: + """List the fact packs a plan or a wave can ``use``. + + Args: + name: Return only this pack. + + Returns: + dict: ``packs``, each with params, exports and cost class. + """ + return self._get("packs", self._q(name=name)) + + def list_engine_templates(self) -> dict[str, Any]: + """List the templates the engine has loaded for this org. + + Returns: + dict: ``templates`` with name, version, description, applies_to and status. + """ + return self._get("templates") + + def get_effective_policy( + self, + category: str | None = None, + priority: int | None = None, + severity: str | None = None, + target_tags: list[str] | None = None, + action_class: str | None = None, + ) -> dict[str, Any]: + """Explain the effective policy, optionally for a given alert shape. + + Args: + category: Detection category to match run-scoped policy records. + priority: Detection priority. + severity: Detection severity. + target_tags: Tags of the target sensor. + action_class: Restrict the explanation to one action class. + + Returns: + dict: The explained policy, including ``extension_requests``. + """ + return self._get( + "policy/effective", + self._q( + category=category, priority=priority, severity=severity, + target_tags=",".join(target_tags) if target_tags else None, class_=action_class, + ), + ) + + def get_autonomy(self) -> dict[str, Any]: + """Read whether automatic actions are paused for the org. + + Returns: + dict: The pause-switch state and who last flipped it. + """ + return self._get("autonomy") + + def set_autonomy(self, verb: str) -> dict[str, Any]: + """Pause or resume automatic actions for the org. + + Args: + verb: ``pause`` or ``resume``. + + Returns: + dict: The new switch state. + + Raises: + ValidationError: If ``verb`` is neither. + """ + if verb not in ("pause", "resume"): + raise ValidationError("verb must be pause or resume") + return self._call("POST", f"autonomy/{verb}", write=True) + + # ------------------------------------------------------------------ + # Plan validation and dry runs + # ------------------------------------------------------------------ + + def validate_plan(self, source: str) -> dict[str, Any]: + """Type-check a template without running anything. + + Args: + source: The template YAML. + + Returns: + dict: ``valid``, ``errors`` and ``warnings`` (each ``{code, path, + message, suggestion?}``), the worst-case cost and an ``estimate``. + """ + return self._call("POST", "plans/validate", body={"yaml": source}) + + def test_plan( + self, + samples: list[dict[str, Any]], + *, + template: str | None = None, + inline: str | None = None, + overrides: dict[str, Any] | None = None, + strict: bool = False, + mode: str | None = None, + wait_seconds: int = 5, + ) -> dict[str, Any]: + """Start a dry run of a plan on real past alerts. + + Nothing is written: no run, case, label or action. The test runs the + plan's read-only operators under the caller's own token. + + Args: + samples: Each one of ``{detection_id}``, ``{case_id}``, + ``{detect_cat, last}`` or ``{inputs, expected}``; at most 25 + after expansion. + template: A stored template (draft or active). Give this or ``inline``. + inline: Template YAML to test as a draft. + overrides: Tighten-only budget overrides. + strict: Treat any unknown as a failure of the plan. + mode: ``triage`` (default) or ``full``. + wait_seconds: Let the call wait this long for a fast test (0..10). + + Returns: + dict: The test object: ``test_id``, ``state`` (running, done, + failed, invalid), ``validation``, ``results``, ``summary``, + ``next`` and ``done``. + + Raises: + ValidationError: If not exactly one of ``template``/``inline`` is + given, or there are no samples. + """ + if bool(template) == bool(inline): + raise ValidationError("give exactly one of a stored template or an inline template") + if not samples: + raise ValidationError("give at least one sample (detection, case, category or inputs)") + _check_wait(wait_seconds) + request: dict[str, Any] = {"samples": samples, "wait_seconds": wait_seconds} + if template: + request["template"] = template + else: + request["inline"] = inline + if overrides: + request["overrides"] = overrides + if strict: + request["strict"] = True + if mode: + request["mode"] = mode + return self._call( + "POST", "test-plan", body=request, write=True, timeout=wait_seconds + _WAIT_MARGIN, + ) + + def get_test_plan(self, test_id: str, after: int = 0, wait: int = 0) -> dict[str, Any]: + """Read a plan test, optionally waiting for more results. + + Args: + test_id: The ``test_id`` of :meth:`test_plan`. + after: Number of results already seen. + wait: Seconds to wait (0..10) for another result or the end. + + Returns: + dict: The same test object as :meth:`test_plan`. + """ + _check_id("test id", test_id) + _check_wait(wait) + return self._get( + f"test-plan/{_quote(test_id, safe='')}", self._q(after=after, wait=wait), timeout=wait + _WAIT_MARGIN, + ) + + def wait_for_test( + self, + test: dict[str, Any], + timeout: float = 600, + wait: int = MAX_POLL_WAIT, + sleep: Any = time.sleep, + ) -> dict[str, Any]: + """Poll a plan test until it is over or the timeout elapses. + + Args: + test: The object :meth:`test_plan` returned. + timeout: Give up after this many seconds and return the latest state. + wait: Seconds each poll may wait on the engine (0..10). + sleep: Pause between polls that return at once (injectable for tests). + + Returns: + dict: The latest test object. ``done`` tells whether it finished. + """ + deadline = time.monotonic() + timeout + while not test.get("done") and test.get("state") not in ("invalid", "failed", "done"): + if time.monotonic() >= deadline: + break + seen = test.get("samples_done") or 0 + test = self.get_test_plan(test["test_id"], after=seen, wait=wait) + if wait == 0: + sleep(1) + return test + + # ------------------------------------------------------------------ + # Runs + # ------------------------------------------------------------------ + + def start_run(self, request: dict[str, Any]) -> dict[str, Any]: + """Start an investigation run. + + Args: + request: The run request: ``trigger``, ``plan`` (``template``, + ``inline`` or ``auto``), ``inputs``, ``urgency``, ``mode``, + ``case``, ``idempotency_key``, ``wait_seconds``, ... + + Returns: + dict: ``run`` and ``joined`` (True when an open run of the same + key absorbed the request instead of a new one starting). + """ + wait = request.get("wait_seconds") or 0 + return self._call("POST", "runs", body=request, write=True, timeout=max(_WAIT_MARGIN, int(wait) + _WAIT_MARGIN)) + + def get_run(self, run_id: str) -> dict[str, Any]: + """Read a run (state, phase, verdict, actions, usage). + + Args: + run_id: The run id. + + Returns: + dict: The run plus its ``critical_path``. + """ + return self._get(f"runs/{_quote(_check_id('run id', run_id), safe='')}") + + def list_runs( + self, + *, + state: str | None = None, + verdict: str | None = None, + template: str | None = None, + trigger: str | None = None, + entity: str | None = None, + started_by: str | None = None, + idempotency_key: str | None = None, + since: str | None = None, + until: str | None = None, + limit: int | None = None, + cursor: str | None = None, + ) -> dict[str, Any]: + """List one page of the org's runs, newest first. + + Args: + state: Only runs in this state. + verdict: Only runs with this verdict. + template: Only runs of this template. + trigger: Only runs with this trigger source. + entity: Only runs about this entity. + started_by: Only runs started by this identity (``user:``). + idempotency_key: Only the run started with this key. + since: Only runs created at or after this RFC 3339 time. + until: Only runs created before this RFC 3339 time. + limit: Page size, 1..200 (engine default 50). + cursor: ``next_cursor`` of the previous page. + + Returns: + dict: ``runs`` and ``next_cursor`` ("" on the last page). + + Raises: + ValidationError: If ``limit`` is out of range. + """ + if limit is not None and not 1 <= limit <= 200: + raise ValidationError("limit must be between 1 and 200") + return self._get( + "runs", + self._q( + state=state, verdict=verdict, template=template, trigger=trigger, entity=entity, + started_by=started_by, idempotency_key=idempotency_key, since=since, until=until, + limit=limit, cursor=cursor, + ), + ) + + def iter_runs(self, **filters: Any) -> Iterator[dict[str, Any]]: + """Walk every run matching the filters, following cursors. + + Args: + **filters: The keyword arguments of :meth:`list_runs`, ``cursor`` included. + + Yields: + dict: One run at a time. + + Raises: + ApiError: If the engine returns a cursor that does not advance. + """ + while True: + page = self.list_runs(**filters) + yield from page.get("runs") or [] + nxt = page.get("next_cursor") or "" + if not nxt: + return + if nxt == filters.get("cursor"): + raise ApiError("Run listing did not advance: the engine returned the same cursor.") + filters["cursor"] = nxt + + def get_run_steps(self, run_id: str, values: bool = False) -> dict[str, Any]: + """Read a run's step log, plan graph and belief curve. + + Args: + run_id: The run id. + values: Include customer values (default: structure only). + + Returns: + dict: ``steps``, ``critical_path``, ``edges``, ``usage``, ... + """ + return self._get( + f"runs/{_quote(_check_id('run id', run_id), safe='')}/steps", self._q(values=True if values else None), + ) + + def get_run_case(self, run_id: str) -> dict[str, Any]: + """Read the case document a run wrote. + + Args: + run_id: The run id. + + Returns: + dict: The case view; a 404 ``not_found`` until the run has one. + """ + return self._get(f"runs/{_quote(_check_id('run id', run_id), safe='')}/case") + + def get_run_events(self, run_id: str, after: int = 0, wait: int = 0, limit: int | None = None) -> dict[str, Any]: + """Read a run's events after a sequence number (one long poll). + + Args: + run_id: The run id. + after: Return events with a greater ``seq``. + wait: Seconds to wait (0..10) for the first new event. + limit: Events per page, 1..500. + + Returns: + dict: ``events``, ``next`` (pass as ``after`` next time) and + ``done`` (the run is over and everything was returned). + + Raises: + ValidationError: If an argument is out of range. + """ + _check_wait(wait) + if after < 0: + raise ValidationError("after must be 0 or more") + if limit is not None and not 1 <= limit <= MAX_EVENTS_LIMIT: + raise ValidationError(f"limit must be between 1 and {MAX_EVENTS_LIMIT}") + return self._get( + f"runs/{_quote(_check_id('run id', run_id), safe='')}/events", + self._q(after=after, wait=wait, limit=limit), + timeout=wait + _WAIT_MARGIN, + ) + + def iter_run_events( + self, + run_id: str, + after: int = 0, + wait: int = MAX_POLL_WAIT, + limit: int | None = None, + timeout: float | None = None, + state: dict[str, Any] | None = None, + ) -> Iterator[dict[str, Any]]: + """Follow a run's events until the run is over. + + Args: + run_id: The run id. + after: Start after this sequence number. + wait: Seconds each poll waits for a new event (0..10). + limit: Events per poll, 1..500. + timeout: Stop after this many seconds (None: follow to the end). + state: Dict the generator fills with ``next`` (resume point), + ``done`` and ``timed_out`` so the caller can report how it ended. + + Yields: + dict: Events in order, each ``{seq, at, type, data}``. + """ + state = state if state is not None else {} + state.update(next=after, done=False, timed_out=False) + deadline = None if timeout is None else time.monotonic() + timeout + while True: + page = self.get_run_events(run_id, after=state["next"], wait=wait, limit=limit) + for event in page.get("events") or []: + yield event + state["next"] = page.get("next", state["next"]) + if page.get("done"): + state["done"] = True + return + if deadline is not None and time.monotonic() >= deadline: + state["timed_out"] = True + return + if wait == 0: + time.sleep(1) + + def cancel_run(self, run_id: str, revert_leases: bool = False) -> dict[str, Any]: + """Cancel a run. + + Args: + run_id: The run id. + revert_leases: Also revert the leases of the run's active actions. + + Returns: + dict: The cancelled run. + """ + body = {"revert_leases": True} if revert_leases else {} + return self._call("POST", f"runs/{_quote(_check_id('run id', run_id), safe='')}/cancel", body=body, write=True) + + # ------------------------------------------------------------------ + # Waves + # ------------------------------------------------------------------ + + def run_wave(self, request: dict[str, Any]) -> dict[str, Any]: + """Run a wave: a batch of operator and fact-pack questions in one call. + + An invalid wave raises :class:`InvestigationError` whose ``issues`` + carry the engine's coded errors. + + Args: + request: ``steps`` (each with ``id`` and one of ``op`` or ``use``, + ``args``, ``select``, ``limit``), ``stop_when``, + ``deadline_ms``, ``inputs``, ``full``, ``validate_only``. + + Returns: + dict: ``results``, ``stopped_by``, ``usage`` and ``compact``; with + ``validate_only``, the validation report. + """ + if not isinstance(request.get("steps"), list) or not request["steps"]: + raise ValidationError("a wave needs a non-empty 'steps' list") + deadline = request.get("deadline_ms") or 30000 + return self._call("POST", "waves", body=request, write=True, timeout=max(130, int(deadline / 1000) + _WAIT_MARGIN)) + + # ------------------------------------------------------------------ + # Actions (the approvals inbox) + # ------------------------------------------------------------------ + + def list_actions( + self, + state: str | list[str] | None = None, + action_class: str | None = None, + run_id: str | None = None, + limit: int | None = None, + cursor: str | None = None, + ) -> dict[str, Any]: + """List the org's actions across all runs, what needs a person first. + + Args: + state: One action state or several (``awaiting_approval``, ``proposed``, ...). + action_class: Only this action class. + run_id: Only the actions of this run. + limit: Page size, 1..200. + cursor: ``next_cursor`` of the previous page. + + Returns: + dict: ``actions`` (each with a ``run`` summary) and ``next_cursor``. + """ + if limit is not None and not 1 <= limit <= 200: + raise ValidationError("limit must be between 1 and 200") + if run_id: + _check_id("run id", run_id) + if isinstance(state, (list, tuple)): + state = ",".join(state) + return self._get("actions", self._q(state=state, class_=action_class, run_id=run_id, limit=limit, cursor=cursor)) + + def decide_action( + self, run_id: str, action_id: str, verb: str, note: str | None = None, seconds: int | None = None, + ) -> dict[str, Any]: + """Approve, deny, revert, extend or confirm an action. + + The caller also needs the permission of the action's class. + + Args: + run_id: The run the action belongs to. + action_id: The action id. + verb: One of :data:`ACTION_VERBS`. + note: A reason (deny, revert). + seconds: How long to extend a lease (extend). + + Returns: + dict: The action after the decision. + + Raises: + ValidationError: If the verb is unknown. + """ + if verb not in ACTION_VERBS: + raise ValidationError(f"verb must be one of {', '.join(ACTION_VERBS)}") + body: dict[str, Any] = {} + if note: + body["note"] = note + if seconds is not None: + body["seconds"] = seconds + path = f"runs/{_quote(_check_id('run id', run_id), safe='')}/actions/{_quote(_check_id('action id', action_id), safe='')}/{verb}" + return self._call("POST", path, body=body, write=True) + + # ------------------------------------------------------------------ + # Templates (hive records) + # ------------------------------------------------------------------ + + def _plans(self) -> Hive: + return Hive(self._org, PLAN_HIVE) + + def get_template(self, name: str) -> dict[str, Any]: + """Read a stored template. + + Args: + name: The template (record) name. + + Returns: + dict: ``name``, ``status``, ``yaml`` (the source), ``data`` (the + record), ``etag``, ``tags``, ``comment``, ``enabled`` and + modification info. + + Raises: + ApiError: If the record does not exist or cannot be read. + """ + return self._template_view(self._plans().get(name)) + + @staticmethod + def _template_view(record: HiveRecord) -> dict[str, Any]: + data = record.data or {} + return { + "name": record.name, + "status": data.get("status") or STATUS_DRAFT, + "version": data.get("version"), + "yaml": dump_template(data), + "data": data, + "etag": record.etag, + "tags": record.tags or [], + "comment": record.comment, + "enabled": record.enabled, + "last_author": record.last_author, + "last_modified": record.last_modified, + } + + def save_template( + self, + name: str, + source: str, + *, + status: str | None = None, + overwrite: bool = False, + comment: str | None = None, + tags: list[str] | None = None, + ignore_source_status: bool = False, + ) -> dict[str, Any]: + """Validate a template and write it as an ``investigation_plan`` record. + + The record is a DRAFT unless ``status="active"`` is passed. The + source may not say ``status: active`` by itself: activating is an + explicit act of a person, not something a template can smuggle in. + Nothing is written when validation reports an error, when the record + exists and ``overwrite`` is false, or when the record is active and + the new status is not (an active template is replaced only by an + explicit ``status="active"`` with ``overwrite=True``). + + Args: + name: Record name: lowercase letters, digits, ``-`` and ``_``, at most 64. + source: The template YAML, without a ``name:`` key. + status: ``draft`` (default) or ``active``. + overwrite: Replace an existing draft. + comment: Stored as the record's comment (what it was tested on). + tags: Tags to add to the record (an existing record keeps its own). + ignore_source_status: Replace whatever ``status`` the source says + with ``status`` instead of refusing an ``active`` source. For + installing library templates, which are written as active. + + Returns: + dict: ``saved`` (bool). When true: ``name``, ``status``, ``etag``, + ``warnings``. When false: ``reason`` and, for an invalid + template, the ``validation`` report. + + Raises: + ValidationError: For a malformed name, status or source. + InvestigationError: On engine failures. + ApiError: On hive failures. + """ + if not TEMPLATE_NAME_RE.match(name or ""): + raise ValidationError( + "name must be lowercase letters, digits, - and _, start with a letter or digit and be at most 64 characters" + ) + explicit_active = status == STATUS_ACTIVE + if status not in (None, STATUS_DRAFT, STATUS_ACTIVE): + raise ValidationError("status must be draft or active") + data = parse_template_source(source) + if "name" in data: + raise ValidationError( + "the template has a `name:` key; the name is the record's name and is not part of the YAML" + ) + written_status = data.get("status") + if written_status not in (None, STATUS_DRAFT, STATUS_ACTIVE): + raise ValidationError(f"the template sets status to {written_status!r}; use draft or active") + if written_status == STATUS_ACTIVE and not explicit_active and not ignore_source_status: + return { + "saved": False, + "reason": "the template sets status: active. A template cannot activate itself: leave status out " + "(it is saved as a draft), or pass status=active explicitly as the person who reviewed it.", + } + final_status = STATUS_ACTIVE if explicit_active else STATUS_DRAFT + data["status"] = final_status + + report = self.validate_plan(dump_template(data)) + if report.get("valid") is not True or report.get("errors"): + return { + "saved": False, + "reason": "the template does not validate; fix every error and call again", + "validation": report, + } + + hive = self._plans() + record = HiveRecord(name=name, data=data) + try: + existing = hive.get(name) + except ApiError as exc: + if not is_record_not_found(exc): + raise + existing = None + if existing is None: + record.etag = _CREATE_ONLY_ETAG + else: + stored = (existing.data or {}).get("status") or STATUS_DRAFT + if stored == STATUS_ACTIVE and final_status != STATUS_ACTIVE: + return { + "saved": False, + "reason": f"template {name!r} is active: a draft never replaces an active template. " + "Save your version under a new name; a person decides whether it replaces the active one.", + } + if stored not in (STATUS_DRAFT, STATUS_ACTIVE): + return {"saved": False, "reason": f"record {name!r} has status {stored!r}, which is neither draft nor active: save under another name"} + if not overwrite: + return { + "saved": False, + "reason": f"a {stored} template named {name!r} already exists: pass overwrite to replace it, or save under another name", + } + record.etag = existing.etag + if tags or comment: + # Metadata replaces the stored metadata wholesale, and a block + # sent without `enabled` is stored as disabled: start from what is + # stored and add to it. + if existing is not None: + record.tags, record.comment, record.expiry = existing.tags, existing.comment, existing.expiry + record.ui_actions = existing.ui_actions + record.enabled = True if existing is None or existing.enabled is None else existing.enabled + if tags: + record.tags = list(dict.fromkeys((record.tags or []) + tags)) + if comment: + record.comment = comment + hive.set(record) + + # Say what is stored, not what was sent. + stored_record = hive.get(name) + stored_status = (stored_record.data or {}).get("status") or STATUS_DRAFT + if stored_status != final_status: + raise ApiError( + f"the record {name!r} was written but its status reads {stored_status!r}, not {final_status!r}: " + "have a person check it in the console" + ) + return { + "saved": True, + "name": name, + "status": stored_status, + "etag": stored_record.etag, + "last_author": stored_record.last_author, + "warnings": report.get("warnings") or [], + "next_step": ( + "A draft runs only through a test or a run a person starts, with actions capped at suggest. " + "A person reviews it and promotes it to active." + if stored_status == STATUS_DRAFT else "The template is active." + ), + } + + def promote_template(self, name: str) -> dict[str, Any]: + """Make a stored draft template active. + + This is the person's decision step. The stored template is validated + again first and nothing changes when it has errors. The write is + conditional on the etag that was read, so a concurrent edit is not + overwritten. + + Args: + name: The template (record) name. + + Returns: + dict: ``promoted`` (bool), ``name``, ``status`` and, when not + promoted, ``reason``/``validation``. + """ + hive = self._plans() + record = hive.get(name) + data = dict(record.data or {}) + if (data.get("status") or STATUS_DRAFT) == STATUS_ACTIVE: + return {"promoted": False, "name": name, "status": STATUS_ACTIVE, "reason": "the template is already active"} + data["status"] = STATUS_ACTIVE + report = self.validate_plan(dump_template(data)) + if report.get("valid") is not True or report.get("errors"): + return {"promoted": False, "name": name, "status": STATUS_DRAFT, + "reason": "the template does not validate", "validation": report} + hive.set(HiveRecord(name=name, data=data, etag=record.etag)) + stored = hive.get(name) + return { + "promoted": (stored.data or {}).get("status") == STATUS_ACTIVE, + "name": name, + "status": (stored.data or {}).get("status"), + "etag": stored.etag, + "warnings": report.get("warnings") or [], + } + + def delete_template(self, name: str) -> dict[str, Any]: + """Delete a stored template record. + + Args: + name: The template (record) name. + + Returns: + dict: The hive's answer. + """ + return self._plans().delete(name) + + def install_default_templates( + self, + names: list[str] | None = None, + *, + overwrite: bool = False, + status: str = STATUS_DRAFT, + ) -> dict[str, Any]: + """Write the engine's default template library to the org's records. + + Reads the authoring kit's ``library`` section. An existing record is + skipped unless ``overwrite`` is set, and an active record is never + replaced by a draft. The result lists everything written and skipped, + with the installed and library versions so an update can be offered. + + Args: + names: Install only these library templates (default: all). + overwrite: Replace existing drafts (and existing active records + only when ``status`` is ``active``). + status: ``draft`` (default) or ``active``. Library entries are + written with this status whatever they say themselves. + + Returns: + dict: ``written``, ``skipped`` (each with a ``reason``) and + ``failed`` lists. + + Raises: + ValidationError: If the engine returns no library, a requested + name is not in it, or ``status`` is invalid. + """ + if status not in (STATUS_DRAFT, STATUS_ACTIVE): + raise ValidationError("status must be draft or active") + kit = self.get_authoring_kit(["library"]) + library = kit.get("library") + if not isinstance(library, list): + raise ValidationError( + "the engine returned no `library` section: this engine version has no default template library yet" + ) + by_name = {str(e.get("name")): e for e in library if isinstance(e, dict)} + if names: + missing = [n for n in names if n not in by_name] + if missing: + raise ValidationError( + f"not in the default library: {', '.join(missing)}. Available: {', '.join(sorted(by_name)) or 'none'}" + ) + selected = [by_name[n] for n in names] + else: + selected = [by_name[n] for n in sorted(by_name)] + result: dict[str, list[dict[str, Any]]] = {"written": [], "skipped": [], "failed": []} + for entry in selected: + name = str(entry.get("name")) + try: + outcome = self.save_template( + name, entry.get("yaml") or "", status=status, overwrite=overwrite, + comment=f"default library template v{entry.get('version', '?')}", + ignore_source_status=True, + ) + except LimaCharlieError as exc: + result["failed"].append({"name": name, "error": exc.raw_message}) + continue + if outcome.get("saved"): + result["written"].append({"name": name, "status": outcome["status"], "version": entry.get("version")}) + continue + item: dict[str, Any] = {"name": name, "reason": outcome.get("reason"), "library_version": entry.get("version")} + if "validation" in outcome: + result["failed"].append({"name": name, "error": outcome["reason"], "validation": outcome["validation"]}) + continue + try: + installed = self.get_template(name) + except LimaCharlieError: + installed = None + if installed is not None: + item["installed_version"] = installed.get("version") + item["installed_status"] = installed.get("status") + item["update_available"] = installed.get("version") != entry.get("version") + result["skipped"].append(item) + return result From 1f90fb08c789e05840268f9b07b2840d4be4889e Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 13:16:55 -0800 Subject: [PATCH 02/14] Investigation Engine: tests, docs, help topic, changelog Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 23 + doc/README.md | 1 + doc/cli/README.md | 1 + doc/cli/investigation-engine.md | 208 +++++++ doc/sdk/other-classes.md | 4 + limacharlie/help_topics.py | 37 ++ limacharlie/sdk/investigation.py | 7 +- tests/unit/test_cli_investigate.py | 499 +++++++++++++++ .../unit/test_cli_lazy_loading_regression.py | 4 +- tests/unit/test_sdk_investigation.py | 569 ++++++++++++++++++ 10 files changed, 1351 insertions(+), 2 deletions(-) create mode 100644 doc/cli/investigation-engine.md create mode 100644 tests/unit/test_cli_investigate.py create mode 100644 tests/unit/test_sdk_investigation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a3dad375..c29b3e70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ ## Unreleased +### Investigation Engine — `limacharlie investigate` (experimental) + +- New `investigate` command group and `limacharlie.sdk.investigation.Investigation` + SDK class to build, test and tune Investigation Engine templates and to run + investigations, so a person or an AI agent can drive the whole loop from the + CLI: `kit` (authoring kit, fetched in sections), `operators`, `packs`, + `templates list|get|validate|test|save|promote|delete|install-defaults`, + `runs start|get|list|events|steps|case|cancel`, `wave`, + `actions list|approve|deny` and `policy get|status|pause|resume`. See + `doc/cli/investigation-engine.md`. +- `templates save` always writes a DRAFT (a source that says `status: active` + is refused; only a person's explicit `--status active` activates), never + overwrites a record without `--overwrite`, and never replaces an active + template with a draft. `templates test` dry-runs a template on real past + detections or cases and prints per-sample verdicts and the agreement summary. +- Engine failures keep the engine's coded issues (`code`, `path`, `message`, + `suggestion`) and error code verbatim, and say when the feature is not + enabled for the deployment or the permissions are missing. +- `templates install-defaults` reads the authoring kit's `library` section; it + needs an engine version that serves it. +- The typed 401/403/404/422/429 errors raised for an HTTP error answer now carry + the parsed answer as `response_body`, as `ApiError` already did. + ### Sync — `sync push` no longer turns quoted strings into numbers - `sync push` (and `Configs.push` / `Configs.push_from_file`) stored some diff --git a/doc/README.md b/doc/README.md index 8a3b880b..cc01e74e 100644 --- a/doc/README.md +++ b/doc/README.md @@ -18,6 +18,7 @@ | [Hive & Data Stores](cli/hive-data.md) | hive, secret, lookup, playbook, note, sop, app-control, adapter, cloud-sensor, extension | | [Infrastructure](cli/infrastructure.md) | sync, output, artifact, payload, yara, integrity, logging, exfil | | [Cloud Security & Code Security](cli/cloud-security.md) | cloudsec (findings, inventory, graph, compliance, CAASM, code lane, container images, fleet, exports) | +| [Investigation Engine](cli/investigation-engine.md) | investigate (authoring kit, templates, dry runs on past alerts, runs, waves, approvals, policy) | | [Email Security](cli/email-security.md) | mailsec (onboarding, coverage, triage, EML, remediation, campaigns, reports, sample submission, rules, tenant purge) | | [Other Commands](cli/other-commands.md) | api, arl, usp, spotcheck, job, schema, completion, help/discover | diff --git a/doc/cli/README.md b/doc/cli/README.md index 81625c3a..bb21ab07 100644 --- a/doc/cli/README.md +++ b/doc/cli/README.md @@ -162,6 +162,7 @@ limacharlie schema dr create | [Infrastructure](infrastructure.md) | sync, output, artifact, payload, yara, integrity, logging, exfil | | [Cloud Security & Code Security](cloud-security.md) | cloudsec (findings, inventory, graph, compliance, CAASM, code lane, container images, fleet, exports) | | [Email Security](email-security.md) | mailsec (onboarding, coverage, triage, EML, remediation, campaigns, reports, sample submission, rules, tenant purge) | +| [Investigation Engine](investigation-engine.md) | investigate (authoring kit, templates, dry runs on past alerts, runs, waves, approvals, policy) | | [Other Commands](other-commands.md) | api, arl, usp, spotcheck, job, schema, completion, help/discover, case | ## See Also diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md new file mode 100644 index 00000000..d2e658ff --- /dev/null +++ b/doc/cli/investigation-engine.md @@ -0,0 +1,208 @@ +[Documentation](../README.md) > [CLI](README.md) > Investigation Engine + +# Investigation Engine + +Install or upgrade with `python -m pip install --upgrade limacharlie`. See [installation](../getting-started.md#installation) for setup. + +The `investigate` group builds, tests and tunes Investigation Engine templates and runs investigations. The engine gathers facts for an alert with typed operators, weighs them and reaches a verdict (`true_positive`, `false_positive` or `inconclusive`) with a proof. A person or an AI agent (Claude Code, Codex, ...) can drive the whole loop from the CLI. + +The Investigation Engine is **experimental and gated per deployment**. Where it is not enabled the API does not serve the routes and the CLI says so (a 404 with no engine error code). Where your organization has no engine, the API answers 503. + +Every command supports `--ai-help`, and `limacharlie help investigation-engine` prints the workflow below. Output is JSON when piped (`--output` selects another format); see [output formats](README.md#output-formats). + +## Permissions + +All the listed permissions are required. + +| Operation | Permissions | +|---|---| +| Reads: runs, templates, operators, packs, kit, policy, test results, the approvals inbox | `investigation.get` + `insight.evt.get` | +| Writes: start a run, wave, cancel, test a template, decide an action | `investigation.set` + `insight.evt.get` | +| Approve / deny an action | the writes above + the permission of the action's class (for example `sensor.task`, `mailsec.act`, `sensor.tag`) | +| `policy pause` / `policy resume` | `org.conf.set` | +| Template records (`investigation_plan` hive) | `investigation_plan.get`, `investigation_plan.set`, `investigation_plan.del` (explicit grants; no default role has them) | + +Runs, waves and tests execute with **your own token**, so the operators enforce your permissions. + +## AI agent workflow + +Hand an agent this loop. It never activates anything on its own: a person does the last step. + +```bash +# 1. Learn the language, in parts (the whole kit is ~33k tokens) +limacharlie investigate kit --section guide --section policy --section codes +limacharlie investigate kit --section schema --section library +limacharlie investigate packs --brief +limacharlie investigate operators --brief + +# 2. Look at real data, try operators and packs with a wave +limacharlie investigate wave --file wave.yaml --validate-only +limacharlie investigate wave --file wave.yaml + +# 3. Draft plan.yaml, then validate until there are no errors +limacharlie investigate templates validate --file plan.yaml + +# 4. Test on past alerts; read validation, summary, each disagree, then undecided +limacharlie investigate templates test --file plan.yaml --detect-cat my-category --last 10 + +# 5. Refine and repeat 3-4, then save a DRAFT with what you tested +limacharlie investigate templates save my-plan --file plan.yaml \ + --comment "10 samples: 9 agree, 1 undecided (index_lag), 0 errors" + +# 6. A PERSON reviews the draft and its test results, then promotes it +limacharlie investigate templates promote my-plan --confirm +``` + +Rules the CLI enforces: + +- `templates save` writes a **draft**. A source that says `status: active` is refused, and only a person's explicit `--status active` writes an active template. AI agents must not pass `--status active` and must not run `templates promote`. +- A record is never overwritten without `--overwrite`, and a draft **never** replaces an active template (save under a new name; a person decides whether it replaces the active one). +- A draft runs only through `templates test` or a run a person starts, with response actions capped at `suggest`. A run started by a D&R rule or a case never uses a draft. +- Nothing is saved when validation reports an error. + +## Reference data + +```bash +limacharlie investigate kit [--section schema|guide|operators|examples|policy|codes|packs|library]... +limacharlie investigate operators [--brief] [--name OP]... [--match TEXT] +limacharlie investigate packs [--brief] [--name PACK] +``` + +`kit` returns the template JSON Schema, the authoring guide, the org's typed operator catalogue (and which operators the org may use), examples, the effective policy and budgets, every validation code with its usual fix, and the fact packs. The `size` entry of the answer says what was returned. `--section library` returns the default template library; it needs an engine version that serves that section. + +## Templates + +Templates are plain Hive records (`investigation_plan`). Their YAML cannot contain a `name:` key: the record name is the template name (lowercase letters, digits, `-` and `_`, at most 64 characters). + +```bash +limacharlie investigate templates list [--status draft|active] +limacharlie investigate templates get NAME [--yaml] +limacharlie investigate templates validate --file plan.yaml # '-' reads stdin; exit 4 when invalid +limacharlie investigate templates test [NAME | --file plan.yaml] [samples] [options] +limacharlie investigate templates save NAME --file plan.yaml [--overwrite] [--comment TEXT] [--tag T]... +limacharlie investigate templates promote NAME --confirm +limacharlie investigate templates delete NAME --confirm +limacharlie investigate templates install-defaults [--name NAME]... [--overwrite] +``` + +`validate` prints the report: `valid`, `errors` and `warnings` (each `{code, path, message, suggestion?}`), the worst-case step count and an `estimate` of the worst-case cost. + +### Test on past alerts + +`templates test` dry-runs a template on real past alerts. The plan's read-only operators run with your token; nothing is written (no run, case, label or action). Response steps that would run are only listed in `would_propose`. A test counts against the org's concurrent run cap and may bill LLM steps. + +Samples (at most 25 after expansion): + +| Flag | Sample | +|---|---| +| `--detection-id ID` | a stored detection | +| `--case-id ID` | a case; a closed case's classification is the expected label | +| `--detect-cat CATEGORY --last N` | the newest N alerts of a category (N is 1..25) | +| `--sample '{"inputs": {...}, "expected": "false_positive"}'` | free inputs with an optional expected label | +| `--samples-file FILE` | a YAML/JSON list of sample objects | + +Use 5 to 10 samples and include known false **and** true positives. Each sample reports a verdict, the `expected` label when there is one, and an `agreement` of `agree`, `disagree`, `undecided` or `none`; `summary.agreement_rate` is the share that agreed. `unknowns` lists the facts that could not be resolved and why (the usual cause is a wrong field path or argument). Steps that succeeded are dropped from the output (`problem_steps` keeps the others); `--include-steps` keeps all. + +The command waits for the test (`--timeout`, default 600 s). A test still running prints its `test_id`; continue with `--test-id`. `--no-wait` returns right after the start. In table output the per-sample verdicts and the agreement summary are printed as text. + +Exit codes: `0` done, `4` the template is `invalid` (the `validation` report says why), `1` the test failed. + +### Save, promote, delete + +`save` validates the exact text it stores, then writes the record as a draft and reads it back. A refusal (validation errors, existing record, active record) prints `{"saved": false, "reason": ...}` and exits `4`. `promote` validates again and writes conditionally on the version it read. `install-defaults` writes the engine's default template library, **never overwriting silently**: existing records are skipped with their installed and library versions (so an available update is visible) unless `--overwrite`. Library templates are installed as drafts; use `--status active` (a person's choice) to install them active. + +## Runs + +```bash +limacharlie investigate runs start (--template NAME | --file plan.yaml | --auto) \ + (--detection-id ID | --case-id ID | --entity-id ID | --question TEXT) \ + [--input K=V]... [--inputs-json JSON] [--urgency low|normal|high] [--mode triage|full|scope] \ + [--case-mode create|none|attach:ID] [--idempotency-key KEY] [--wait SECONDS] [--follow] +limacharlie investigate runs get RUN_ID +limacharlie investigate runs list [--state S] [--verdict V] [--template T] [--trigger T] [--entity E] \ + [--started-by user:UID] [--since TIME] [--until TIME] [--limit N] [--cursor C | --all] +limacharlie investigate runs events RUN_ID [--after SEQ] [--wait SECONDS] [--limit N] [--follow [--timeout S]] +limacharlie investigate runs steps RUN_ID [--values] +limacharlie investigate runs case RUN_ID +limacharlie investigate runs cancel RUN_ID [--revert-leases] +``` + +`--auto` picks the active template whose `applies_to` matches the alert. A start is sent once and never retried automatically; `--idempotency-key` makes a repeated start return the same run (`joined: true`). + +`runs list` pages: structured formats print `{"runs": [...], "next_cursor": "..."}` and table/CSV/JSONL print rows with the cursor on stderr. Continue with `--cursor`, or use `--all`. + +`runs events --follow` polls until the run is over and writes **one JSON event per line** (a text line in table mode), flushing as they arrive. It exits `0` when the run finished and `1` when `--timeout` (default 900 s) elapsed first; the resume point is printed on stderr (`--after N`). Events carry structure only, never customer values. + +## Waves + +A wave is a batch of questions: the engine runs every ready step in parallel with your token (no template, no verdict, nothing stored), reuses what it already knows and can stop as soon as the answer is known. + +```yaml +# wave.yaml +steps: + - {id: host, use: host_context, args: {sid: ""}} + - {id: prev, op: ioc.prevalence, args: {file_hash: ""}, select: [hosts_30d, first_seen]} + - {id: kids, op: event.children, args: {sid: "", atom: ""}, limit: 5} +stop_when: + - "prev.hosts_30d >= 50 and kids.count == 0" +deadline_ms: 20000 +``` + +```bash +limacharlie investigate wave --file wave.yaml [--validate-only] [--full] \ + [--stop-when EXPR]... [--deadline-ms N] [--input K=V]... +``` + +A step has an `id` and exactly one of `op` (an operator) or `use` (a fact pack). The answer is compact: per step `status` (`ok`, `unknown` with a `reason`, `cancelled`, `partial`), `complete` (`false` means a lower bound, not proof of absence), `ms` and the selected `value`, plus `stopped_by` and `usage`. An `unknown` step established nothing: it is never an absence. `--full` returns raw values (many more tokens). + +An invalid wave prints `{"valid": false, "errors": [...], "warnings": [...]}` with the engine's coded errors on **stdout** and exits `4`; `--validate-only` checks without running anything. + +## Approvals and policy + +```bash +limacharlie investigate actions list [--state awaiting_approval,proposed] [--class C] [--run-id ID] [--limit N] [--cursor C | --all] +limacharlie investigate actions approve RUN_ID ACTION_ID +limacharlie investigate actions deny RUN_ID ACTION_ID [--note TEXT] +limacharlie investigate policy get [--category C] [--priority N] [--severity S] [--target-tag T]... [--class C] +limacharlie investigate policy status +limacharlie investigate policy pause +limacharlie investigate policy resume +``` + +`actions list` is the approvals inbox across all runs; what needs a person comes first. `policy pause` stops automatic actions for the whole org (the kill switch) and needs `org.conf.set`. + +## Errors + +In a structured output format (`--output json`, the default when piped) a failure is written to **stderr** as the engine's answer and the command exits non-zero: + +```json +{ + "error": "the wave is invalid\n [unknown_field] steps[0].select[0]: no such field (did you mean hosts_30d)", + "status": 400, + "error_code": "bad_request", + "errors": [{"code": "unknown_field", "path": "steps[0].select[0]", "message": "no such field", "suggestion": "hosts_30d"}], + "warnings": [] +} +``` + +`error_code` is the engine's (`bad_request`, `forbidden`, `not_found`, `conflict`, `too_many_runs`, `not_implemented`, `unavailable`). It is absent when the answer did not come from the engine: a 404 without it means the feature is not enabled, and `suggestion` says what to check. Exit codes follow the CLI's: `2` authentication or permission, `3` not found, `4` validation or a refused result, `5` rate limit. + +## SDK + +```python +from limacharlie.sdk.organization import Organization +from limacharlie.client import Client +from limacharlie.sdk.investigation import Investigation, InvestigationError + +inv = Investigation(Organization(Client())) +report = inv.validate_plan(open("plan.yaml").read()) +test = inv.wait_for_test(inv.test_plan([{"detect_cat": "my-category", "last": 10}], inline=open("plan.yaml").read())) +result = inv.save_template("my-plan", open("plan.yaml").read(), comment="10 samples") # a draft +``` + +`InvestigationError` carries `status_code`, `error_code`, `issues` and `warnings`. + +## See Also + +- [Hive & Data Stores](hive-data.md) - the `investigation_plan` records are ordinary hive records +- [Detection & Response](detection-response.md) - D&R rules that start runs diff --git a/doc/sdk/other-classes.md b/doc/sdk/other-classes.md index 31abdd91..9cbcebd6 100644 --- a/doc/sdk/other-classes.md +++ b/doc/sdk/other-classes.md @@ -194,6 +194,10 @@ The Cloud Security (CNAPP) surface also includes Code Security: repository and c `limacharlie.sdk.mailsec.Mailsec` wraps Email Security onboarding, provider tests, coverage, messages, justified EML downloads, verdict revisions, campaigns, remediation, and user reports. See [Security Products](security-products.md#email-security) for setup, examples, and pagination, and [CLI: mailsec](../cli/email-security.md) for complete workflows. +## Investigation + +`limacharlie.sdk.investigation.Investigation` wraps the Investigation Engine (experimental): the authoring kit, plan validation and dry runs on past alerts, template records (saved as drafts unless a person passes `status="active"`), runs, waves, the approvals inbox and the autonomy switch. Failures raise `InvestigationError`, which keeps the engine's `error_code` and coded `issues`. See [CLI: investigate](../cli/investigation-engine.md) for the workflow and permissions. + ## See Also - [CLI: case](../cli/other-commands.md#case) — Case CLI commands diff --git a/limacharlie/help_topics.py b/limacharlie/help_topics.py index 1973fe39..ef73815d 100644 --- a/limacharlie/help_topics.py +++ b/limacharlie/help_topics.py @@ -770,6 +770,43 @@ Quick reference: limacharlie help cheatsheet --name email-security """ +HELP_TOPICS["investigation-engine"] = """\ +Investigation Engine +==================== + +The Investigation Engine (experimental, gated per deployment) gathers facts +for an alert with typed operators, weighs them and reaches a verdict with a +proof. Its plans are YAML templates, stored as `investigation_plan` hive +records. `limacharlie investigate` builds, tests and runs them. + +Author a template (an AI agent or a person): + 1. limacharlie investigate kit --section guide --section schema --section codes + 2. draft the YAML (limacharlie investigate packs --brief, operators --brief) + 3. limacharlie investigate templates validate --file plan.yaml + 4. limacharlie investigate templates test --file plan.yaml --detect-cat --last 10 + Read `validation`, `summary`, each `disagree`, then `undecided` samples. + 5. refine, repeat 3-4 + 6. limacharlie investigate templates save my-plan --file plan.yaml --comment "" + Always a DRAFT. A draft runs only through `templates test` or a run a + person starts, with response actions capped at `suggest`. + 7. A PERSON reviews and promotes: limacharlie investigate templates promote my-plan --confirm + AI agents must not promote or save with --status active. + +Questions without a template: `investigate packs --brief`, then +`investigate wave --file wave.yaml` (steps with `use:` packs, `select`, +`stop_when`). + +Runs: `investigate runs start|get|list|events --follow|steps|case|cancel`. +Approvals: `investigate actions list --state awaiting_approval`, `approve`, `deny`. +Policy: `investigate policy get|status|pause|resume`. + +Errors carry the engine's coded issues ({code, path, message, suggestion}). +A 404 with no engine error means the feature is not enabled for the region. + +Commands: limacharlie help discover --profile investigation_engine +Reference: doc/cli/investigation-engine.md +""" + # --------------------------------------------------------------------------- # Cheatsheets registry # --------------------------------------------------------------------------- diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index 316def91..59920ed5 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -194,7 +194,7 @@ def _wrap_error(exc: LimaCharlieError) -> LimaCharlieError: issues=issues, warnings=warnings, response_body=body, - suggestion=suggestion, + suggestion=suggestion or "", # "": no generic advice when the engine's own message is enough exit_code=exc.exit_code, ) @@ -1141,5 +1141,10 @@ def install_default_templates( item["installed_version"] = installed.get("version") item["installed_status"] = installed.get("status") item["update_available"] = installed.get("version") != entry.get("version") + if not overwrite: + item["reason"] = ( + f"a {installed.get('status')} template named {name!r} is already installed; " + "pass overwrite to replace it" + ) result["skipped"].append(item) return result diff --git a/tests/unit/test_cli_investigate.py b/tests/unit/test_cli_investigate.py new file mode 100644 index 00000000..a714a2c6 --- /dev/null +++ b/tests/unit/test_cli_investigate.py @@ -0,0 +1,499 @@ +"""Tests for the ``limacharlie investigate`` CLI group.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest +from click.testing import CliRunner + +from limacharlie.cli import cli +from limacharlie.errors import error_from_status_code +from limacharlie.sdk.investigation import InvestigationError + + +@pytest.fixture +def sdk(): + """A mocked Investigation instance behind the CLI (no Client, no network).""" + with patch("limacharlie.commands.investigate.Client"), \ + patch("limacharlie.commands.investigate.Organization"), \ + patch("limacharlie.commands.investigate.Investigation") as cls: + inst = MagicMock() + cls.return_value = inst + yield inst + + +def run(args, input=None, fmt="json"): + result = CliRunner(mix_stderr=False).invoke(cli, ["--output", fmt] + args, input=input) + return result + + +def out_json(result): + return json.loads(result.stdout) + + +def issue_error(status=400, issues=None): + issues = issues or [{"code": "unknown_field", "path": "steps[0].select[0]", "message": "no field", "suggestion": "hosts_30d"}] + return InvestigationError( + "the wave is invalid", status_code=status, issues=issues, warnings=[{"code": "operator_denied"}], + response_body={"error": "the wave is invalid", "error_code": "bad_request", "errors": issues}, + exit_code=1, + ) + + +class TestHelp: + def test_group_and_nested_help(self): + runner = CliRunner() + out = runner.invoke(cli, ["investigate", "--help"]).output + for name in ("kit", "operators", "packs", "templates", "runs", "wave", "actions", "policy"): + assert name in out + out = runner.invoke(cli, ["investigate", "templates", "--help"]).output + for name in ("list", "get", "validate", "test", "save", "promote", "delete", "install-defaults"): + assert name in out + + def test_ai_help_carries_the_agent_workflow(self): + result = CliRunner().invoke(cli, ["investigate", "--ai-help"]) + assert result.exit_code == 0 + assert "promote" in result.output and "validate" in result.output + + def test_kit_section_is_a_closed_set(self): + result = CliRunner().invoke(cli, ["investigate", "kit", "--section", "bogus"]) + assert result.exit_code != 0 + + +class TestKitAndCatalogues: + def test_kit_sections_are_passed_through(self, sdk): + sdk.get_authoring_kit.return_value = {"guide": "g", "size": {"approx_tokens": 10}} + result = run(["investigate", "kit", "--section", "guide", "--section", "library"]) + assert result.exit_code == 0, result.stderr + sdk.get_authoring_kit.assert_called_once_with(["guide", "library"]) + assert out_json(result)["guide"] == "g" + + def test_no_section_asks_for_the_default_kit(self, sdk): + sdk.get_authoring_kit.return_value = {} + run(["investigate", "kit"]) + sdk.get_authoring_kit.assert_called_once_with(None) + + def test_operators_brief_and_name_filters(self, sdk): + sdk.list_operators.return_value = { + "operators": [ + {"name": "ioc.prevalence", "class": "index", "description": "how widespread", "available": True, "why": "", "args": [1], "output": {}}, + {"name": "entity.get", "class": "free", "description": "an entity", "available": False, "why": "denied", "args": [2]}, + ], + "extension_requests": ["ext/a"], + } + data = out_json(run(["investigate", "operators", "--brief", "--name", "entity.get"])) + assert data["operators"] == [{"name": "entity.get", "class": "free", "description": "an entity", "available": False, "why": "denied"}] + assert data["extension_requests"] == ["ext/a"] + data = out_json(run(["investigate", "operators", "--match", "WIDESPREAD"])) + assert [o["name"] for o in data["operators"]] == ["ioc.prevalence"] + assert "args" in data["operators"][0] + + def test_packs_brief_is_a_compact_index(self, sdk): + sdk.list_packs.return_value = {"packs": [{ + "name": "host_context", "brief": "who", "description": "long", "version": 1, + "params": [{"name": "sid", "type": "string", "required": True, "description": "Sensor id."}], + "exports": [{"name": "hostname", "type": "string", "ref": "__card.display_name", "brief": "name", "description": "long"}], + }]} + data = out_json(run(["investigate", "packs", "--brief", "--name", "host_context"])) + sdk.list_packs.assert_called_once_with("host_context") + assert data["packs"] == [{ + "name": "host_context", "brief": "who", + "params": [{"name": "sid", "type": "string", "required": True}], + "exports": [{"name": "hostname", "type": "string", "ref": "__card.display_name", "brief": "name"}], + }] + + +class TestTemplates: + def test_list_filters_by_status(self, sdk): + sdk.list_engine_templates.return_value = {"templates": [ + {"name": "a", "status": "active"}, {"name": "b", "status": "draft"}, {"name": "c"}]} + data = out_json(run(["investigate", "templates", "list", "--status", "draft"])) + assert [t["name"] for t in data["templates"]] == ["b", "c"] # no status means draft + + def test_get_yaml_prints_only_the_source(self, sdk): + sdk.get_template.return_value = {"name": "a", "yaml": "description: x\n"} + result = run(["investigate", "templates", "get", "a", "--yaml"]) + assert result.stdout == "description: x\n" + + def test_validate_exit_code_follows_the_report(self, sdk, tmp_path): + f = tmp_path / "plan.yaml" + f.write_text("description: x\n") + sdk.validate_plan.return_value = {"valid": True, "errors": [], "warnings": []} + assert run(["investigate", "templates", "validate", "--file", str(f)]).exit_code == 0 + sdk.validate_plan.assert_called_with("description: x\n") + report = {"valid": False, "errors": [{"code": "unknown_field", "path": "p", "message": "m", "suggestion": "s"}], "warnings": []} + sdk.validate_plan.return_value = report + result = run(["investigate", "templates", "validate", "--file", str(f)]) + assert result.exit_code == 4 + assert out_json(result)["errors"] == report["errors"] + + def test_validate_reads_stdin(self, sdk): + sdk.validate_plan.return_value = {"valid": True, "errors": []} + run(["investigate", "templates", "validate", "--file", "-"], input="description: piped\n") + sdk.validate_plan.assert_called_with("description: piped\n") + + def test_save_is_a_draft_unless_a_person_says_otherwise(self, sdk, tmp_path): + f = tmp_path / "plan.yaml" + f.write_text("description: x\n") + sdk.save_template.return_value = {"saved": True, "status": "draft"} + assert run(["investigate", "templates", "save", "my-plan", "--file", str(f), "--comment", "10 samples"]).exit_code == 0 + sdk.save_template.assert_called_with( + "my-plan", "description: x\n", status="draft", overwrite=False, comment="10 samples", tags=None) + run(["investigate", "templates", "save", "my-plan", "--file", str(f), "--status", "active", "--overwrite", "--tag", "t"]) + sdk.save_template.assert_called_with( + "my-plan", "description: x\n", status="active", overwrite=True, comment=None, tags=["t"]) + + def test_refused_save_exits_4_and_says_why(self, sdk, tmp_path): + f = tmp_path / "plan.yaml" + f.write_text("description: x\n") + sdk.save_template.return_value = {"saved": False, "reason": "already exists", "validation": {"errors": []}} + result = run(["investigate", "templates", "save", "my-plan", "--file", str(f)]) + assert result.exit_code == 4 + assert out_json(result)["reason"] == "already exists" + + def test_promote_and_delete_need_confirm(self, sdk): + for verb in ("promote", "delete"): + result = run(["investigate", "templates", verb, "my-plan"]) + assert result.exit_code != 0 and "--confirm" in result.stderr + sdk.promote_template.assert_not_called() + sdk.delete_template.assert_not_called() + + def test_promote_confirmed(self, sdk): + sdk.promote_template.return_value = {"promoted": True, "status": "active"} + assert run(["investigate", "templates", "promote", "my-plan", "--confirm"]).exit_code == 0 + sdk.promote_template.assert_called_once_with("my-plan") + sdk.promote_template.return_value = {"promoted": False, "reason": "does not validate"} + assert run(["investigate", "templates", "promote", "my-plan", "--confirm"]).exit_code == 4 + + def test_install_defaults_arguments_and_failure_exit(self, sdk): + sdk.install_default_templates.return_value = {"written": [], "skipped": [], "failed": []} + run(["investigate", "templates", "install-defaults", "--name", "a", "--name", "b", "--overwrite"]) + sdk.install_default_templates.assert_called_with(["a", "b"], overwrite=True, status="draft") + run(["investigate", "templates", "install-defaults"]) + sdk.install_default_templates.assert_called_with(None, overwrite=False, status="draft") + sdk.install_default_templates.return_value = {"written": [], "skipped": [], "failed": [{"name": "a"}]} + assert run(["investigate", "templates", "install-defaults"]).exit_code == 1 + + +class TestTemplateTest: + DONE = { + "test_id": "t1", "state": "done", "done": True, "samples_done": 2, "samples_total": 2, + "results": [ + {"index": 0, "sample": {"detection_id": "d1"}, "state": "ok", "verdict": {"label": "false_positive", "p": 0.04}, + "expected": {"label": "false_positive"}, "agreement": "agree", "unknowns": [], + "steps": [{"fact": "a", "status": "ok"}, {"fact": "b", "status": "unknown", "reason": "index_lag"}]}, + {"index": 1, "sample": {"case_id": "9"}, "state": "ok", "verdict": {"label": "true_positive", "p": 0.9}, + "expected": {"label": "false_positive"}, "agreement": "disagree", "unknowns": [{"fact": "b"}]}, + ], + "summary": {"samples": 2, "agree": 1, "disagree": 1, "agreement_rate": 0.5, "unknown_reasons": {"index_lag": 1}}, + } + + def test_samples_are_built_from_every_flag(self, sdk, tmp_path): + sdk.test_plan.return_value = dict(self.DONE) + sf = tmp_path / "s.yaml" + sf.write_text("- {detection_id: from-file}\n") + result = run([ + "investigate", "templates", "test", "my-plan", "--detection-id", "d1", "--case-id", "9", + "--detect-cat", "cat-a", "--last", "3", "--sample", '{"inputs": {"k": "v"}, "expected": "false_positive"}', + "--samples-file", str(sf), "--strict", "--mode", "full", "--wait", "2", + ]) + assert result.exit_code == 0, result.stderr + args, kwargs = sdk.test_plan.call_args + assert args[0] == [ + {"detection_id": "d1"}, {"case_id": "9"}, {"detect_cat": "cat-a", "last": 3}, + {"inputs": {"k": "v"}, "expected": "false_positive"}, {"detection_id": "from-file"}, + ] + assert kwargs == {"template": "my-plan", "inline": None, "overrides": None, "strict": True, "mode": "full", "wait_seconds": 2} + + def test_inline_file_is_sent_as_the_plan(self, sdk, tmp_path): + sdk.test_plan.return_value = dict(self.DONE) + f = tmp_path / "plan.yaml" + f.write_text("description: draft\n") + run(["investigate", "templates", "test", "--file", str(f), "--detection-id", "d1"]) + assert sdk.test_plan.call_args.kwargs["inline"] == "description: draft\n" + assert sdk.test_plan.call_args.kwargs["template"] is None + + def test_usage_errors_before_any_call(self, sdk): + assert run(["investigate", "templates", "test", "--detection-id", "d"]).exit_code != 0 # no plan + assert run(["investigate", "templates", "test", "a", "--file", "f", "--detection-id", "d"]).exit_code != 0 + assert run(["investigate", "templates", "test", "a"]).exit_code != 0 # no sample + assert run(["investigate", "templates", "test", "--test-id", "t", "a"]).exit_code != 0 + sdk.test_plan.assert_not_called() + + def test_json_drops_the_steps_that_succeeded(self, sdk): + sdk.test_plan.return_value = dict(self.DONE) + results = out_json(run(["investigate", "templates", "test", "p", "--detection-id", "d1"]))["results"] + assert "steps" not in results[0] + assert results[0]["problem_steps"] == [{"fact": "b", "status": "unknown", "reason": "index_lag"}] + assert results[0]["steps_total"] == 2 + full = out_json(run(["investigate", "templates", "test", "p", "--detection-id", "d1", "--include-steps"]))["results"] + assert len(full[0]["steps"]) == 2 + + def test_a_running_test_is_waited_for_and_a_timeout_leaves_the_handle(self, sdk): + running = {"test_id": "t1", "state": "running", "done": False, "samples_done": 0} + sdk.test_plan.return_value = running + sdk.wait_for_test.return_value = dict(self.DONE) + result = run(["investigate", "templates", "test", "p", "--detection-id", "d", "--timeout", "30"]) + sdk.wait_for_test.assert_called_once_with(running, timeout=30.0) + assert out_json(result)["state"] == "done" + + sdk.wait_for_test.return_value = running + result = run(["investigate", "templates", "test", "p", "--detection-id", "d"]) + assert result.exit_code == 0 + assert "--test-id t1" in result.stderr + + def test_no_wait_returns_the_started_test(self, sdk): + sdk.test_plan.return_value = {"test_id": "t1", "state": "running", "done": False} + result = run(["investigate", "templates", "test", "p", "--detection-id", "d", "--no-wait"]) + sdk.wait_for_test.assert_not_called() + assert out_json(result)["test_id"] == "t1" + + def test_resume_by_test_id(self, sdk): + sdk.get_test_plan.return_value = dict(self.DONE) + result = run(["investigate", "templates", "test", "--test-id", "t1"]) + assert result.exit_code == 0 + assert sdk.get_test_plan.call_args.args == ("t1",) + sdk.test_plan.assert_not_called() + + def test_invalid_and_failed_tests_have_exit_codes(self, sdk): + invalid = {"test_id": "t", "state": "invalid", "done": True, + "validation": {"valid": False, "errors": [{"code": "bad_expression", "path": "x", "message": "m"}]}} + sdk.test_plan.return_value = invalid + result = run(["investigate", "templates", "test", "p", "--detection-id", "d"]) + assert result.exit_code == 4 + assert out_json(result)["validation"]["errors"][0]["code"] == "bad_expression" + sdk.test_plan.return_value = {"test_id": "t", "state": "failed", "done": True, "error": "gone"} + assert run(["investigate", "templates", "test", "p", "--detection-id", "d"]).exit_code == 1 + + def test_table_output_shows_per_sample_verdicts_and_the_agreement_summary(self, sdk): + sdk.test_plan.return_value = dict(self.DONE) + result = run(["investigate", "templates", "test", "p", "--detection-id", "d"], fmt="table") + assert result.exit_code == 0, result.stderr + assert "agree" in result.stdout and "disagree" in result.stdout + assert "agreement_rate=0.5" in result.stdout + assert "unknown reasons: {'index_lag': 1}" in result.stdout + + +class TestRuns: + def test_start_builds_the_request(self, sdk): + sdk.start_run.return_value = {"run": {"run_id": "r1"}, "joined": False} + result = run([ + "investigate", "runs", "start", "--template", "my-plan", "--detection-id", "d1", + "--input", "hash=0e123", "--inputs-json", '{"n": 3}', "--urgency", "high", "--mode", "full", + "--case-mode", "none", "--idempotency-key", "k", "--wait", "20", + ]) + assert result.exit_code == 0, result.stderr + sdk.start_run.assert_called_once_with({ + "trigger": {"detection_id": "d1"}, "plan": {"template": "my-plan"}, + "inputs": {"hash": "0e123", "n": 3}, "urgency": "high", "mode": "full", "case": "none", + "idempotency_key": "k", "wait_seconds": 20, + }) + + def test_start_plan_choices(self, sdk, tmp_path): + sdk.start_run.return_value = {"run": {"run_id": "r1"}} + run(["investigate", "runs", "start", "--auto", "--case-id", "5"]) + assert sdk.start_run.call_args.args[0] == {"trigger": {"case_id": "5"}, "plan": {"auto": True}} + f = tmp_path / "p.yaml" + f.write_text("description: x\n") + run(["investigate", "runs", "start", "--file", str(f), "--question", "why?"]) + assert sdk.start_run.call_args.args[0] == {"trigger": {"question": "why?"}, "plan": {"inline": "description: x\n"}} + + def test_start_usage_errors(self, sdk): + assert run(["investigate", "runs", "start", "--detection-id", "d"]).exit_code != 0 + assert run(["investigate", "runs", "start", "--template", "a", "--auto", "--detection-id", "d"]).exit_code != 0 + assert run(["investigate", "runs", "start", "--template", "a"]).exit_code != 0 + assert run(["investigate", "runs", "start", "--template", "a", "--detection-id", "d", "--input", "novalue"]).exit_code != 0 + sdk.start_run.assert_not_called() + + def test_start_follow_streams_the_new_run(self, sdk): + sdk.start_run.return_value = {"run": {"run_id": "r1"}, "joined": False} + + def events(run_id, **kw): + state = kw["state"] + state.update(next=2, done=True) + yield {"seq": 1, "type": "step", "data": {}} + yield {"seq": 2, "type": "verdict", "data": {}} + sdk.iter_run_events.side_effect = events + result = run(["investigate", "runs", "start", "--template", "p", "--detection-id", "d", "--follow"]) + assert result.exit_code == 0 + assert sdk.iter_run_events.call_args.args == ("r1",) + assert [json.loads(line)["seq"] for line in result.stdout.splitlines() if '"seq"' in line and line.startswith('{"seq"')] == [1, 2] + + def test_list_pages_with_a_cursor_envelope(self, sdk): + sdk.list_runs.return_value = {"runs": [{"run_id": "r1"}], "next_cursor": "c2"} + result = run(["investigate", "runs", "list", "--state", "running", "--limit", "10", "--cursor", "c1", "--since", "2026-11-01T00:00:00Z"]) + assert out_json(result) == {"runs": [{"run_id": "r1"}], "next_cursor": "c2"} + kwargs = sdk.list_runs.call_args.kwargs + assert kwargs["state"] == "running" and kwargs["limit"] == 10 and kwargs["cursor"] == "c1" + assert kwargs["since"] == "2026-11-01T00:00:00Z" and kwargs["verdict"] is None + + def test_list_table_rows_and_cursor_on_stderr(self, sdk): + sdk.list_runs.return_value = {"runs": [{ + "run_id": "r1", "state": "completed", "template": "p", "verdict": {"label": "false_positive", "p": 0.02}, + "started_by": {"identity": "user:u", "actor": "a@x.io"}, "created": "2026-11-10T00:00:00Z"}], "next_cursor": "c2"} + result = run(["investigate", "runs", "list"], fmt="csv") + assert "false_positive" in result.stdout and "a@x.io" in result.stdout and "p=" not in result.stdout + assert "next_cursor: c2" in result.stderr + + def test_list_all_follows_every_page(self, sdk): + sdk.iter_runs.return_value = iter([{"run_id": "a"}, {"run_id": "b"}]) + data = out_json(run(["investigate", "runs", "list", "--all", "--template", "p"])) + assert data == {"runs": [{"run_id": "a"}, {"run_id": "b"}], "next_cursor": ""} + assert sdk.iter_runs.call_args.kwargs["template"] == "p" + + def test_events_single_poll(self, sdk): + sdk.get_run_events.return_value = {"events": [{"seq": 4}], "next": 4, "done": False} + data = out_json(run(["investigate", "runs", "events", "r1", "--after", "3", "--wait", "5", "--limit", "20"])) + sdk.get_run_events.assert_called_once_with("r1", after=3, wait=5, limit=20) + assert data["next"] == 4 + + def test_follow_streams_one_json_event_per_line_and_ends_ok_when_done(self, sdk): + def events(run_id, **kw): + kw["state"].update(next=2, done=True, timed_out=False) + yield {"seq": 1, "at": "t", "type": "step", "data": {"a": 1}} + yield {"seq": 2, "at": "t", "type": "run_done", "data": {}} + sdk.iter_run_events.side_effect = events + result = run(["investigate", "runs", "events", "r1", "--follow", "--after", "0"]) + assert result.exit_code == 0 + assert [json.loads(line)["seq"] for line in result.stdout.splitlines()] == [1, 2] + assert sdk.iter_run_events.call_args.kwargs["wait"] == 10 + assert sdk.iter_run_events.call_args.kwargs["timeout"] == 900 + + def test_follow_timeout_exits_1_with_the_resume_point(self, sdk): + def events(run_id, **kw): + kw["state"].update(next=7, done=False, timed_out=True) + yield {"seq": 7, "at": "t", "type": "step", "data": {}} + sdk.iter_run_events.side_effect = events + result = run(["investigate", "runs", "events", "r1", "--follow", "--timeout", "5"]) + assert result.exit_code == 1 + assert "--after 7" in result.stderr + + def test_follow_text_lines_in_table_mode(self, sdk): + def events(run_id, **kw): + kw["state"].update(next=1, done=True) + yield {"seq": 1, "at": "2026-11-10T00:00:00Z", "type": "step", "data": {"fact": "a"}} + sdk.iter_run_events.side_effect = events + result = run(["investigate", "runs", "events", "r1", "--follow"], fmt="table") + assert result.stdout.strip() == '1\t2026-11-10T00:00:00Z\tstep\t{"fact":"a"}' + + def test_get_steps_case_cancel(self, sdk): + sdk.get_run.return_value = {"run_id": "r1"} + run(["investigate", "runs", "get", "r1"]) + sdk.get_run.assert_called_once_with("r1") + run(["investigate", "runs", "steps", "r1", "--values"]) + sdk.get_run_steps.assert_called_once_with("r1", values=True) + run(["investigate", "runs", "case", "r1"]) + sdk.get_run_case.assert_called_once_with("r1") + run(["investigate", "runs", "cancel", "r1", "--revert-leases"]) + sdk.cancel_run.assert_called_once_with("r1", revert_leases=True) + + +class TestWave: + def test_flags_are_merged_into_the_file(self, sdk, tmp_path): + f = tmp_path / "wave.yaml" + f.write_text( + "steps:\n - {id: host, use: host_context, args: {sid: abc}}\n" + "stop_when: ['host__resolve.found == false']\ndeadline_ms: 5000\ninputs: {a: '1'}\n") + sdk.run_wave.return_value = {"results": [], "compact": True} + result = run(["investigate", "wave", "--file", str(f), "--stop-when", "x == 1", "--deadline-ms", "9000", + "--input", "b=2", "--full", "--validate-only"]) + assert result.exit_code == 0, result.stderr + assert sdk.run_wave.call_args.args[0] == { + "steps": [{"id": "host", "use": "host_context", "args": {"sid": "abc"}}], + "stop_when": ["host__resolve.found == false", "x == 1"], "deadline_ms": 9000, + "inputs": {"a": "1", "b": "2"}, "full": True, "validate_only": True, + } + + def test_a_bare_list_is_the_steps(self, sdk, tmp_path): + f = tmp_path / "wave.json" + f.write_text('[{"id": "a", "op": "entity.get", "args": {}}]') + sdk.run_wave.return_value = {} + run(["investigate", "wave", "--file", str(f)]) + assert sdk.run_wave.call_args.args[0] == {"steps": [{"id": "a", "op": "entity.get", "args": {}}]} + + def test_a_refused_wave_prints_the_coded_errors_on_stdout_and_exits_4(self, sdk, tmp_path): + f = tmp_path / "wave.json" + f.write_text('{"steps": [{"id": "a", "op": "x"}]}') + err = issue_error() + sdk.run_wave.side_effect = err + result = run(["investigate", "wave", "--file", str(f)]) + assert result.exit_code == 4 + body = out_json(result) + assert body["valid"] is False and body["errors"] == err.issues and body["warnings"] == err.warnings + + def test_validate_only_with_an_invalid_report_exits_4(self, sdk, tmp_path): + f = tmp_path / "wave.json" + f.write_text('{"steps": [{"id": "a", "op": "x"}]}') + sdk.run_wave.return_value = {"valid": False, "errors": [{"code": "unknown_operator"}], "warnings": []} + assert run(["investigate", "wave", "--file", str(f), "--validate-only"]).exit_code == 4 + + def test_a_wave_failure_without_issues_is_a_plain_error(self, sdk, tmp_path): + f = tmp_path / "wave.json" + f.write_text('{"steps": [{"id": "a", "op": "x"}]}') + sdk.run_wave.side_effect = InvestigationError("timed out", status_code=504, exit_code=1) + result = run(["investigate", "wave", "--file", str(f)]) + assert result.exit_code == 1 + assert json.loads(result.stderr)["error"] == "timed out" + + +class TestActionsAndPolicy: + def test_actions_list_paging(self, sdk): + sdk.list_actions.side_effect = [ + {"actions": [{"id": "a1", "run_id": "r1", "class": "tag", "state": "awaiting_approval"}], "next_cursor": "c2"}, + {"actions": [{"id": "a2", "run_id": "r2", "class": "tag", "state": "proposed"}], "next_cursor": ""}, + ] + data = out_json(run(["investigate", "actions", "list", "--state", "awaiting_approval,proposed", "--class", "tag", "--all"])) + assert [a["id"] for a in data["actions"]] == ["a1", "a2"] and data["next_cursor"] == "" + first, second = sdk.list_actions.call_args_list + assert first.kwargs == {"state": "awaiting_approval,proposed", "action_class": "tag", "run_id": None, "limit": None, "cursor": None} + assert second.kwargs["cursor"] == "c2" + + def test_one_page_keeps_the_cursor(self, sdk): + sdk.list_actions.return_value = {"actions": [], "next_cursor": "c9"} + assert out_json(run(["investigate", "actions", "list"]))["next_cursor"] == "c9" + assert sdk.list_actions.call_count == 1 + + def test_approve_and_deny(self, sdk): + sdk.decide_action.return_value = {"state": "executing"} + run(["investigate", "actions", "approve", "r1", "a1"]) + sdk.decide_action.assert_called_with("r1", "a1", "approve") + run(["investigate", "actions", "deny", "r1", "a1", "--note", "known admin"]) + sdk.decide_action.assert_called_with("r1", "a1", "deny", note="known admin") + + def test_policy_commands(self, sdk): + sdk.get_effective_policy.return_value = {} + run(["investigate", "policy", "get", "--category", "c", "--priority", "4", "--target-tag", "a", "--target-tag", "b", "--class", "tag"]) + sdk.get_effective_policy.assert_called_with( + category="c", priority=4, severity=None, target_tags=["a", "b"], action_class="tag") + run(["investigate", "policy", "status"]) + sdk.get_autonomy.assert_called_once() + run(["investigate", "policy", "pause"]) + sdk.set_autonomy.assert_called_with("pause") + run(["investigate", "policy", "resume"]) + sdk.set_autonomy.assert_called_with("resume") + + +class TestErrorReporting: + def test_machine_formats_get_the_engine_error_as_a_document_on_stderr(self, sdk): + err = issue_error() + sdk.get_run.side_effect = err + result = run(["investigate", "runs", "get", "r1"]) + assert result.exit_code == err.exit_code + assert result.stdout == "" + body = json.loads(result.stderr) + assert body["error_code"] == "bad_request" and body["status"] == 400 + assert body["errors"] == err.issues and body["warnings"] == err.warnings + + def test_disabled_feature_error_names_the_cause(self, sdk): + from limacharlie.sdk import investigation as m + exc = m._wrap_error(error_from_status_code(404, {"error": "not found"})) + sdk.list_runs.side_effect = exc + result = run(["investigate", "runs", "list"]) + assert result.exit_code == 3 + assert "gated per deployment" in json.loads(result.stderr)["suggestion"] + + def test_table_mode_raises_the_error_for_the_normal_text_rendering(self, sdk): + sdk.get_run.side_effect = issue_error() + result = run(["investigate", "runs", "get", "r1"], fmt="table") + assert result.exit_code != 0 + assert isinstance(result.exception, InvestigationError) diff --git a/tests/unit/test_cli_lazy_loading_regression.py b/tests/unit/test_cli_lazy_loading_regression.py index d9718f44..f8f6ac5b 100644 --- a/tests/unit/test_cli_lazy_loading_regression.py +++ b/tests/unit/test_cli_lazy_loading_regression.py @@ -49,7 +49,7 @@ "audit", "auth", "billing", "case", "cloud-adapter", "cloudsec", "completion", "config", "detection", "download", "dr", "endpoint-policy", "event", "exfil", "extension", "external-adapter", "feedback", "fp", "group", "help", "hive", - "ingestion-key", "installation-key", "integrity", "ioc", "job", "logging", + "ingestion-key", "installation-key", "integrity", "investigate", "ioc", "job", "logging", "lookup", "mailsec", "note", "org", "output", "payload", "playbook", "replay", "schema", "search", "secret", "sensor", "sop", "spotcheck", "stream", "sync", "tag", "task", "user", "usp", "vulnerability", "yara", @@ -92,6 +92,7 @@ "ingestion_key": ("group", "ingestion-key"), "installation_key": ("group", "installation-key"), "integrity": ("group", "integrity"), + "investigate": ("group", "investigate"), "ioc": ("group", "ioc"), "job": ("group", "job"), "logging_cmd": ("group", "logging"), @@ -190,6 +191,7 @@ "ingestion-key": frozenset({"create", "delete", "list"}), "installation-key": frozenset({"create", "delete", "get", "list"}), "integrity": frozenset({"create", "delete", "get", "list"}), + "investigate": frozenset({"actions", "kit", "operators", "packs", "policy", "runs", "templates", "wave"}), "ioc": frozenset({"batch-enrich", "batch-search", "enrich", "hosts", "search"}), "job": frozenset({"delete", "get", "list", "wait"}), "logging": frozenset({"create", "delete", "get", "list"}), diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py new file mode 100644 index 00000000..3007a36a --- /dev/null +++ b/tests/unit/test_sdk_investigation.py @@ -0,0 +1,569 @@ +"""Tests for limacharlie.sdk.investigation (Investigation Engine).""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from limacharlie.errors import ( + ApiError, + AuthenticationError, + ValidationError, + error_from_status_code, +) +from limacharlie.sdk import investigation as inv_mod +from limacharlie.sdk.hive import HiveRecord +from limacharlie.sdk.investigation import ( + Investigation, + InvestigationError, + dump_template, + parse_template_source, +) + + +@pytest.fixture +def mock_org(): + org = MagicMock() + org.oid = "oid-1" + org.client = MagicMock() + return org + + +@pytest.fixture +def inv(mock_org): + return Investigation(mock_org) + + +@pytest.fixture +def hive(): + """The Hive the SDK writes templates to.""" + with patch.object(inv_mod, "Hive") as cls: + h = MagicMock() + cls.return_value = h + yield h + + +def _not_found(): + return error_from_status_code(400, {"error": "lc_error_code:RECORD_NOT_FOUND - no such record"}) + + +VALID = {"valid": True, "errors": [], "warnings": [{"code": "unused_fact", "path": "facts.x", "message": "m"}]} +INVALID = {"valid": False, "errors": [{"code": "unknown_field", "path": "facts.a", "message": "m", "suggestion": "b"}], "warnings": []} + +SRC = "description: test plan\nfacts:\n a:\n op: entity.get\n args: {entity_id: $entity}\n" + + +def _request_args(mock_org): + call = mock_org.client.request.call_args + return call.args[0], call.args[1], call.kwargs + + +# --------------------------------------------------------------------------- +# Routes and parameters +# --------------------------------------------------------------------------- + +class TestRoutes: + def test_kit_sections_are_comma_separated(self, inv, mock_org): + mock_org.client.request.return_value = {} + inv.get_authoring_kit(["guide", "library"]) + verb, path, kw = _request_args(mock_org) + assert (verb, path) == ("GET", "investigate/oid-1/authoring-kit") + assert kw["query_params"] == [("sections", "guide,library")] + + def test_kit_without_sections_sends_no_query(self, inv, mock_org): + mock_org.client.request.return_value = {} + inv.get_authoring_kit() + assert _request_args(mock_org)[2]["query_params"] is None + + def test_kit_rejects_unknown_section_before_calling(self, inv, mock_org): + with pytest.raises(ValidationError, match="nope"): + inv.get_authoring_kit(["guide", "nope"]) + mock_org.client.request.assert_not_called() + + def test_effective_policy_sends_class_not_class_underscore(self, inv, mock_org): + mock_org.client.request.return_value = {} + inv.get_effective_policy(category="c", priority=3, target_tags=["a", "b"], action_class="contain.host") + query = dict(_request_args(mock_org)[2]["query_params"]) + assert query == {"category": "c", "priority": "3", "target_tags": "a,b", "class": "contain.host"} + + def test_list_runs_sends_only_set_filters(self, inv, mock_org): + mock_org.client.request.return_value = {"runs": []} + inv.list_runs(state="running", limit=20, cursor="cur") + verb, path, kw = _request_args(mock_org) + assert (verb, path) == ("GET", "investigate/oid-1/runs") + assert dict(kw["query_params"]) == {"state": "running", "limit": "20", "cursor": "cur"} + + def test_list_runs_limit_bounds(self, inv): + with pytest.raises(ValidationError): + inv.list_runs(limit=201) + + def test_iter_runs_follows_cursors_and_rejects_a_stuck_one(self, inv, mock_org): + mock_org.client.request.side_effect = [ + {"runs": [{"run_id": "a"}], "next_cursor": "c1"}, + {"runs": [{"run_id": "b"}], "next_cursor": ""}, + ] + assert [r["run_id"] for r in inv.iter_runs(state="completed")] == ["a", "b"] + second = dict(mock_org.client.request.call_args.kwargs["query_params"]) + assert second == {"state": "completed", "cursor": "c1"} + + mock_org.client.request.side_effect = None + mock_org.client.request.return_value = {"runs": [], "next_cursor": "same"} + with pytest.raises(ApiError, match="did not advance"): + list(inv.iter_runs(cursor="same")) + + def test_run_id_is_escaped_and_validated(self, inv, mock_org): + mock_org.client.request.return_value = {} + inv.get_run("run:1.a-b") + assert _request_args(mock_org)[1] == "investigate/oid-1/runs/run%3A1.a-b" + for bad in ("..", ".", "a/b", "", "x" * 129): + with pytest.raises(ValidationError): + inv.get_run(bad) + + def test_events_bounds_and_params(self, inv, mock_org): + mock_org.client.request.return_value = {"events": [], "next": 4, "done": False} + inv.get_run_events("r1", after=4, wait=10, limit=500) + verb, path, kw = _request_args(mock_org) + assert path == "investigate/oid-1/runs/r1/events" + assert dict(kw["query_params"]) == {"after": "4", "wait": "10", "limit": "500"} + assert kw["timeout"] > 10 + for kwargs in ({"wait": 11}, {"wait": -1}, {"after": -1}, {"limit": 0}, {"limit": 501}): + with pytest.raises(ValidationError): + inv.get_run_events("r1", **kwargs) + + def test_follow_stops_at_done_and_resumes_from_next(self, inv, mock_org): + mock_org.client.request.side_effect = [ + {"events": [{"seq": 1}, {"seq": 2}], "next": 2, "done": False}, + {"events": [], "next": 2, "done": False}, + {"events": [{"seq": 3}], "next": 3, "done": True}, + ] + state = {} + seqs = [e["seq"] for e in inv.iter_run_events("r1", state=state)] + assert seqs == [1, 2, 3] + assert state == {"next": 3, "done": True, "timed_out": False} + afters = [dict(c.kwargs["query_params"])["after"] for c in mock_org.client.request.call_args_list] + assert afters == ["0", "2", "2"] + + def test_follow_reports_a_timeout_with_the_resume_point(self, inv, mock_org): + mock_org.client.request.return_value = {"events": [{"seq": 7}], "next": 7, "done": False} + state = {} + events = list(inv.iter_run_events("r1", after=5, timeout=0, state=state)) + assert [e["seq"] for e in events] == [7] + assert state["done"] is False and state["timed_out"] is True and state["next"] == 7 + + def test_writes_are_sent_once_and_as_json(self, inv, mock_org): + mock_org.client.request.return_value = {"run": {}, "joined": False} + inv.start_run({"trigger": {"detection_id": "d"}, "plan": {"auto": True}, "wait_seconds": 20}) + verb, path, kw = _request_args(mock_org) + assert (verb, path) == ("POST", "investigate/oid-1/runs") + assert json.loads(kw["raw_body"]) == {"trigger": {"detection_id": "d"}, "plan": {"auto": True}, "wait_seconds": 20} + assert kw["content_type"] == "application/json" + assert kw["retry_gateway_timeout"] is False # a run is never replayed after a timeout + assert kw["timeout"] >= 20 + + def test_reads_may_be_retried(self, inv, mock_org): + mock_org.client.request.return_value = {} + inv.list_operators() + assert "retry_gateway_timeout" not in _request_args(mock_org)[2] + + def test_decide_action_path_and_body(self, inv, mock_org): + mock_org.client.request.return_value = {} + inv.decide_action("r1", "a:1", "deny", note="legit") + verb, path, kw = _request_args(mock_org) + assert (verb, path) == ("POST", "investigate/oid-1/runs/r1/actions/a%3A1/deny") + assert json.loads(kw["raw_body"]) == {"note": "legit"} + with pytest.raises(ValidationError): + inv.decide_action("r1", "a1", "delete") + + def test_autonomy_switch(self, inv, mock_org): + mock_org.client.request.return_value = {} + inv.set_autonomy("pause") + assert _request_args(mock_org)[:2] == ("POST", "investigate/oid-1/autonomy/pause") + with pytest.raises(ValidationError): + inv.set_autonomy("stop") + + def test_list_actions_joins_states_and_uses_class(self, inv, mock_org): + mock_org.client.request.return_value = {"actions": []} + inv.list_actions(state=["awaiting_approval", "proposed"], action_class="tag", run_id="r1", limit=5) + query = dict(_request_args(mock_org)[2]["query_params"]) + assert query == {"state": "awaiting_approval,proposed", "class": "tag", "run_id": "r1", "limit": "5"} + + def test_wave_needs_steps_and_gets_a_long_timeout(self, inv, mock_org): + with pytest.raises(ValidationError): + inv.run_wave({"steps": []}) + mock_org.client.request.return_value = {"results": []} + inv.run_wave({"steps": [{"id": "a", "op": "x"}], "deadline_ms": 200000}) + assert _request_args(mock_org)[2]["timeout"] >= 200 + + +# --------------------------------------------------------------------------- +# Plan tests +# --------------------------------------------------------------------------- + +class TestPlanTest: + def test_needs_exactly_one_plan_and_samples(self, inv): + with pytest.raises(ValidationError): + inv.test_plan([{"detection_id": "d"}]) + with pytest.raises(ValidationError): + inv.test_plan([{"detection_id": "d"}], template="a", inline="b") + with pytest.raises(ValidationError): + inv.test_plan([], template="a") + + def test_request_shape(self, inv, mock_org): + mock_org.client.request.return_value = {"test_id": "t"} + inv.test_plan([{"detect_cat": "c", "last": 3}], inline=SRC, strict=True, mode="full", wait_seconds=2, + overrides={"budgets": {"max_wall_seconds": 60}}) + verb, path, kw = _request_args(mock_org) + assert (verb, path) == ("POST", "investigate/oid-1/test-plan") + assert json.loads(kw["raw_body"]) == { + "samples": [{"detect_cat": "c", "last": 3}], "wait_seconds": 2, "inline": SRC, "strict": True, + "mode": "full", "overrides": {"budgets": {"max_wall_seconds": 60}}, + } + + def test_wait_for_test_polls_with_results_seen_until_done(self, inv, mock_org): + mock_org.client.request.side_effect = [ + {"test_id": "t1", "state": "running", "samples_done": 2, "done": False}, + {"test_id": "t1", "state": "done", "samples_done": 3, "done": True}, + ] + out = inv.wait_for_test({"test_id": "t1", "state": "running", "samples_done": 0, "done": False}) + assert out["done"] is True + queries = [dict(c.kwargs["query_params"]) for c in mock_org.client.request.call_args_list] + assert [q["after"] for q in queries] == ["0", "2"] + assert mock_org.client.request.call_args.args[1] == "investigate/oid-1/test-plan/t1" + + def test_wait_for_test_gives_up_at_the_timeout(self, inv, mock_org): + running = {"test_id": "t1", "state": "running", "samples_done": 0, "done": False} + assert inv.wait_for_test(running, timeout=0) is running + mock_org.client.request.assert_not_called() + + def test_an_invalid_test_is_not_polled(self, inv, mock_org): + invalid = {"test_id": "t1", "state": "invalid", "done": False} + assert inv.wait_for_test(invalid) is invalid + mock_org.client.request.assert_not_called() + + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + +class TestErrors: + def test_typed_errors_keep_the_response_body(self): + for status, body in ((403, {"error": "x"}), (404, {"error": "y", "error_code": "not_found"}), (429, {"a": 1})): + assert error_from_status_code(status, body).response_body == body + + def test_coded_issues_survive_verbatim(self, inv, mock_org): + issues = [{"code": "unknown_field", "path": "steps[0].select[0]", "message": "no such field", "suggestion": "hosts_30d"}] + mock_org.client.request.side_effect = error_from_status_code( + 400, {"error": "the wave is invalid", "error_code": "bad_request", "errors": issues, "warnings": [{"code": "operator_denied"}]}, + ) + with pytest.raises(InvestigationError) as ei: + inv.run_wave({"steps": [{"id": "a", "op": "x"}]}) + exc = ei.value + assert exc.issues == issues + assert exc.warnings == [{"code": "operator_denied"}] + assert exc.error_code == "bad_request" and exc.status_code == 400 + assert "[unknown_field] steps[0].select[0]: no such field (did you mean hosts_30d)" in str(exc) + assert exc.to_dict()["errors"] == issues + + def test_unknown_path_404_says_the_feature_is_off(self, inv, mock_org): + mock_org.client.request.side_effect = error_from_status_code(404, {"error": "not found"}) + with pytest.raises(InvestigationError) as ei: + inv.list_runs() + assert ei.value.error_code is None + assert "gated per deployment" in ei.value.suggestion + assert ei.value.exit_code == 3 + + def test_engine_404_is_a_missing_run_not_a_disabled_feature(self, inv, mock_org): + mock_org.client.request.side_effect = error_from_status_code( + 404, {"error": "run not found", "error_code": "not_found"}) + with pytest.raises(InvestigationError) as ei: + inv.get_run("r1") + assert ei.value.error_code == "not_found" + assert not ei.value.suggestion + assert "run not found" in str(ei.value) + + def test_missing_permissions_are_named(self, inv, mock_org): + mock_org.client.request.side_effect = error_from_status_code(403, {"error": "Missing permissions"}) + with pytest.raises(InvestigationError) as ei: + inv.start_run({"trigger": {}, "plan": {}}) + assert "insight.evt.get" in ei.value.suggestion and "investigation.set" in ei.value.suggestion + + def test_engine_403_keeps_its_own_message(self, inv, mock_org): + mock_org.client.request.side_effect = error_from_status_code( + 403, {"error": "only the starter may add inputs", "error_code": "forbidden"}) + with pytest.raises(InvestigationError) as ei: + inv.decide_action("r", "a", "approve") + assert not ei.value.suggestion and ei.value.error_code == "forbidden" + + def test_no_engine_for_the_org_is_a_503(self, inv, mock_org): + mock_org.client.request.side_effect = error_from_status_code( + 503, {"error": "the investigation engine is not available for this organization"}) + with pytest.raises(InvestigationError) as ei: + inv.get_autonomy() + assert "not available for this organization" in str(ei.value) + assert "feature off" in ei.value.suggestion + + def test_run_cap(self, inv, mock_org): + mock_org.client.request.side_effect = error_from_status_code( + 429, {"error": "too many runs", "error_code": "too_many_runs"}) + with pytest.raises(InvestigationError) as ei: + inv.start_run({}) + assert "concurrent run cap" in ei.value.suggestion + + def test_authentication_errors_pass_through_untouched(self, inv, mock_org): + mock_org.client.request.side_effect = AuthenticationError("bad key", code=401) + with pytest.raises(AuthenticationError): + inv.list_runs() + + +# --------------------------------------------------------------------------- +# Template source +# --------------------------------------------------------------------------- + +class TestTemplateSource: + def test_yaml_1_1_surprises_stay_text(self): + data = parse_template_source("a: yes\nb: off\nc: 2026-11-10\nd: true\ne: [on]\nf: 0o17\n") + assert data == {"a": "yes", "b": "off", "c": "2026-11-10", "d": True, "e": ["on"], "f": "0o17"} + + def test_dump_round_trips_text_that_looks_like_something_else(self): + data = {"a": "yes", "c": "2026-11-10", "n": "0308"} + assert parse_template_source(dump_template(data)) == data + + def test_dump_keeps_key_order(self): + text = dump_template({"zebra": 1, "alpha": 2}) + assert text.index("zebra") < text.index("alpha") + + @pytest.mark.parametrize("src,match", [ + ("", "empty"), + ("- a\n- b\n", "mapping"), + ("a: [unclosed\n", "not valid YAML"), + ("a: &x 1\nb: *x\n", "anchors and aliases"), + ]) + def test_refused_sources(self, src, match): + with pytest.raises(ValidationError, match=match): + parse_template_source(src) + + +# --------------------------------------------------------------------------- +# save_template +# --------------------------------------------------------------------------- + +class TestSaveTemplate: + def _stored(self, status, etag="e2"): + return HiveRecord(name="my-plan", data={"description": "test plan", "status": status}, etag=etag) + + def test_saves_a_draft_by_default_creating_only(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = [_not_found(), self._stored("draft")] + out = inv.save_template("my-plan", SRC) + assert out["saved"] is True and out["status"] == "draft" + record = hive.set.call_args.args[0] + assert record.name == "my-plan" and record.data["status"] == "draft" + assert record.etag == inv_mod._CREATE_ONLY_ETAG + assert record.tags is None and record.comment is None # no metadata block: the hive default applies + assert out["warnings"] == VALID["warnings"] + + def test_validates_the_exact_text_that_is_stored(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = [_not_found(), self._stored("draft")] + inv.save_template("my-plan", SRC) + sent = json.loads(mock_org.client.request.call_args.kwargs["raw_body"])["yaml"] + assert parse_template_source(sent)["status"] == "draft" + + def test_a_template_cannot_activate_itself(self, inv, mock_org, hive): + out = inv.save_template("my-plan", SRC + "status: active\n") + assert out["saved"] is False and "cannot activate itself" in out["reason"] + mock_org.client.request.assert_not_called() + hive.set.assert_not_called() + + def test_explicit_active_is_written_active(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = [_not_found(), self._stored("active")] + out = inv.save_template("my-plan", SRC, status="active") + assert out["saved"] is True and out["status"] == "active" + assert hive.set.call_args.args[0].data["status"] == "active" + + def test_invalid_template_is_not_written_and_carries_the_report(self, inv, mock_org, hive): + mock_org.client.request.return_value = INVALID + out = inv.save_template("my-plan", SRC) + assert out["saved"] is False and out["validation"] == INVALID + hive.get.assert_not_called() + hive.set.assert_not_called() + + def test_existing_record_is_not_overwritten_silently(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.return_value = self._stored("draft") + out = inv.save_template("my-plan", SRC) + assert out["saved"] is False and "overwrite" in out["reason"] + hive.set.assert_not_called() + + def test_overwrite_replaces_a_draft_with_its_etag_and_keeps_its_metadata(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + existing = HiveRecord(name="my-plan", data={"status": "draft"}, etag="e1", tags=["mine"], comment="old", enabled=True) + hive.get.side_effect = [existing, self._stored("draft")] + out = inv.save_template("my-plan", SRC, overwrite=True, comment="9/10 agree", tags=["ai-draft"]) + assert out["saved"] is True + record = hive.set.call_args.args[0] + assert record.etag == "e1" + assert record.tags == ["mine", "ai-draft"] and record.comment == "9/10 agree" and record.enabled is True + + def test_overwrite_without_metadata_flags_sends_no_metadata(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + existing = HiveRecord(name="my-plan", data={"status": "draft"}, etag="e1", tags=["mine"], enabled=True) + hive.get.side_effect = [existing, self._stored("draft")] + inv.save_template("my-plan", SRC, overwrite=True) + record = hive.set.call_args.args[0] + assert record.tags is None and record.enabled is None + + def test_new_record_with_metadata_is_enabled(self, inv, mock_org, hive): + # Metadata sent without `enabled` would be stored as disabled. + mock_org.client.request.return_value = VALID + hive.get.side_effect = [_not_found(), self._stored("draft")] + inv.save_template("my-plan", SRC, comment="c", tags=["t"]) + assert hive.set.call_args.args[0].enabled is True + + def test_a_draft_never_replaces_an_active_template_even_with_overwrite(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.return_value = self._stored("active") + out = inv.save_template("my-plan", SRC, overwrite=True) + assert out["saved"] is False and "is active" in out["reason"] + hive.set.assert_not_called() + + def test_an_active_template_is_replaced_only_by_explicit_active_and_overwrite(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = [self._stored("active", "e1"), self._stored("active", "e2")] + assert inv.save_template("my-plan", SRC, status="active", overwrite=True)["saved"] is True + assert hive.set.call_args.args[0].etag == "e1" + hive.get.side_effect = [self._stored("active", "e1")] + assert inv.save_template("my-plan", SRC, status="active")["saved"] is False + + def test_a_record_with_an_unknown_status_is_left_alone(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.return_value = self._stored("retired") + out = inv.save_template("my-plan", SRC, overwrite=True) + assert out["saved"] is False and "retired" in out["reason"] + hive.set.assert_not_called() + + def test_a_hive_failure_other_than_not_found_stops_the_save(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = error_from_status_code(500, "boom") + with pytest.raises(ApiError): + inv.save_template("my-plan", SRC) + hive.set.assert_not_called() + + def test_stored_status_that_differs_is_reported_as_an_error(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = [_not_found(), self._stored("active")] + with pytest.raises(ApiError, match="status reads 'active', not 'draft'"): + inv.save_template("my-plan", SRC) + + @pytest.mark.parametrize("name", ["", "Upper", "-lead", "a b", "x" * 65, "a/b"]) + def test_bad_names_are_refused_before_any_call(self, inv, mock_org, hive, name): + with pytest.raises(ValidationError): + inv.save_template(name, SRC) + mock_org.client.request.assert_not_called() + + def test_name_key_in_the_source_is_refused(self, inv): + with pytest.raises(ValidationError, match="name"): + inv.save_template("my-plan", "name: other\n" + SRC) + + def test_bad_status_in_source_or_argument(self, inv): + with pytest.raises(ValidationError): + inv.save_template("my-plan", SRC + "status: retired\n") + with pytest.raises(ValidationError): + inv.save_template("my-plan", SRC, status="retired") + + +class TestPromote: + def test_promotes_with_the_etag_that_was_read(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = [ + HiveRecord(name="p", data={"description": "d", "status": "draft"}, etag="e1"), + HiveRecord(name="p", data={"description": "d", "status": "active"}, etag="e2"), + ] + out = inv.promote_template("p") + assert out["promoted"] is True and out["status"] == "active" + record = hive.set.call_args.args[0] + assert record.etag == "e1" and record.data["status"] == "active" and record.tags is None + sent = parse_template_source(json.loads(mock_org.client.request.call_args.kwargs["raw_body"])["yaml"]) + assert sent["status"] == "active" # what is validated is what is stored + + def test_an_invalid_draft_stays_a_draft(self, inv, mock_org, hive): + mock_org.client.request.return_value = INVALID + hive.get.return_value = HiveRecord(name="p", data={"status": "draft"}, etag="e1") + out = inv.promote_template("p") + assert out["promoted"] is False and out["validation"] == INVALID + hive.set.assert_not_called() + + def test_already_active_is_a_no_op(self, inv, mock_org, hive): + hive.get.return_value = HiveRecord(name="p", data={"status": "active"}, etag="e1") + assert inv.promote_template("p")["promoted"] is False + hive.set.assert_not_called() + + +class TestInstallDefaults: + LIB = [ + {"name": "alpha", "version": 2, "status": "active", "yaml": "description: a\nstatus: active\nfacts: {}\n"}, + {"name": "beta", "version": 1, "status": "active", "yaml": "description: b\nfacts: {}\n"}, + ] + + def _kit(self, mock_org, library): + def request(verb, path, **kw): + if path.endswith("authoring-kit"): + assert dict(kw["query_params"]) == {"sections": "library"} + return {"library": library} + return VALID + mock_org.client.request.side_effect = request + + def test_writes_every_entry_as_a_draft_even_when_the_library_says_active(self, inv, mock_org, hive): + self._kit(mock_org, self.LIB) + drafts = [HiveRecord(name=n, data={"status": "draft"}, etag="e") for n in ("alpha", "beta")] + hive.get.side_effect = [_not_found(), drafts[0], _not_found(), drafts[1]] + out = inv.install_default_templates() + assert [w["name"] for w in out["written"]] == ["alpha", "beta"] + assert {w["status"] for w in out["written"]} == {"draft"} + assert [c.args[0].data["status"] for c in hive.set.call_args_list] == ["draft", "draft"] + assert out["skipped"] == [] and out["failed"] == [] + + def test_existing_records_are_skipped_with_versions_not_overwritten(self, inv, mock_org, hive): + self._kit(mock_org, self.LIB) + hive.get.return_value = HiveRecord(name="x", data={"status": "active", "version": 1}, etag="e") + out = inv.install_default_templates(["alpha"]) + hive.set.assert_not_called() + assert out["written"] == [] + assert out["skipped"] == [{ + "name": "alpha", "reason": out["skipped"][0]["reason"], "library_version": 2, + "installed_version": 1, "installed_status": "active", "update_available": True, + }] + assert "already installed" in out["skipped"][0]["reason"] + + def test_only_the_named_entries_are_installed(self, inv, mock_org, hive): + self._kit(mock_org, self.LIB) + hive.get.side_effect = [_not_found(), HiveRecord(name="beta", data={"status": "draft"}, etag="e")] + out = inv.install_default_templates(["beta"]) + assert [w["name"] for w in out["written"]] == ["beta"] + + def test_unknown_name_is_refused_with_the_available_ones(self, inv, mock_org, hive): + self._kit(mock_org, self.LIB) + with pytest.raises(ValidationError, match="gamma.*alpha, beta"): + inv.install_default_templates(["gamma"]) + hive.set.assert_not_called() + + def test_an_engine_without_a_library_says_so(self, inv, mock_org, hive): + mock_org.client.request.return_value = {"kit_version": 2, "guide": "g"} + with pytest.raises(ValidationError, match="no default template library"): + inv.install_default_templates() + + def test_an_entry_that_does_not_validate_is_reported_failed(self, inv, mock_org, hive): + def request(verb, path, **kw): + return {"library": self.LIB} if path.endswith("authoring-kit") else INVALID + mock_org.client.request.side_effect = request + out = inv.install_default_templates(["alpha"]) + assert out["written"] == [] and out["failed"][0]["name"] == "alpha" + assert out["failed"][0]["validation"] == INVALID + hive.set.assert_not_called() + + def test_active_status_is_written_only_when_asked(self, inv, mock_org, hive): + self._kit(mock_org, self.LIB) + hive.get.side_effect = [_not_found(), HiveRecord(name="alpha", data={"status": "active"}, etag="e")] + out = inv.install_default_templates(["alpha"], status="active") + assert out["written"][0]["status"] == "active" From cad3f10e26b11e2c86d7800b9a83338483a318a8 Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 13:21:12 -0800 Subject: [PATCH 03/14] Investigation Engine CLI: review round 1 YAML 1.2 number handling for template sources, iter_actions cursor guard, JSON-lines stream for runs start --follow, landed-write guard for template saves, hive permission hint for template records. Co-Authored-By: Claude Sonnet 5.5 --- limacharlie/commands/investigate.py | 25 ++++--- limacharlie/sdk/investigation.py | 99 ++++++++++++++++++++++++++-- tests/unit/test_cli_investigate.py | 27 +++++--- tests/unit/test_sdk_investigation.py | 52 ++++++++++++++- 4 files changed, 176 insertions(+), 27 deletions(-) diff --git a/limacharlie/commands/investigate.py b/limacharlie/commands/investigate.py index bc92083a..24c1235b 100644 --- a/limacharlie/commands/investigate.py +++ b/limacharlie/commands/investigate.py @@ -234,7 +234,9 @@ --idempotency-key makes a repeated start return the same run. With --wait N the call blocks up to N seconds for the verdict; --follow then -streams the run's events until it ends. +streams the run's events until it ends. With --follow in a structured output +format stdout is JSON lines: the first line is the start answer ({"run": ..., +"joined": ...}), the following lines are events ({"seq": ..., "type": ...}). Examples: limacharlie investigate runs start --template my-plan --detection-id @@ -873,7 +875,12 @@ def runs_start(ctx, template, file_path, auto, detection_id, case_id, entity_id, request["wait_seconds"] = wait_seconds inv = _get_investigation(ctx) answer = inv.start_run(request) - command_output(ctx, answer) + if follow and _fmt(ctx) in _STRUCTURED: + # One compact line, like the events that follow: the stream is JSON lines. + if not ctx.obj.quiet: + click.echo(json.dumps(answer, separators=(",", ":"), default=str)) + else: + command_output(ctx, answer) if follow: run_id = (answer.get("run") or {}).get("run_id") if run_id: @@ -1061,14 +1068,12 @@ def actions_group() -> None: def actions_list(ctx, state, action_class, run_id, limit, cursor, fetch_all) -> None: """List the org's actions across runs; what needs a person comes first.""" inv = _get_investigation(ctx) - items: list[Any] = [] - while True: - page = inv.list_actions(state=state, action_class=action_class, run_id=run_id, limit=limit, cursor=cursor) - items += page.get("actions") or [] - cursor = page.get("next_cursor") or "" - if not (fetch_all and cursor): - break - _emit_page(ctx, "actions", items, cursor, _action_row) + filters = dict(state=state, action_class=action_class, run_id=run_id, limit=limit, cursor=cursor) + if fetch_all: + _emit_page(ctx, "actions", list(inv.iter_actions(**filters)), "", _action_row) + return + page = inv.list_actions(**filters) + _emit_page(ctx, "actions", page.get("actions") or [], page.get("next_cursor") or "", _action_row) @actions_group.command("approve") diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index 59920ed5..6483653d 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -33,6 +33,7 @@ from __future__ import annotations +import functools import json import re import time @@ -42,7 +43,7 @@ import yaml from ..errors import ApiError, LimaCharlieError, ValidationError -from .hive import Hive, HiveRecord, is_record_not_found +from .hive import Hive, HiveRecord, is_permission_denied, is_record_not_found if TYPE_CHECKING: from .organization import Organization @@ -68,8 +69,10 @@ # refused there. "." and ".." are refused as path traversal. _ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") -# Sent as the etag of a write that must create a record. It matches no stored -# etag, so the hive refuses the write if the record exists by then. +# Sent as the etag of a write that must create a record. The hive ignores the +# etag when no record exists and refuses the write when one does (it matches no +# stored etag), so a record created between our check and the write is never +# overwritten. _CREATE_ONLY_ETAG = "create-only: the record must not exist" # The engine's long polls (run events, test results) wait at most this long. @@ -199,6 +202,30 @@ def _wrap_error(exc: LimaCharlieError) -> LimaCharlieError: ) +_TEMPLATE_PERMISSION_HINT = ( + "Template records are the hive 'investigation_plan': reading needs investigation_plan.get, writing " + "investigation_plan.set, deleting investigation_plan.del. They are explicit grants that no default role has." +) + + +def _template_record_errors(fn: Any) -> Any: + """Say which permission a refused template-record operation needs.""" + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + try: + return fn(*args, **kwargs) + except InvestigationError: + raise + except LimaCharlieError as exc: + if is_permission_denied(exc): + raise InvestigationError( + exc.raw_message, status_code=403, response_body=getattr(exc, "response_body", None), + suggestion=_TEMPLATE_PERMISSION_HINT, exit_code=exc.exit_code, + ) from exc + raise + return wrapper + + def _check_id(kind: str, value: str) -> str: if not isinstance(value, str) or value in (".", "..") or not _ID_RE.match(value): raise ValidationError(f"invalid {kind} {value!r}: use letters, digits and . _ : - (at most 128 characters)") @@ -218,19 +245,43 @@ class _TemplateLoader(yaml.SafeLoader): would turn ``yes``/``no``/``on``/``off`` into booleans and an unquoted date into a ``datetime``, so a template would be stored as something other than what its author wrote. This loader keeps both as text and accepts - only ``true``/``false`` as booleans. + only ``true``/``false`` as booleans. Numbers follow the YAML 1.2 core + schema; one written the 1.1 way (``012``, ``1_000``, ``10:30``) stays text. """ _BOOL_TAG = "tag:yaml.org,2002:bool" +_INT_TAG = "tag:yaml.org,2002:int" +_FLOAT_TAG = "tag:yaml.org,2002:float" _TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" _TemplateLoader.yaml_implicit_resolvers = { - ch: [(tag, rx) for tag, rx in resolvers if tag not in (_BOOL_TAG, _TIMESTAMP_TAG)] + ch: [(tag, rx) for tag, rx in resolvers if tag not in (_BOOL_TAG, _INT_TAG, _FLOAT_TAG, _TIMESTAMP_TAG)] for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() } +# YAML 1.2 core schema. A number written with a leading zero, an underscore or +# a colon (`012`, `1_000`, `10:30`) is text here, as the author wrote it, +# rather than the number YAML 1.1 reads it as. _TemplateLoader.add_implicit_resolver( _BOOL_TAG, re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$"), list("tTfF"), ) +_TemplateLoader.add_implicit_resolver( + _INT_TAG, re.compile(r"^(?:[-+]?(?:0|[1-9][0-9]*)|0o[0-7]+|0x[0-9a-fA-F]+)$"), list("-+0123456789"), +) +_TemplateLoader.add_implicit_resolver( + _FLOAT_TAG, + re.compile(r"^(?:[-+]?(?:[0-9]+\.[0-9]*|\.[0-9]+|[0-9]+(?=[eE]))(?:[eE][-+]?[0-9]+)?|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$"), + list("-+0123456789."), +) + + +def _construct_core_int(loader: yaml.SafeLoader, node: yaml.Node) -> int: + value = str(loader.construct_scalar(node)) + if value[:2] == "0o": + return int(value[2:], 8) + return int(value, 0) if value[:2] == "0x" else int(value) + + +_TemplateLoader.add_constructor(_INT_TAG, _construct_core_int) def parse_template_source(source: str) -> dict[str, Any]: @@ -815,6 +866,28 @@ def list_actions( state = ",".join(state) return self._get("actions", self._q(state=state, class_=action_class, run_id=run_id, limit=limit, cursor=cursor)) + def iter_actions(self, **filters: Any) -> Iterator[dict[str, Any]]: + """Walk every action matching the filters, following cursors. + + Args: + **filters: The keyword arguments of :meth:`list_actions`, ``cursor`` included. + + Yields: + dict: One action at a time. + + Raises: + ApiError: If the engine returns a cursor that does not advance. + """ + while True: + page = self.list_actions(**filters) + yield from page.get("actions") or [] + nxt = page.get("next_cursor") or "" + if not nxt: + return + if nxt == filters.get("cursor"): + raise ApiError("Action listing did not advance: the engine returned the same cursor.") + filters["cursor"] = nxt + def decide_action( self, run_id: str, action_id: str, verb: str, note: str | None = None, seconds: int | None = None, ) -> dict[str, Any]: @@ -852,6 +925,7 @@ def decide_action( def _plans(self) -> Hive: return Hive(self._org, PLAN_HIVE) + @_template_record_errors def get_template(self, name: str) -> dict[str, Any]: """Read a stored template. @@ -885,6 +959,7 @@ def _template_view(record: HiveRecord) -> dict[str, Any]: "last_modified": record.last_modified, } + @_template_record_errors def save_template( self, name: str, @@ -997,7 +1072,17 @@ def save_template( record.tags = list(dict.fromkeys((record.tags or []) + tags)) if comment: record.comment = comment - hive.set(record) + try: + hive.set(record) + except ApiError: + # A write the hive applied whose answer was lost, then replayed with + # its now stale etag, fails although the record is saved. + try: + landed = (hive.get(name).data or {}) == data + except LimaCharlieError: + landed = False + if not landed: + raise # Say what is stored, not what was sent. stored_record = hive.get(name) @@ -1021,6 +1106,7 @@ def save_template( ), } + @_template_record_errors def promote_template(self, name: str) -> dict[str, Any]: """Make a stored draft template active. @@ -1056,6 +1142,7 @@ def promote_template(self, name: str) -> dict[str, Any]: "warnings": report.get("warnings") or [], } + @_template_record_errors def delete_template(self, name: str) -> dict[str, Any]: """Delete a stored template record. diff --git a/tests/unit/test_cli_investigate.py b/tests/unit/test_cli_investigate.py index a714a2c6..bf55c343 100644 --- a/tests/unit/test_cli_investigate.py +++ b/tests/unit/test_cli_investigate.py @@ -317,7 +317,19 @@ def events(run_id, **kw): result = run(["investigate", "runs", "start", "--template", "p", "--detection-id", "d", "--follow"]) assert result.exit_code == 0 assert sdk.iter_run_events.call_args.args == ("r1",) - assert [json.loads(line)["seq"] for line in result.stdout.splitlines() if '"seq"' in line and line.startswith('{"seq"')] == [1, 2] + assert [json.loads(line).get("seq") for line in result.stdout.splitlines()] == [None, 1, 2] + + def test_follow_in_a_structured_format_is_json_lines_from_the_first_line(self, sdk): + sdk.start_run.return_value = {"run": {"run_id": "r1"}, "joined": False} + + def events(run_id, **kw): + kw["state"].update(next=1, done=True) + yield {"seq": 1, "type": "step", "data": {}} + sdk.iter_run_events.side_effect = events + result = run(["investigate", "runs", "start", "--template", "p", "--detection-id", "d", "--follow"]) + lines = result.stdout.splitlines() + assert [json.loads(line) for line in lines] == [ + {"run": {"run_id": "r1"}, "joined": False}, {"seq": 1, "type": "step", "data": {}}] def test_list_pages_with_a_cursor_envelope(self, sdk): sdk.list_runs.return_value = {"runs": [{"run_id": "r1"}], "next_cursor": "c2"} @@ -437,16 +449,13 @@ def test_a_wave_failure_without_issues_is_a_plain_error(self, sdk, tmp_path): class TestActionsAndPolicy: - def test_actions_list_paging(self, sdk): - sdk.list_actions.side_effect = [ - {"actions": [{"id": "a1", "run_id": "r1", "class": "tag", "state": "awaiting_approval"}], "next_cursor": "c2"}, - {"actions": [{"id": "a2", "run_id": "r2", "class": "tag", "state": "proposed"}], "next_cursor": ""}, - ] + def test_actions_list_all_walks_every_page(self, sdk): + sdk.iter_actions.return_value = iter([{"id": "a1", "run_id": "r1"}, {"id": "a2", "run_id": "r2"}]) data = out_json(run(["investigate", "actions", "list", "--state", "awaiting_approval,proposed", "--class", "tag", "--all"])) assert [a["id"] for a in data["actions"]] == ["a1", "a2"] and data["next_cursor"] == "" - first, second = sdk.list_actions.call_args_list - assert first.kwargs == {"state": "awaiting_approval,proposed", "action_class": "tag", "run_id": None, "limit": None, "cursor": None} - assert second.kwargs["cursor"] == "c2" + assert sdk.iter_actions.call_args.kwargs == { + "state": "awaiting_approval,proposed", "action_class": "tag", "run_id": None, "limit": None, "cursor": None} + sdk.list_actions.assert_not_called() def test_one_page_keeps_the_cursor(self, sdk): sdk.list_actions.return_value = {"actions": [], "next_cursor": "c9"} diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index 3007a36a..88bfd25e 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -111,6 +111,18 @@ def test_iter_runs_follows_cursors_and_rejects_a_stuck_one(self, inv, mock_org): with pytest.raises(ApiError, match="did not advance"): list(inv.iter_runs(cursor="same")) + def test_iter_actions_follows_cursors_and_rejects_a_stuck_one(self, inv, mock_org): + mock_org.client.request.side_effect = [ + {"actions": [{"id": "a"}], "next_cursor": "c1"}, + {"actions": [{"id": "b"}], "next_cursor": ""}, + ] + assert [a["id"] for a in inv.iter_actions(state="awaiting_approval")] == ["a", "b"] + assert dict(mock_org.client.request.call_args.kwargs["query_params"]) == {"state": "awaiting_approval", "cursor": "c1"} + mock_org.client.request.side_effect = None + mock_org.client.request.return_value = {"actions": [], "next_cursor": "same"} + with pytest.raises(ApiError, match="did not advance"): + list(inv.iter_actions(cursor="same")) + def test_run_id_is_escaped_and_validated(self, inv, mock_org): mock_org.client.request.return_value = {} inv.get_run("run:1.a-b") @@ -321,8 +333,14 @@ def test_authentication_errors_pass_through_untouched(self, inv, mock_org): class TestTemplateSource: def test_yaml_1_1_surprises_stay_text(self): - data = parse_template_source("a: yes\nb: off\nc: 2026-11-10\nd: true\ne: [on]\nf: 0o17\n") - assert data == {"a": "yes", "b": "off", "c": "2026-11-10", "d": True, "e": ["on"], "f": "0o17"} + data = parse_template_source("a: yes\nb: off\nc: 2026-11-10\nd: true\ne: [on]\nf: 10:30\ng: 012\nh: 1_000\n") + assert data == {"a": "yes", "b": "off", "c": "2026-11-10", "d": True, "e": ["on"], + "f": "10:30", "g": "012", "h": "1_000"} + + def test_yaml_1_2_numbers_are_numbers(self): + data = parse_template_source("a: 12\nb: -3\nc: 1.5\nd: 1e3\ne: 0x1f\nf: 0o17\ng: 0\nh: .5\ni: null\n") + assert data == {"a": 12, "b": -3, "c": 1.5, "d": 1000.0, "e": 31, "f": 15, "g": 0, "h": 0.5, "i": None} + assert type(data["a"]) is int def test_dump_round_trips_text_that_looks_like_something_else(self): data = {"a": "yes", "c": "2026-11-10", "n": "0308"} @@ -473,6 +491,36 @@ def test_bad_status_in_source_or_argument(self, inv): inv.save_template("my-plan", SRC, status="retired") + def test_a_write_that_landed_but_failed_on_replay_is_not_reported_as_an_error(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + data = {"description": "test plan", "facts": {"a": {"op": "entity.get", "args": {"entity_id": "$entity"}}}, "status": "draft"} + landed = HiveRecord(name="my-plan", data=data, etag="e9") + hive.get.side_effect = [_not_found(), landed, landed] + hive.set.side_effect = error_from_status_code(400, {"error": "lc_error_code:ETAG_MISMATCH"}) + assert inv.save_template("my-plan", SRC)["saved"] is True + + def test_a_write_that_failed_and_did_not_land_is_an_error(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = [_not_found(), _not_found()] + hive.set.side_effect = error_from_status_code(400, {"error": "lc_error_code:ETAG_MISMATCH"}) + with pytest.raises(ApiError): + inv.save_template("my-plan", SRC) + + +class TestTemplateRecordPermissions: + def test_a_refused_hive_operation_names_the_investigation_plan_permissions(self, inv, hive): + hive.get.side_effect = error_from_status_code(400, {"error": "lc_error_code:UNAUTHORIZED - nope"}) + with pytest.raises(InvestigationError) as ei: + inv.get_template("x") + assert "investigation_plan.get" in ei.value.suggestion and ei.value.status_code == 403 + + def test_other_hive_failures_are_not_rewritten(self, inv, hive): + hive.delete.side_effect = error_from_status_code(500, "boom") + with pytest.raises(ApiError) as ei: + inv.delete_template("x") + assert not isinstance(ei.value, InvestigationError) + + class TestPromote: def test_promotes_with_the_etag_that_was_read(self, inv, mock_org, hive): mock_org.client.request.return_value = VALID From 8fc24014ab1a2347e2997b76137d8927b49fa6b8 Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 13:23:55 -0800 Subject: [PATCH 04/14] Investigation Engine CLI: landed-write guard also compares comment and tags Co-Authored-By: Claude Sonnet 5.5 --- limacharlie/sdk/investigation.py | 7 ++++++- tests/unit/test_sdk_investigation.py | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index 6483653d..b2f3c0e4 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -1078,7 +1078,12 @@ def save_template( # A write the hive applied whose answer was lost, then replayed with # its now stale etag, fails although the record is saved. try: - landed = (hive.get(name).data or {}) == data + now = hive.get(name) + landed = ( + (now.data or {}) == data + and (not comment or now.comment == record.comment) + and (not tags or set(tags) <= set(now.tags or [])) + ) except LimaCharlieError: landed = False if not landed: diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index 88bfd25e..af8f1443 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -499,6 +499,15 @@ def test_a_write_that_landed_but_failed_on_replay_is_not_reported_as_an_error(se hive.set.side_effect = error_from_status_code(400, {"error": "lc_error_code:ETAG_MISMATCH"}) assert inv.save_template("my-plan", SRC)["saved"] is True + def test_a_failed_overwrite_does_not_pass_as_saved_when_the_comment_was_lost(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + data = {"description": "test plan", "facts": {"a": {"op": "entity.get", "args": {"entity_id": "$entity"}}}, "status": "draft"} + same = HiveRecord(name="my-plan", data=data, etag="e1", comment="old", tags=[], enabled=True) + hive.get.side_effect = [same, same] + hive.set.side_effect = error_from_status_code(500, "boom") + with pytest.raises(ApiError): + inv.save_template("my-plan", SRC, overwrite=True, comment="new: 9/10 agree") + def test_a_write_that_failed_and_did_not_land_is_an_error(self, inv, mock_org, hive): mock_org.client.request.return_value = VALID hive.get.side_effect = [_not_found(), _not_found()] From 4c42051c6fae0fa1947dc45b06307e453802dc0b Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 14:41:22 -0800 Subject: [PATCH 05/14] Investigation Engine CLI: YAML 1.2 input files, structured errors for every failure, run mode docs, example check - wave and --samples-file are read with YAML 1.2 scalar rules (off/yes stay text, dates stay strings) - unreadable or malformed input files and every Hive/API failure render as structured errors - help and docs: the run mode default is full; triage is cheaper - docs wave example selects the real ioc.prevalence field (seen); a test checks every documented wave against a recorded operator catalogue Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 4 + doc/cli/investigation-engine.md | 8 +- limacharlie/commands/investigate.py | 77 +- limacharlie/sdk/investigation.py | 76 +- tests/unit/fixtures/ie_catalogue.json | 1116 +++++++++++++++++++++ tests/unit/test_cli_investigate.py | 73 +- tests/unit/test_investigation_examples.py | 99 ++ tests/unit/test_sdk_investigation.py | 47 +- 8 files changed, 1464 insertions(+), 36 deletions(-) create mode 100644 tests/unit/fixtures/ie_catalogue.json create mode 100644 tests/unit/test_investigation_examples.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c29b3e70..30998650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ - Engine failures keep the engine's coded issues (`code`, `path`, `message`, `suggestion`) and error code verbatim, and say when the feature is not enabled for the deployment or the permissions are missing. +- `wave` and `templates test --samples-file` read YAML with 1.2 scalar rules (`off`/`yes` + stay text, an unquoted date stays a string) so values reach the engine as written; + unreadable or malformed input files and every Hive/API failure are reported as + structured JSON (with the backend's `error_code`) in a structured output format. - `templates install-defaults` reads the authoring kit's `library` section; it needs an engine version that serves it. - The typed 401/403/404/422/429 errors raised for an HTTP error answer now carry diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md index d2e658ff..d893e4e1 100644 --- a/doc/cli/investigation-engine.md +++ b/doc/cli/investigation-engine.md @@ -129,6 +129,8 @@ limacharlie investigate runs cancel RUN_ID [--revert-leases] `--auto` picks the active template whose `applies_to` matches the alert. A start is sent once and never retried automatically; `--idempotency-key` makes a repeated start return the same run (`joined: true`). +`--mode` is `full` by default (triage, scoping and containment); `triage` stops at the verdict and costs less, `scope` is for scoping an already-triaged alert. (`templates test` defaults to `triage`.) + `runs list` pages: structured formats print `{"runs": [...], "next_cursor": "..."}` and table/CSV/JSONL print rows with the cursor on stderr. Continue with `--cursor`, or use `--all`. `runs events --follow` polls until the run is over and writes **one JSON event per line** (a text line in table mode), flushing as they arrive. It exits `0` when the run finished and `1` when `--timeout` (default 900 s) elapsed first; the resume point is printed on stderr (`--after N`). Events carry structure only, never customer values. @@ -141,7 +143,7 @@ A wave is a batch of questions: the engine runs every ready step in parallel wit # wave.yaml steps: - {id: host, use: host_context, args: {sid: ""}} - - {id: prev, op: ioc.prevalence, args: {file_hash: ""}, select: [hosts_30d, first_seen]} + - {id: prev, op: ioc.prevalence, args: {file_hash: ""}, select: [hosts_30d, seen]} - {id: kids, op: event.children, args: {sid: "", atom: ""}, limit: 5} stop_when: - "prev.hosts_30d >= 50 and kids.count == 0" @@ -187,6 +189,10 @@ In a structured output format (`--output json`, the default when piped) a failur `error_code` is the engine's (`bad_request`, `forbidden`, `not_found`, `conflict`, `too_many_runs`, `not_implemented`, `unavailable`). It is absent when the answer did not come from the engine: a 404 without it means the feature is not enabled, and `suggestion` says what to check. Exit codes follow the CLI's: `2` authentication or permission, `3` not found, `4` validation or a refused result, `5` rate limit. +Every other failure is written the same way: a Hive or API error carries its `message`, HTTP `status` and the backend's code as `error_code` (the `lc_error_code` of a Hive answer, for example a refused promote), and a refused input file (unreadable, not YAML/JSON, wrong shape) carries `errors: [{code: invalid_input, path: --file, message}]` and exits `4`. + +Wave files, sample files and templates are read with YAML 1.2 scalar rules, so `off`, `yes`, `no` and an unquoted date such as `2026-11-10` are sent as the text you wrote, not as a boolean or a date. + ## SDK ```python diff --git a/limacharlie/commands/investigate.py b/limacharlie/commands/investigate.py index 24c1235b..bf551756 100644 --- a/limacharlie/commands/investigate.py +++ b/limacharlie/commands/investigate.py @@ -24,14 +24,15 @@ from ..client import Client from ..discovery import register_explain from ..output import detect_output_format, format_output +from ..errors import LimaCharlieError from ..sdk.investigation import ( KIT_SECTIONS, MAX_POLL_WAIT, Investigation, InvestigationError, + parse_document, ) from ..sdk.organization import Organization -from ._input_helpers import load_file from ._output_helpers import command_output, note # Exit code of a result that is well-formed but says "no" (invalid template, @@ -412,27 +413,67 @@ def _fmt(ctx: click.Context) -> str: return ctx.obj.output_format or detect_output_format() +def _error_document(exc: LimaCharlieError) -> dict[str, Any]: + """The machine-readable form of any SDK failure. + + An engine failure keeps the engine's own answer. Any other failure (a Hive + or API error, bad input) carries its message, HTTP status, the backend's + ``error_code`` (the ``lc_error_code`` of a Hive answer) and suggestion. + """ + if isinstance(exc, InvestigationError): + return exc.to_dict() + out: dict[str, Any] = {"error": exc.raw_message, "status": exc.status_code} + code = getattr(exc, "error_code", None) + if code: + out["error_code"] = code + if exc.suggestion: + out["suggestion"] = exc.suggestion + return out + + def _engine_errors(fn: Callable[..., Any]) -> Callable[..., Any]: - """Report an engine failure as a document when the output is machine-read. + """Report a failure as a document when the output is machine-read. - In a structured output format the failure is written to stderr as the - engine's own answer (message, ``error_code``, coded ``errors`` and - ``warnings``) and the command exits with the error's exit code. In table - mode the error propagates and prints as text, issues included. + In a structured output format every SDK failure (the engine's, a Hive or + API error, a refused input) is written to stderr as a document (message, + ``error_code``, coded ``errors`` and ``warnings`` when the engine sent + them) and the command exits with the error's exit code. In table mode the + error propagates and prints as text, issues included. """ @functools.wraps(fn) def wrapper(*args: Any, **kwargs: Any) -> Any: try: return fn(*args, **kwargs) - except InvestigationError as exc: + except LimaCharlieError as exc: ctx = click.get_current_context() if _fmt(ctx) not in _STRUCTURED: raise - click.echo(format_output(exc.to_dict(), _fmt(ctx)), err=True) + click.echo(format_output(_error_document(exc), _fmt(ctx)), err=True) ctx.exit(exc.exit_code or 1) return wrapper +def _input_error(hint: str, message: str) -> InvestigationError: + """A refused input, as a coded error (``invalid_input``) that renders as JSON when asked.""" + return InvestigationError( + f"{hint}: {message}", status_code=None, exit_code=EXIT_REFUSED, + issues=[{"code": "invalid_input", "path": hint, "message": message}], + ) + + +def _load_document(path: str, hint: str) -> Any: + """Read a YAML or JSON file; scalars keep the meaning they have in YAML 1.2 (``off`` is text).""" + try: + with open(path, "r", encoding="utf-8") as f: + content = f.read() + except OSError as exc: + raise _input_error(hint, f"cannot read file: {exc}") from exc + try: + return parse_document(content, "the file") + except LimaCharlieError as exc: + raise _input_error(hint, exc.raw_message) from exc + + def _read_source(path: str) -> str: """Read a template file, or stdin for ``-``.""" if path == "-": @@ -441,14 +482,14 @@ def _read_source(path: str) -> str: with open(path, "r", encoding="utf-8") as f: return f.read() except OSError as exc: - raise click.BadParameter(f"cannot read file: {exc}", param_hint="--file") + raise _input_error("--file", f"cannot read file: {exc}") from exc def _parse_kv(pairs: tuple[str, ...], hint: str) -> dict[str, str]: out: dict[str, str] = {} for pair in pairs: if "=" not in pair: - raise click.BadParameter(f"expected KEY=VALUE, got {pair!r}", param_hint=hint) + raise _input_error(hint, f"expected KEY=VALUE, got {pair!r}") key, value = pair.split("=", 1) out[key] = value return out @@ -460,9 +501,9 @@ def _json_object(text: str | None, hint: str) -> dict[str, Any] | None: try: value = json.loads(text) except json.JSONDecodeError as exc: - raise click.BadParameter(f"invalid JSON: {exc}", param_hint=hint) + raise _input_error(hint, f"invalid JSON: {exc}") from exc if not isinstance(value, dict): - raise click.BadParameter("must be a JSON object", param_hint=hint) + raise _input_error(hint, "must be a JSON object") return value @@ -699,7 +740,7 @@ def _print_test(ctx: click.Context, test: dict[str, Any], include_steps: bool) - @click.option("--last", type=click.IntRange(1, 25), default=10, show_default=True, help="How many alerts per --detect-cat.") @click.option("--sample", "samples_json", multiple=True, help="Sample as a JSON object, e.g. '{\"inputs\": {...}, \"expected\": \"false_positive\"}' (repeatable).") @click.option("--samples-file", default=None, help="YAML/JSON file holding a list of samples.") -@click.option("--mode", type=click.Choice(["triage", "full"]), default=None, help="triage (default, the cheap pass) or full.") +@click.option("--mode", type=click.Choice(["triage", "full"]), default=None, help="triage (default for a test, the cheap pass) or full.") @click.option("--strict", is_flag=True, default=False, help="Treat any unknown as a failure of the plan.") @click.option("--overrides-json", default=None, help="Tighten-only budget overrides, e.g. '{\"budgets\": {\"max_wall_seconds\": 120}}'.") @click.option("--wait", "wait_seconds", type=click.IntRange(0, MAX_POLL_WAIT), default=5, show_default=True, @@ -727,9 +768,9 @@ def templates_test(ctx, name, file_path, test_id, detection_ids, case_ids, detec for text in samples_json: samples.append(_json_object(text, "--sample") or {}) if samples_file: - loaded = load_file(samples_file, "--samples-file") + loaded = _load_document(samples_file, "--samples-file") if not isinstance(loaded, list) or not all(isinstance(s, dict) for s in loaded): - raise click.BadParameter("must be a list of sample objects", param_hint="--samples-file") + raise _input_error("--samples-file", "must be a list of sample objects") samples += loaded if not samples: raise click.UsageError("Give at least one sample: --detection-id, --case-id, --detect-cat, --sample or --samples-file.") @@ -840,7 +881,7 @@ def runs_group() -> None: @click.option("--input", "inputs", multiple=True, help="Run input KEY=VALUE (repeatable; values are text).") @click.option("--inputs-json", default=None, help="Run inputs as a JSON object (typed values).") @click.option("--urgency", type=click.Choice(["low", "normal", "high"]), default=None, help="Run urgency.") -@click.option("--mode", type=click.Choice(["triage", "full", "scope"]), default=None, help="Run mode (engine default: triage).") +@click.option("--mode", type=click.Choice(["triage", "full", "scope"]), default=None, help="Run mode (engine default: full; triage is cheaper).") @click.option("--case-mode", "case_mode", default=None, help="create, none or attach: (default per the org's policy).") @click.option("--idempotency-key", default=None, help="Starting twice with the same key returns the same run.") @click.option("--wait", "wait_seconds", type=click.IntRange(0, 60), default=0, help="Block up to this many seconds (0..60) for the verdict.") @@ -1018,11 +1059,11 @@ def runs_cancel(ctx, run_id, revert_leases) -> None: @_engine_errors def wave_cmd(ctx, file_path, stop_when, deadline_ms, inputs, full, validate_only) -> None: """Run a batch of operator and fact-pack questions in one call.""" - wave = load_file(file_path, "--file") + wave = _load_document(file_path, "--file") if isinstance(wave, list): wave = {"steps": wave} if not isinstance(wave, dict): - raise click.BadParameter("must hold a mapping (steps, stop_when, ...) or a list of steps", param_hint="--file") + raise _input_error("--file", "must hold a mapping (steps, stop_when, ...) or a list of steps") wave = dict(wave) if stop_when: wave["stop_when"] = list(wave.get("stop_when") or []) + list(stop_when) diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index b2f3c0e4..14cbb11d 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -209,7 +209,13 @@ def _wrap_error(exc: LimaCharlieError) -> LimaCharlieError: def _template_record_errors(fn: Any) -> Any: - """Say which permission a refused template-record operation needs.""" + """Report a failed template-record operation as an :class:`InvestigationError`. + + A refused permission also says which permission the operation needs. Any + other Hive or API failure keeps its message, HTTP status, response body (so + ``error_code`` still carries the backend's ``lc_error_code``) and exit code, + so callers handle one error type whatever went wrong. + """ @functools.wraps(fn) def wrapper(*args: Any, **kwargs: Any) -> Any: try: @@ -217,12 +223,16 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: except InvestigationError: raise except LimaCharlieError as exc: - if is_permission_denied(exc): - raise InvestigationError( - exc.raw_message, status_code=403, response_body=getattr(exc, "response_body", None), - suggestion=_TEMPLATE_PERMISSION_HINT, exit_code=exc.exit_code, - ) from exc - raise + denied = is_permission_denied(exc) + if not denied and not isinstance(exc, ApiError): + raise # input errors and the like are not Hive answers + raise InvestigationError( + exc.raw_message, + status_code=403 if denied else exc.status_code, + response_body=getattr(exc, "response_body", None), + suggestion=_TEMPLATE_PERMISSION_HINT if denied else (exc.suggestion or ""), + exit_code=exc.exit_code, + ) from exc return wrapper @@ -284,6 +294,51 @@ def _construct_core_int(loader: yaml.SafeLoader, node: yaml.Node) -> int: _TemplateLoader.add_constructor(_INT_TAG, _construct_core_int) +def _load_yaml(source: str, allow_aliases: bool) -> Any: + """Parse YAML with the engine's YAML 1.2 scalar rules; raise ``yaml.YAMLError``.""" + if not allow_aliases: + for event in yaml.parse(source, Loader=_TemplateLoader): + if isinstance(event, yaml.AliasEvent): + raise ValidationError( + "YAML anchors and aliases are not supported in a template; write the value out" + ) + return yaml.load(source, Loader=_TemplateLoader) # noqa: S506 - SafeLoader subclass + + +def parse_document(source: str, what: str = "the document") -> Any: + """Parse a YAML or JSON document into plain JSON-compatible values. + + Scalars follow the same YAML 1.2 rules as a template: ``off``/``yes``/``no`` + stay text and an unquoted date stays a string, so every value is sent to + the engine as it was written. + + Args: + source: The YAML or JSON text. + what: What the text is, for error messages (for example ``"the wave file"``). + + Returns: + Any: The parsed value. + + Raises: + ValidationError: If the text is not YAML or JSON, or holds a value + JSON cannot carry (a binary or set tag, for example). + """ + if not isinstance(source, str) or not source.strip(): + raise ValidationError(f"{what} is empty") + try: + doc = _load_yaml(source, allow_aliases=True) + except yaml.YAMLError as exc: + try: + doc = json.loads(source) + except ValueError: + raise ValidationError(f"{what} is neither valid YAML nor JSON: {exc}") from exc + try: + json.dumps(doc) + except (TypeError, ValueError) as exc: + raise ValidationError(f"{what} holds a value JSON cannot carry: {exc}") from exc + return doc + + def parse_template_source(source: str) -> dict[str, Any]: """Parse a template's YAML source into the mapping stored in the hive record. @@ -300,12 +355,7 @@ def parse_template_source(source: str) -> dict[str, Any]: if not isinstance(source, str) or not source.strip(): raise ValidationError("the template is empty") try: - for event in yaml.parse(source, Loader=_TemplateLoader): - if isinstance(event, yaml.AliasEvent): - raise ValidationError( - "YAML anchors and aliases are not supported in a template; write the value out" - ) - doc = yaml.load(source, Loader=_TemplateLoader) # noqa: S506 - SafeLoader subclass + doc = _load_yaml(source, allow_aliases=False) except yaml.YAMLError as exc: raise ValidationError(f"the template is not valid YAML: {exc}") from exc if not isinstance(doc, dict): diff --git a/tests/unit/fixtures/ie_catalogue.json b/tests/unit/fixtures/ie_catalogue.json new file mode 100644 index 00000000..903ced29 --- /dev/null +++ b/tests/unit/fixtures/ie_catalogue.json @@ -0,0 +1,1116 @@ +{ + "_source": "Recorded from the Investigation Engine operator catalogue and managed packs: operator names, argument names, top-level output fields; pack params and exports. Refresh when the engine's catalogue changes.", + "operators": [ + { + "args": [ + { + "name": "indicators", + "required": true + } + ], + "class": "index", + "name": "case.related", + "output": { + "properties": [ + "cases", + "labeled_cases", + "labels", + "total_cases" + ] + } + }, + { + "args": [ + { + "name": "detect_id", + "required": false + }, + { + "name": "msg_uuid", + "required": false + }, + { + "name": "cat", + "required": false + }, + { + "name": "skip_if", + "required": false + }, + { + "name": "window", + "required": true + } + ], + "class": "index", + "name": "detection.get", + "output": { + "properties": [ + "atom", + "attack", + "cat", + "command_line", + "detect_id", + "event_type", + "file_path", + "found", + "gen_time", + "hash", + "hostname", + "mailbox", + "msg_uuid", + "parent_atom", + "pid", + "priority", + "sid", + "skipped", + "user_name" + ] + } + }, + { + "args": [ + { + "name": "window", + "required": true + }, + { + "name": "sids", + "required": false + }, + { + "name": "cat", + "required": false + }, + { + "name": "match", + "required": false + } + ], + "class": "index", + "name": "detection.list", + "output": { + "properties": [ + "count", + "detections", + "ids" + ] + } + }, + { + "args": [ + { + "name": "copies", + "required": true + }, + { + "name": "domains", + "required": true + }, + { + "name": "weak_domains", + "required": false + }, + { + "name": "window", + "required": true + } + ], + "class": "scan", + "name": "endpoint.click_check", + "output": { + "properties": [ + "all_known", + "any_clicked", + "any_possible", + "clicked", + "clicked_sids", + "count", + "domains", + "event", + "per_recipient", + "possible", + "unknown_count", + "weak_domains" + ] + } + }, + { + "args": [ + { + "name": "entity_id", + "required": true + }, + { + "name": "sources", + "required": false + }, + { + "name": "window", + "required": false + } + ], + "class": "index", + "name": "entity.activity", + "output": { + "properties": [ + "detection_count", + "detections", + "sources" + ] + } + }, + { + "args": [ + { + "name": "entity_id", + "required": true + } + ], + "class": "index", + "name": "entity.get", + "output": { + "properties": [ + "attrs", + "directory", + "display_name", + "emails", + "entity_id", + "kind", + "open_findings", + "owned_by", + "owns", + "sids" + ] + } + }, + { + "args": [ + { + "name": "type", + "required": true + }, + { + "name": "value", + "required": true + } + ], + "class": "index", + "name": "entity.resolve", + "output": { + "properties": [ + "ambiguous", + "confidence", + "count", + "display_name", + "entity_id", + "kind", + "matches", + "resolved" + ] + } + }, + { + "args": [ + { + "name": "entity_id", + "required": true + }, + { + "name": "kind", + "required": false + } + ], + "class": "index", + "name": "entity.sightings", + "output": { + "properties": [ + "best_effort", + "count", + "hostnames", + "ips", + "users" + ] + } + }, + { + "args": [ + { + "name": "sid", + "required": true + }, + { + "name": "atom", + "required": true + } + ], + "class": "index", + "name": "event.children", + "output": { + "properties": [ + "count", + "destinations", + "domains", + "events" + ] + } + }, + { + "args": [ + { + "name": "sid", + "required": true + }, + { + "name": "atom", + "required": true + } + ], + "class": "index", + "name": "event.get", + "output": { + "properties": [ + "atom", + "event", + "event_type", + "fields", + "hostname", + "parent", + "sid", + "ts" + ] + } + }, + { + "args": [ + { + "name": "extension", + "required": true + }, + { + "name": "action", + "required": true + }, + { + "name": "data", + "required": false + } + ], + "class": "ext", + "name": "extension.request", + "output": { + "properties": [] + } + }, + { + "args": [ + { + "name": "domain", + "required": false + }, + { + "name": "ip", + "required": false + }, + { + "name": "file_hash", + "required": false + }, + { + "name": "hash", + "required": false + }, + { + "name": "file_path", + "required": false + }, + { + "name": "user", + "required": false + } + ], + "class": "scan", + "name": "ioc.prevalence", + "output": { + "properties": [ + "hosts_1d", + "hosts_30d", + "hosts_365d", + "hosts_7d", + "hosts_max", + "seen" + ] + } + }, + { + "args": [ + { + "name": "name", + "required": true + }, + { + "name": "keys", + "required": true + } + ], + "class": "index", + "name": "lookup.get", + "output": { + "properties": [ + "all_found", + "count", + "found", + "missing", + "name", + "results" + ] + } + }, + { + "args": [ + { + "name": "domain", + "required": true + } + ], + "class": "index", + "name": "mailsec.domain_profile", + "output": { + "properties": [ + "distinct_recipients", + "domain", + "first_seen", + "first_seen_days_ago", + "flagged_180d", + "known", + "messages_30d" + ] + } + }, + { + "args": [ + { + "name": "group_id", + "required": true + } + ], + "class": "index", + "name": "mailsec.group_recipients", + "output": { + "properties": [ + "action_attrs", + "copies", + "count", + "recipients" + ] + } + }, + { + "args": [ + { + "name": "links", + "required": true + }, + { + "name": "sender_domain", + "required": false + }, + { + "name": "reply_to_domain", + "required": false + } + ], + "class": "index", + "name": "mailsec.link_resolve", + "output": { + "properties": [ + "count", + "derived", + "final_differs_from_shown", + "final_domain", + "final_domain_matches_shown", + "final_matches_shown", + "final_url", + "first_hop_domain", + "first_hop_matches_reply_to", + "first_hop_root", + "hops", + "lands_on_sender_domain", + "limitation", + "links", + "open_redirect", + "shown_domain", + "verified", + "via_redirect", + "via_tracker" + ] + } + }, + { + "args": [ + { + "name": "msg_uuid", + "required": true + } + ], + "class": "index", + "name": "mailsec.message_get", + "output": { + "properties": [ + "actions", + "campaign_id", + "decision", + "direction", + "fired_signals", + "group_id", + "links", + "lookalike", + "mailbox", + "msg_id", + "msg_uuid", + "provider", + "received", + "reply_to", + "reply_to_domain", + "reply_to_mismatch", + "reply_to_root", + "score", + "sender", + "sender_domain", + "sender_name", + "sender_sld", + "severity", + "signals", + "snapshot", + "state", + "subject", + "unranked_link_domains", + "verdict" + ] + } + }, + { + "args": [ + { + "name": "mailbox", + "required": true + }, + { + "name": "window", + "required": true + }, + { + "name": "subject_contains", + "required": false + }, + { + "name": "exclude_sender_domain", + "required": false + }, + { + "name": "exclude_msg", + "required": false + }, + { + "name": "before", + "required": false + } + ], + "class": "index", + "name": "mailsec.search_messages", + "output": { + "properties": [ + "count", + "found", + "matches", + "minutes_before" + ] + } + }, + { + "args": [ + { + "name": "sender", + "required": true + }, + { + "name": "domain", + "required": false + }, + { + "name": "before", + "required": false + }, + { + "name": "exclude_group", + "required": false + } + ], + "class": "index", + "name": "mailsec.sender_profile", + "output": { + "properties": [ + "distinct_recipients", + "domain", + "domain_messages_30d", + "first_prior_days_ago", + "first_seen", + "first_seen_days_ago", + "flagged_180d", + "known", + "messages_30d", + "prior_messages_30d", + "sender" + ] + } + }, + { + "args": [ + { + "name": "limit", + "required": false + }, + { + "name": "online_only", + "required": false + }, + { + "name": "selector", + "required": false + }, + { + "name": "cursor", + "required": false + }, + { + "name": "owners", + "required": false + } + ], + "class": "index", + "name": "org.roster", + "output": { + "properties": [ + "machine_count", + "machines", + "more", + "next_cursor", + "offline_count", + "online_count", + "owner_lookup", + "people", + "people_count", + "unowned_count" + ] + } + }, + { + "args": [ + { + "name": "candidates", + "required": true + }, + { + "name": "optional", + "required": false + } + ], + "class": "index", + "name": "plan.pick", + "output": { + "properties": [] + } + }, + { + "args": [ + { + "name": "window", + "required": true + }, + { + "name": "sids", + "required": false + }, + { + "name": "selector", + "required": false + }, + { + "name": "event_types", + "required": false + }, + { + "name": "filter", + "required": false + }, + { + "name": "params", + "required": false + }, + { + "name": "dns_domain", + "required": false + }, + { + "name": "before", + "required": false + } + ], + "class": "scan", + "name": "search.exists", + "output": { + "properties": [ + "count", + "event", + "events_scanned", + "first", + "found", + "query" + ] + } + }, + { + "args": [ + { + "name": "window", + "required": true + }, + { + "name": "sids", + "required": false + }, + { + "name": "selector", + "required": false + }, + { + "name": "event_types", + "required": false + }, + { + "name": "filter", + "required": false + }, + { + "name": "params", + "required": false + }, + { + "name": "dns_domain", + "required": false + }, + { + "name": "before", + "required": false + }, + { + "name": "limit", + "required": false + } + ], + "class": "scan", + "name": "search.rows", + "output": { + "properties": [ + "count", + "events_scanned", + "found", + "query", + "rows", + "truncated" + ] + } + }, + { + "args": [ + { + "name": "sid", + "required": true + }, + { + "name": "hash", + "required": false + }, + { + "name": "contains", + "required": false + }, + { + "name": "limit", + "required": false + } + ], + "class": "live", + "name": "sensor.read.autoruns", + "output": { + "properties": [ + "autoruns", + "count", + "filtered", + "found", + "match_count", + "matches", + "sid", + "truncated" + ] + } + }, + { + "args": [ + { + "name": "sid", + "required": true + }, + { + "name": "path", + "required": true + }, + { + "name": "hash", + "required": false + } + ], + "class": "live", + "name": "sensor.read.file_hash", + "output": { + "properties": [ + "cert_issuer", + "cert_subject", + "error_code", + "hash", + "hashed", + "matches", + "path", + "sid", + "signed", + "verified" + ] + } + }, + { + "args": [ + { + "name": "sid", + "required": true + }, + { + "name": "pid", + "required": false + }, + { + "name": "limit", + "required": false + } + ], + "class": "live", + "name": "sensor.read.netstat", + "output": { + "properties": [ + "connections", + "count", + "public_remote_count", + "public_remote_ips", + "remote_ips", + "sid", + "truncated" + ] + } + }, + { + "args": [ + { + "name": "sid", + "required": true + }, + { + "name": "pid", + "required": false + }, + { + "name": "hash", + "required": false + }, + { + "name": "contains", + "required": false + }, + { + "name": "limit", + "required": false + } + ], + "class": "live", + "name": "sensor.read.processes", + "output": { + "properties": [ + "count", + "filtered", + "found", + "match_count", + "matches", + "processes", + "sid", + "truncated" + ] + } + }, + { + "args": [ + { + "name": "sid", + "required": true + }, + { + "name": "hash", + "required": false + }, + { + "name": "contains", + "required": false + }, + { + "name": "limit", + "required": false + } + ], + "class": "live", + "name": "sensor.read.services", + "output": { + "properties": [ + "count", + "filtered", + "found", + "match_count", + "matches", + "services", + "sid", + "truncated" + ] + } + } + ], + "packs": [ + { + "exports": [ + "found", + "cat", + "sid", + "hostname", + "event_type", + "atom", + "parent_atom", + "hash", + "file_path", + "command_line", + "user_name", + "gen_time", + "host_detection_count", + "same_rule_count" + ], + "name": "detection_context", + "params": [ + { + "name": "detection_id", + "required": true + } + ] + }, + { + "exports": [ + "sensors_30d", + "sensors_365d", + "looked_up_7d", + "first_lookup", + "mail_known", + "mail_messages_30d", + "mail_flagged_180d", + "mail_first_seen" + ], + "name": "domain_reputation", + "params": [ + { + "name": "domain", + "required": true + } + ] + }, + { + "exports": [ + "seen", + "hosts_1d", + "hosts_7d", + "hosts_30d", + "hosts_365d" + ], + "name": "hash_spread", + "params": [ + { + "name": "hash", + "required": true + } + ] + }, + { + "exports": [ + "resolved", + "entity_id", + "hostname", + "kind", + "sids", + "open_findings", + "owner_id", + "owner_name" + ], + "name": "host_context", + "params": [ + { + "name": "sid", + "required": true + } + ] + }, + { + "exports": [ + "sender", + "sender_domain", + "subject", + "received", + "state", + "verdict", + "fired_signals", + "decision", + "recipient_count", + "final_domain", + "open_redirect", + "via_tracker", + "any_clicked", + "clicked", + "per_recipient" + ], + "name": "message_outcome", + "params": [ + { + "name": "msg_uuid", + "required": true + } + ] + }, + { + "exports": [ + "machine_count", + "online_count", + "offline_count", + "people_count", + "unowned_count", + "owner_lookup", + "more", + "next_cursor", + "machines", + "people" + ], + "name": "org_roster", + "params": [ + { + "name": "limit", + "required": false + }, + { + "name": "online_only", + "required": false + }, + { + "name": "selector", + "required": false + }, + { + "name": "cursor", + "required": false + } + ] + }, + { + "exports": [ + "event_type", + "hostname", + "fields", + "parent_atom", + "parent_type", + "parent_fields", + "child_count", + "children", + "domains", + "destinations" + ], + "name": "process_tree", + "params": [ + { + "name": "sid", + "required": true + }, + { + "name": "atom", + "required": true + } + ] + }, + { + "exports": [ + "known", + "messages_30d", + "prior_messages_30d", + "distinct_recipients", + "flagged_180d", + "first_seen", + "domain_messages_30d", + "domain_known", + "domain_flagged_180d" + ], + "name": "sender_reputation", + "params": [ + { + "name": "sender", + "required": true + }, + { + "name": "sender_domain", + "required": true + }, + { + "name": "before", + "required": false + }, + { + "name": "exclude_group", + "required": false + } + ] + }, + { + "exports": [ + "resolved", + "entity_id", + "display_name", + "department", + "job_title", + "account_status", + "mfa", + "laptop_name", + "laptop_sids", + "detection_count", + "detections", + "sources" + ], + "name": "user_activity", + "params": [ + { + "name": "user", + "required": true + }, + { + "name": "window", + "required": false + } + ] + } + ] +} diff --git a/tests/unit/test_cli_investigate.py b/tests/unit/test_cli_investigate.py index bf55c343..51244edf 100644 --- a/tests/unit/test_cli_investigate.py +++ b/tests/unit/test_cli_investigate.py @@ -7,7 +7,7 @@ from click.testing import CliRunner from limacharlie.cli import cli -from limacharlie.errors import error_from_status_code +from limacharlie.errors import AuthenticationError, error_from_status_code from limacharlie.sdk.investigation import InvestigationError @@ -506,3 +506,74 @@ def test_table_mode_raises_the_error_for_the_normal_text_rendering(self, sdk): result = run(["investigate", "runs", "get", "r1"], fmt="table") assert result.exit_code != 0 assert isinstance(result.exception, InvestigationError) + + +class TestYaml12Input: + """Wave and sample files keep YAML 1.2 scalars: values reach the engine as written.""" + + def test_wave_file_words_and_dates_are_not_converted(self, sdk, tmp_path): + f = tmp_path / "wave.yaml" + f.write_text( + "steps:\n - {id: a, op: entity.get, args: {mode: off, flag: yes, since: 2026-11-10}}\n") + sdk.run_wave.return_value = {} + result = run(["investigate", "wave", "--file", str(f)]) + assert result.exit_code == 0, result.stderr + assert sdk.run_wave.call_args.args[0]["steps"][0]["args"] == {"mode": "off", "flag": "yes", "since": "2026-11-10"} + + def test_samples_file_words_and_dates_are_not_converted(self, sdk, tmp_path): + f = tmp_path / "samples.yaml" + f.write_text("- inputs: {since: 2026-11-10, quiet: off}\n expected: false_positive\n") + sdk.test_plan.return_value = {"state": "done", "done": True} + result = run(["investigate", "templates", "test", "my-plan", "--samples-file", str(f)]) + assert result.exit_code == 0, result.stderr + assert sdk.test_plan.call_args.args[0] == [ + {"inputs": {"since": "2026-11-10", "quiet": "off"}, "expected": "false_positive"}] + + def test_an_unparseable_wave_file_is_a_coded_json_error(self, sdk, tmp_path): + f = tmp_path / "wave.yaml" + f.write_text("steps: [unclosed") + result = run(["investigate", "wave", "--file", str(f)]) + assert result.exit_code == 4 + doc = json.loads(result.stderr) + assert doc["errors"][0]["code"] == "invalid_input" and doc["errors"][0]["path"] == "--file" + sdk.run_wave.assert_not_called() + + def test_a_wrong_shape_is_a_coded_json_error(self, sdk, tmp_path): + f = tmp_path / "samples.yaml" + f.write_text("just: a mapping\n") + result = run(["investigate", "templates", "test", "my-plan", "--samples-file", str(f)]) + assert result.exit_code == 4 + assert json.loads(result.stderr)["errors"][0]["path"] == "--samples-file" + + def test_a_missing_file_is_a_coded_json_error(self, sdk, tmp_path): + result = run(["investigate", "wave", "--file", str(tmp_path / "nope.yaml")]) + assert result.exit_code == 4 + assert json.loads(result.stderr)["errors"][0]["code"] == "invalid_input" + + def test_table_mode_still_fails_with_text(self, sdk, tmp_path): + f = tmp_path / "wave.yaml" + f.write_text("steps: [unclosed") + result = run(["investigate", "wave", "--file", str(f)], fmt="table") + assert result.exit_code == 1 and isinstance(result.exception, InvestigationError) + + +class TestEveryFailureIsStructured: + def test_a_coded_hive_refusal_renders_as_json_with_its_code_and_exit_code(self, sdk): + sdk.promote_template.side_effect = error_from_status_code( + 400, {"error": "lc_error_code:API_KEY_NOT_ALLOWED - an API key cannot promote a template"}) + result = run(["investigate", "templates", "promote", "my-plan", "--confirm"]) + assert result.exit_code == 1 + doc = json.loads(result.stderr) + assert doc["error_code"] == "API_KEY_NOT_ALLOWED" and "cannot promote" in doc["error"] + assert doc["status"] == 400 + + def test_other_sdk_errors_keep_their_own_exit_code(self, sdk): + sdk.get_template.side_effect = AuthenticationError("bad key") + result = run(["investigate", "templates", "get", "my-plan"]) + assert result.exit_code == AuthenticationError.exit_code + assert json.loads(result.stderr)["error"] == "bad key" + + def test_table_mode_leaves_the_error_to_the_cli(self, sdk): + sdk.promote_template.side_effect = error_from_status_code(400, {"error": "lc_error_code:X - y"}) + result = run(["investigate", "templates", "promote", "my-plan", "--confirm"], fmt="table") + assert result.exception is not None and not result.stderr.startswith("{") diff --git a/tests/unit/test_investigation_examples.py b/tests/unit/test_investigation_examples.py new file mode 100644 index 00000000..1b41cc13 --- /dev/null +++ b/tests/unit/test_investigation_examples.py @@ -0,0 +1,99 @@ +"""Every wave example in the Investigation Engine docs and help is valid against the operator catalogue. + +The catalogue is a recorded fixture (``fixtures/ie_catalogue.json``: operators with their +argument names and output fields, packs with their params and exports), so the check needs +no network. It catches an example that names an operator, pack, argument or ``select`` field +the engine does not have. +""" + +import json +import re +from pathlib import Path + +import pytest + +from limacharlie.commands import investigate +from limacharlie.sdk.investigation import parse_document + +ROOT = Path(__file__).resolve().parents[2] +DOC = ROOT / "doc" / "cli" / "investigation-engine.md" +CATALOGUE = json.loads((Path(__file__).parent / "fixtures" / "ie_catalogue.json").read_text()) +OPERATORS = {o["name"]: o for o in CATALOGUE["operators"]} +PACKS = {p["name"]: p for p in CATALOGUE["packs"]} + + +def _doc_blocks() -> list[str]: + return re.findall(r"```ya?ml\n(.*?)```", DOC.read_text(), flags=re.S) + + +def _help_blocks() -> list[str]: + """Indented YAML examples in the command help texts (a ``steps:`` block).""" + out = [] + for name in dir(investigate): + text = getattr(investigate, name) + if not (name.startswith(("_EXPLAIN", "_HELP")) and isinstance(text, str)): + continue + for m in re.finditer(r"^( +)steps:\n(?:\1.*\n?)+", text, flags=re.M): + out.append("\n".join(line[len(m.group(1)):] if line.startswith(m.group(1)) else line + for line in m.group(0).splitlines())) + return out + + +def _waves() -> list[tuple[str, dict]]: + found = [] + for i, block in enumerate(_doc_blocks()): + doc = parse_document(block, "example") + if isinstance(doc, dict) and "steps" in doc: + found.append((f"doc#{i}", doc)) + for i, block in enumerate(_help_blocks()): + found.append((f"help#{i}", parse_document(block, "example"))) + return found + + +def _check_wave(wave: dict) -> list[str]: + problems = [] + for step in wave["steps"]: + sid = step.get("id", "?") + has_op, has_use = "op" in step, "use" in step + if has_op == has_use: + problems.append(f"{sid}: needs exactly one of op or use") + continue + args = set((step.get("args") or {}).keys()) + select = set(step.get("select") or []) + if has_op: + op = OPERATORS.get(step["op"]) + if op is None: + problems.append(f"{sid}: unknown operator {step['op']}") + continue + for a in sorted(args - {a["name"] for a in op["args"]}): + problems.append(f"{sid}: {step['op']} has no argument {a}") + for f in sorted(select - set(op["output"]["properties"])): + problems.append(f"{sid}: {step['op']} has no output field {f}") + else: + pack = PACKS.get(step["use"]) + if pack is None: + problems.append(f"{sid}: unknown pack {step['use']}") + continue + for a in sorted(args - {p["name"] for p in pack["params"]}): + problems.append(f"{sid}: pack {step['use']} has no param {a}") + for p in pack["params"]: + if p["required"] and p["name"] not in args: + problems.append(f"{sid}: pack {step['use']} needs param {p['name']}") + for f in sorted(select - set(pack["exports"])): + problems.append(f"{sid}: pack {step['use']} has no export {f}") + return problems + + +def test_the_examples_were_found(): + sources = [name for name, _ in _waves()] + assert any(n.startswith("doc#") for n in sources) and any(n.startswith("help#") for n in sources) + + +@pytest.mark.parametrize("name,wave", _waves(), ids=lambda v: v if isinstance(v, str) else "") +def test_wave_example_matches_the_operator_catalogue(name, wave): + assert _check_wave(wave) == [] + + +def test_the_check_catches_a_wrong_select_field(): + wave = {"steps": [{"id": "p", "op": "ioc.prevalence", "args": {"file_hash": "x"}, "select": ["first_seen"]}]} + assert _check_wave(wave) == ["p: ioc.prevalence has no output field first_seen"] diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index af8f1443..e47ab6ff 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -17,6 +17,7 @@ Investigation, InvestigationError, dump_template, + parse_document, parse_template_source, ) @@ -523,11 +524,26 @@ def test_a_refused_hive_operation_names_the_investigation_plan_permissions(self, inv.get_template("x") assert "investigation_plan.get" in ei.value.suggestion and ei.value.status_code == 403 - def test_other_hive_failures_are_not_rewritten(self, inv, hive): + def test_other_hive_failures_keep_their_status_code_and_message(self, inv, hive): hive.delete.side_effect = error_from_status_code(500, "boom") - with pytest.raises(ApiError) as ei: + with pytest.raises(InvestigationError) as ei: inv.delete_template("x") - assert not isinstance(ei.value, InvestigationError) + assert ei.value.status_code == 500 and "boom" in ei.value.raw_message + assert "investigation_plan" not in (ei.value.suggestion or "") + + def test_a_coded_hive_refusal_keeps_its_lc_error_code(self, inv, hive): + hive.get.side_effect = error_from_status_code( + 400, {"error": "lc_error_code:API_KEY_NOT_ALLOWED - an API key cannot promote"}) + record = HiveRecord(name="p", data={"status": "draft"}, etag="e1") + hive.get.side_effect = [record] + hive.set.side_effect = error_from_status_code( + 400, {"error": "lc_error_code:API_KEY_NOT_ALLOWED - an API key cannot promote"}) + with patch.object(Investigation, "validate_plan", return_value=VALID): + with pytest.raises(InvestigationError) as ei: + inv.promote_template("p") + doc = ei.value.to_dict() + assert doc["error_code"] == "API_KEY_NOT_ALLOWED" and "cannot promote" in doc["error"] + assert doc["status"] == 400 and ei.value.exit_code == 1 class TestPromote: @@ -624,3 +640,28 @@ def test_active_status_is_written_only_when_asked(self, inv, mock_org, hive): hive.get.side_effect = [_not_found(), HiveRecord(name="alpha", data={"status": "active"}, etag="e")] out = inv.install_default_templates(["alpha"], status="active") assert out["written"][0]["status"] == "active" + + +class TestParseDocument: + def test_yaml_11_words_and_dates_are_sent_as_written(self): + doc = parse_document("a: off\nb: yes\nc: no\nd: 2026-11-10\ne: true\nf: 012\n") + assert doc == {"a": "off", "b": "yes", "c": "no", "d": "2026-11-10", "e": True, "f": "012"} + json.dumps(doc) + + def test_json_is_accepted(self): + assert parse_document('{"steps": [{"id": "a", "n": 1}]}') == {"steps": [{"id": "a", "n": 1}]} + + def test_anchors_are_expanded_not_refused(self): + assert parse_document("a: &x [1, 2]\nb: *x\n") == {"a": [1, 2], "b": [1, 2]} + + def test_garbage_is_a_validation_error(self): + with pytest.raises(ValidationError): + parse_document("a: [unclosed") + + def test_empty_is_a_validation_error(self): + with pytest.raises(ValidationError): + parse_document(" \n") + + def test_values_json_cannot_carry_are_refused(self): + with pytest.raises(ValidationError): + parse_document("a: !!binary aGk=\n") From 5a4302f41590f7a5d71a1b8daa8c9fccbce4107c Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 14:42:49 -0800 Subject: [PATCH 06/14] Investigation Engine CLI: drop a dead assignment in a test Co-Authored-By: Claude Sonnet 5.5 --- tests/unit/test_sdk_investigation.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index e47ab6ff..c80a061e 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -532,10 +532,7 @@ def test_other_hive_failures_keep_their_status_code_and_message(self, inv, hive) assert "investigation_plan" not in (ei.value.suggestion or "") def test_a_coded_hive_refusal_keeps_its_lc_error_code(self, inv, hive): - hive.get.side_effect = error_from_status_code( - 400, {"error": "lc_error_code:API_KEY_NOT_ALLOWED - an API key cannot promote"}) - record = HiveRecord(name="p", data={"status": "draft"}, etag="e1") - hive.get.side_effect = [record] + hive.get.side_effect = [HiveRecord(name="p", data={"status": "draft"}, etag="e1")] hive.set.side_effect = error_from_status_code( 400, {"error": "lc_error_code:API_KEY_NOT_ALLOWED - an API key cannot promote"}) with patch.object(Investigation, "validate_plan", return_value=VALID): From 5e20b76b28240ad069c3842815ceba063b9677fe Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 15:59:59 -0800 Subject: [PATCH 07/14] Investigation Engine CLI: flags, enums and help match the engine - runs start --urgency is low|medium|high|critical (default derived from the alert, else medium) - --auto sends no plan (the engine refuses plan.auto for a person's run) so the engine picks the active template by applies_to; new --category/--severity/--priority/--tag carry the alert attributes it matches on - templates test --strict help: refuses overrides that would loosen the policy - runs list --trigger takes the trigger kind, --verdict also takes undecided, --started-by also matches a name - policy get --template; --case-mode and --wait help corrected Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 8 ++++ doc/cli/investigation-engine.md | 9 +++-- limacharlie/commands/investigate.py | 59 ++++++++++++++++++---------- limacharlie/sdk/investigation.py | 16 ++++++-- tests/unit/test_cli_investigate.py | 46 ++++++++++++++++++++-- tests/unit/test_sdk_investigation.py | 5 +++ 6 files changed, 112 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30998650..55349f34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,14 @@ needs an engine version that serves it. - The typed 401/403/404/422/429 errors raised for an HTTP error answer now carry the parsed answer as `response_body`, as `ApiError` already did. +- The `investigate` flags and help match the engine: `runs start --urgency` is + `low|medium|high|critical` (default derived from the alert, else medium); + `--auto` sends no plan (the engine refuses `plan.auto` for a person's run) so + the engine picks the active template by `applies_to`, with new `--category`, + `--severity`, `--priority` and `--tag` to say what the alert looks like; + `templates test --strict` refuses overrides that would loosen the policy; + `runs list --trigger` takes the trigger kind (detection, message, entity, case, + question) and `--verdict` also takes `undecided`; `policy get` takes `--template`. ### Sync — `sync push` no longer turns quoted strings into numbers diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md index d893e4e1..f51376bc 100644 --- a/doc/cli/investigation-engine.md +++ b/doc/cli/investigation-engine.md @@ -116,7 +116,8 @@ Exit codes: `0` done, `4` the template is `invalid` (the `validation` report say ```bash limacharlie investigate runs start (--template NAME | --file plan.yaml | --auto) \ (--detection-id ID | --case-id ID | --entity-id ID | --question TEXT) \ - [--input K=V]... [--inputs-json JSON] [--urgency low|normal|high] [--mode triage|full|scope] \ + [--category C] [--severity S] [--priority N] [--tag T]... \ + [--input K=V]... [--inputs-json JSON] [--urgency low|medium|high|critical] [--mode triage|full|scope] \ [--case-mode create|none|attach:ID] [--idempotency-key KEY] [--wait SECONDS] [--follow] limacharlie investigate runs get RUN_ID limacharlie investigate runs list [--state S] [--verdict V] [--template T] [--trigger T] [--entity E] \ @@ -127,7 +128,9 @@ limacharlie investigate runs case RUN_ID limacharlie investigate runs cancel RUN_ID [--revert-leases] ``` -`--auto` picks the active template whose `applies_to` matches the alert. A start is sent once and never retried automatically; `--idempotency-key` makes a repeated start return the same run (`joined: true`). +`--auto` sends no plan, which is how the engine is asked to pick the **active** template whose `applies_to` matches the trigger (a draft is never picked; the engine refuses `plan.auto` itself for a person's run). `applies_to` keys are `detect_cat`, `severity`, `tag` and `source`; the first three are matched against `--category`, `--severity` and `--tag` (a start from the CLI always has `source: api`), so pass the alert's attributes. When no template matches, the start is refused with the reason. `--category`, `--severity`, `--priority` and `--tag` also select the run-scoped policy records. `--urgency` defaults to a value derived from `--priority` (8+ critical, 6+ high, 3+ medium, else low) or `--severity`, and medium when neither is given. A start is sent once and never retried automatically; `--idempotency-key` makes a repeated start return the same run (`joined: true`). + +`runs list` filters: `--state` (queued, running, decided, completed, cancelled, failed, budget_exhausted), `--verdict` (true_positive, false_positive, inconclusive, undecided), `--trigger` (the kind of trigger: detection, message, entity, case, question) and `--started-by` (an identity such as `user:UID` or `apikey:ID`, or a user's email or a key's name). `--mode` is `full` by default (triage, scoping and containment); `triage` stops at the verdict and costs less, `scope` is for scoping an already-triaged alert. (`templates test` defaults to `triage`.) @@ -165,7 +168,7 @@ An invalid wave prints `{"valid": false, "errors": [...], "warnings": [...]}` wi limacharlie investigate actions list [--state awaiting_approval,proposed] [--class C] [--run-id ID] [--limit N] [--cursor C | --all] limacharlie investigate actions approve RUN_ID ACTION_ID limacharlie investigate actions deny RUN_ID ACTION_ID [--note TEXT] -limacharlie investigate policy get [--category C] [--priority N] [--severity S] [--target-tag T]... [--class C] +limacharlie investigate policy get [--category C] [--priority N] [--severity S] [--target-tag T]... [--class C] [--template NAME] limacharlie investigate policy status limacharlie investigate policy pause limacharlie investigate policy resume diff --git a/limacharlie/commands/investigate.py b/limacharlie/commands/investigate.py index bf551756..a9569e9b 100644 --- a/limacharlie/commands/investigate.py +++ b/limacharlie/commands/investigate.py @@ -228,8 +228,12 @@ inconclusive) with a proof. Pick the plan with --template NAME, --file plan.yaml (an inline draft) or ---auto (the active template whose applies_to matches the alert). Say what to -investigate with --detection-id, --case-id, --entity-id or --question. A +--auto (the engine picks the active template whose applies_to matches the +trigger: detect_cat, severity, tag and source, the first three from --category, +--severity and --tag; a draft is never picked, and no match is refused with the +reason). Say what to investigate with --detection-id, --case-id, --entity-id or +--question. --urgency (low, medium, high, critical) defaults to a value derived +from --priority and --severity, else medium. A draft template runs with autonomy capped at suggest; an active one acts as the org's policy says (response actions may wait for a person's approval). --idempotency-key makes a repeated start return the same run. @@ -241,7 +245,7 @@ Examples: limacharlie investigate runs start --template my-plan --detection-id - limacharlie investigate runs start --auto --case-id 42 --wait 30 + limacharlie investigate runs start --auto --case-id 42 --category my-category --wait 30 """ _EXPLAIN_RUNS_GET = """\ @@ -741,7 +745,7 @@ def _print_test(ctx: click.Context, test: dict[str, Any], include_steps: bool) - @click.option("--sample", "samples_json", multiple=True, help="Sample as a JSON object, e.g. '{\"inputs\": {...}, \"expected\": \"false_positive\"}' (repeatable).") @click.option("--samples-file", default=None, help="YAML/JSON file holding a list of samples.") @click.option("--mode", type=click.Choice(["triage", "full"]), default=None, help="triage (default for a test, the cheap pass) or full.") -@click.option("--strict", is_flag=True, default=False, help="Treat any unknown as a failure of the plan.") +@click.option("--strict", is_flag=True, default=False, help="Refuse the test when --overrides-json would loosen the org's policy (default: the loosening part is ignored and reported in policy.clamped).") @click.option("--overrides-json", default=None, help="Tighten-only budget overrides, e.g. '{\"budgets\": {\"max_wall_seconds\": 120}}'.") @click.option("--wait", "wait_seconds", type=click.IntRange(0, MAX_POLL_WAIT), default=5, show_default=True, help="Seconds the starting call waits for a fast test.") @@ -873,23 +877,31 @@ def runs_group() -> None: @runs_group.command("start") @click.option("--template", default=None, help="Stored template to run.") @_file_option -@click.option("--auto", is_flag=True, default=False, help="Pick the active template whose applies_to matches the alert.") +@click.option("--auto", is_flag=True, default=False, + help="Let the engine pick the active template whose applies_to matches the trigger (sends no plan). " + "applies_to keys: detect_cat, severity, tag (set them with --category, --severity, --tag) and source (always api here). " + "Refused with a reason when no template matches.") @click.option("--detection-id", default=None, help="Detection to investigate.") @click.option("--case-id", default=None, help="Case to investigate.") @click.option("--entity-id", default=None, help="Entity to investigate.") @click.option("--question", default=None, help="Free-text question, for a template that takes one.") +@click.option("--category", default=None, help="Detection category of the alert: matched by --auto and by run-scoped policy records.") +@click.option("--severity", default=None, help="Severity of the alert (low, medium, high, critical): matched by --auto and policy records; sets the default urgency.") +@click.option("--priority", type=int, default=None, help="Detection priority: matched by policy records; sets the default urgency (8+ critical, 6+ high, 3+ medium).") +@click.option("--tag", "tags", multiple=True, help="Tag of the alert (repeatable): matched by --auto and by run-scoped policy records.") @click.option("--input", "inputs", multiple=True, help="Run input KEY=VALUE (repeatable; values are text).") @click.option("--inputs-json", default=None, help="Run inputs as a JSON object (typed values).") -@click.option("--urgency", type=click.Choice(["low", "normal", "high"]), default=None, help="Run urgency.") +@click.option("--urgency", type=click.Choice(["low", "medium", "high", "critical"]), default=None, + help="Run urgency; it picks the decision thresholds. Default: derived from --priority / --severity, else medium.") @click.option("--mode", type=click.Choice(["triage", "full", "scope"]), default=None, help="Run mode (engine default: full; triage is cheaper).") -@click.option("--case-mode", "case_mode", default=None, help="create, none or attach: (default per the org's policy).") +@click.option("--case-mode", "case_mode", default=None, help="create, none or attach: (default: the org's policy, or the template's report.case).") @click.option("--idempotency-key", default=None, help="Starting twice with the same key returns the same run.") -@click.option("--wait", "wait_seconds", type=click.IntRange(0, 60), default=0, help="Block up to this many seconds (0..60) for the verdict.") +@click.option("--wait", "wait_seconds", type=click.IntRange(0, 60), default=0, help="Block up to this many seconds for the verdict (this CLI accepts 0..60; the run goes on after it).") @click.option("--follow", is_flag=True, default=False, help="Then stream the run's events until it ends.") @pass_context @_engine_errors -def runs_start(ctx, template, file_path, auto, detection_id, case_id, entity_id, question, inputs, inputs_json, - urgency, mode, case_mode, idempotency_key, wait_seconds, follow) -> None: +def runs_start(ctx, template, file_path, auto, detection_id, case_id, entity_id, question, category, severity, priority, + tags, inputs, inputs_json, urgency, mode, case_mode, idempotency_key, wait_seconds, follow) -> None: """Start an investigation run on a detection, case, entity or question.""" if sum(bool(x) for x in (template, file_path, auto)) != 1: raise click.UsageError("Give exactly one of --template, --file or --auto.") @@ -897,14 +909,18 @@ def runs_start(ctx, template, file_path, auto, detection_id, case_id, entity_id, ("entity_id", entity_id), ("question", question)) if v} if not trigger: raise click.UsageError("Give one of --detection-id, --case-id, --entity-id or --question.") - plan: dict[str, Any] + # The engine refuses plan.auto for a person's run; a start that names no plan at all is the one it gives the + # template whose applies_to matches the trigger, so --auto sends no plan. + for key, value in (("category", category), ("severity", severity), ("priority", priority)): + if value is not None and value != "": + trigger[key] = value + if tags: + trigger["tags"] = list(tags) + request: dict[str, Any] = {"trigger": trigger} if template: - plan = {"template": template} + request["plan"] = {"template": template} elif file_path: - plan = {"inline": _read_source(file_path)} - else: - plan = {"auto": True} - request: dict[str, Any] = {"trigger": trigger, "plan": plan} + request["plan"] = {"inline": _read_source(file_path)} run_inputs: dict[str, Any] = _parse_kv(inputs, "--input") run_inputs.update(_json_object(inputs_json, "--inputs-json") or {}) if run_inputs: @@ -939,11 +955,11 @@ def runs_get(ctx, run_id) -> None: @runs_group.command("list") @click.option("--state", default=None, help="Only runs in this state (queued, running, decided, completed, cancelled, failed, budget_exhausted).") -@click.option("--verdict", default=None, help="Only runs with this verdict (true_positive, false_positive, inconclusive).") +@click.option("--verdict", default=None, help="Only runs with this verdict (true_positive, false_positive, inconclusive, undecided).") @click.option("--template", default=None, help="Only runs of this template.") -@click.option("--trigger", default=None, help="Only runs with this trigger source (api, dr-rule, case).") +@click.option("--trigger", default=None, help="Only runs started from this kind of trigger (detection, message, entity, case, question).") @click.option("--entity", default=None, help="Only runs about this entity.") -@click.option("--started-by", default=None, help="Only runs started by this identity, e.g. user:.") +@click.option("--started-by", default=None, help="Only runs started by this identity (user:, apikey:) or, failing that, this name (a user's email, a key's name).") @click.option("--idempotency-key", default=None, help="Only the run started with this key.") @click.option("--since", default=None, help="Only runs created at or after this RFC 3339 time.") @click.option("--until", default=None, help="Only runs created before this RFC 3339 time.") @@ -1153,13 +1169,14 @@ def policy_group() -> None: @click.option("--severity", default=None, help="Detection severity.") @click.option("--target-tag", "target_tags", multiple=True, help="Tag of the target sensor (repeatable).") @click.option("--class", "action_class", default=None, help="Restrict to one action class.") +@click.option("--template", default=None, help="Template whose run identity to explain.") @pass_context @_engine_errors -def policy_get(ctx, category, priority, severity, target_tags, action_class) -> None: +def policy_get(ctx, category, priority, severity, target_tags, action_class, template) -> None: """Explain the effective policy.""" command_output(ctx, _get_investigation(ctx).get_effective_policy( category=category, priority=priority, severity=severity, - target_tags=list(target_tags) or None, action_class=action_class, + target_tags=list(target_tags) or None, action_class=action_class, template=template, )) diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index 14cbb11d..76e75c88 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -494,6 +494,7 @@ def get_effective_policy( severity: str | None = None, target_tags: list[str] | None = None, action_class: str | None = None, + template: str | None = None, ) -> dict[str, Any]: """Explain the effective policy, optionally for a given alert shape. @@ -503,6 +504,7 @@ def get_effective_policy( severity: Detection severity. target_tags: Tags of the target sensor. action_class: Restrict the explanation to one action class. + template: Template whose run identity (``identity``) to explain. Returns: dict: The explained policy, including ``extension_requests``. @@ -512,6 +514,7 @@ def get_effective_policy( self._q( category=category, priority=priority, severity=severity, target_tags=",".join(target_tags) if target_tags else None, class_=action_class, + template=template, ), ) @@ -578,7 +581,9 @@ def test_plan( template: A stored template (draft or active). Give this or ``inline``. inline: Template YAML to test as a draft. overrides: Tighten-only budget overrides. - strict: Treat any unknown as a failure of the plan. + strict: Refuse the test when ``overrides`` would loosen the org's + policy; without it the loosening part is ignored and listed in + ``policy.clamped``. mode: ``triage`` (default) or ``full``. wait_seconds: Let the call wait this long for a fast test (0..10). @@ -664,9 +669,12 @@ def start_run(self, request: dict[str, Any]) -> dict[str, Any]: """Start an investigation run. Args: - request: The run request: ``trigger``, ``plan`` (``template``, - ``inline`` or ``auto``), ``inputs``, ``urgency``, ``mode``, - ``case``, ``idempotency_key``, ``wait_seconds``, ... + request: The run request: ``trigger``, ``plan`` (``template`` or + ``inline``; omit it to let the engine pick the active template + whose ``applies_to`` matches the trigger; ``{"auto": true}`` is + refused for a person's run), ``inputs``, ``urgency`` (``low``, + ``medium``, ``high`` or ``critical``), ``mode``, ``case``, + ``idempotency_key``, ``wait_seconds``, ... Returns: dict: ``run`` and ``joined`` (True when an open run of the same diff --git a/tests/unit/test_cli_investigate.py b/tests/unit/test_cli_investigate.py index 51244edf..ca878458 100644 --- a/tests/unit/test_cli_investigate.py +++ b/tests/unit/test_cli_investigate.py @@ -292,12 +292,36 @@ def test_start_builds_the_request(self, sdk): def test_start_plan_choices(self, sdk, tmp_path): sdk.start_run.return_value = {"run": {"run_id": "r1"}} run(["investigate", "runs", "start", "--auto", "--case-id", "5"]) - assert sdk.start_run.call_args.args[0] == {"trigger": {"case_id": "5"}, "plan": {"auto": True}} + # The engine refuses plan.auto for a person's run; no plan at all is what asks it to pick by applies_to. + assert sdk.start_run.call_args.args[0] == {"trigger": {"case_id": "5"}} f = tmp_path / "p.yaml" f.write_text("description: x\n") run(["investigate", "runs", "start", "--file", str(f), "--question", "why?"]) assert sdk.start_run.call_args.args[0] == {"trigger": {"question": "why?"}, "plan": {"inline": "description: x\n"}} + def test_start_auto_sends_the_alert_attributes_applies_to_matches_on(self, sdk): + sdk.start_run.return_value = {"run": {"run_id": "r1"}} + result = run(["investigate", "runs", "start", "--auto", "--detection-id", "d", "--category", "encoded-ps", + "--severity", "high", "--priority", "7", "--tag", "a", "--tag", "b"]) + assert result.exit_code == 0, result.stderr + assert sdk.start_run.call_args.args[0] == {"trigger": { + "detection_id": "d", "category": "encoded-ps", "severity": "high", "priority": 7, "tags": ["a", "b"]}} + + def test_start_urgency_is_the_engines_set(self, sdk): + sdk.start_run.return_value = {"run": {"run_id": "r1"}} + for urgency in ("low", "medium", "high", "critical"): + assert run(["investigate", "runs", "start", "--template", "a", "--detection-id", "d", "--urgency", urgency]).exit_code == 0 + assert sdk.start_run.call_args.args[0]["urgency"] == urgency + sdk.start_run.reset_mock() + assert run(["investigate", "runs", "start", "--template", "a", "--detection-id", "d", "--urgency", "normal"]).exit_code != 0 + sdk.start_run.assert_not_called() + + def test_start_help_describes_auto_and_urgency_as_the_engine_does(self): + out = " ".join(CliRunner().invoke(cli, ["investigate", "runs", "start", "--help"]).output.split()) + assert "low|medium|high|critical" in out + assert "normal" not in out + assert "applies_to" in out and "sends no plan" in out + def test_start_usage_errors(self, sdk): assert run(["investigate", "runs", "start", "--detection-id", "d"]).exit_code != 0 assert run(["investigate", "runs", "start", "--template", "a", "--auto", "--detection-id", "d"]).exit_code != 0 @@ -339,6 +363,21 @@ def test_list_pages_with_a_cursor_envelope(self, sdk): assert kwargs["state"] == "running" and kwargs["limit"] == 10 and kwargs["cursor"] == "c1" assert kwargs["since"] == "2026-11-01T00:00:00Z" and kwargs["verdict"] is None + def test_list_filters_pass_the_engines_values_through(self, sdk): + sdk.list_runs.return_value = {"runs": [], "next_cursor": ""} + run(["investigate", "runs", "list", "--verdict", "undecided", "--trigger", "detection", "--started-by", "a@x.io"]) + kwargs = sdk.list_runs.call_args.kwargs + assert (kwargs["verdict"], kwargs["trigger"], kwargs["started_by"]) == ("undecided", "detection", "a@x.io") + + def test_list_help_names_the_engines_trigger_kinds(self): + out = " ".join(CliRunner().invoke(cli, ["investigate", "runs", "list", "--help"]).output.split()) + assert "detection, message, entity, case, question" in out and "undecided" in out + assert "dr-rule" not in out + + def test_strict_help_says_it_refuses_loosening_overrides(self): + out = " ".join(CliRunner().invoke(cli, ["investigate", "templates", "test", "--help"]).output.split()) + assert "loosen" in out and "unknown as a failure" not in out + def test_list_table_rows_and_cursor_on_stderr(self, sdk): sdk.list_runs.return_value = {"runs": [{ "run_id": "r1", "state": "completed", "template": "p", "verdict": {"label": "false_positive", "p": 0.02}, @@ -471,9 +510,10 @@ def test_approve_and_deny(self, sdk): def test_policy_commands(self, sdk): sdk.get_effective_policy.return_value = {} - run(["investigate", "policy", "get", "--category", "c", "--priority", "4", "--target-tag", "a", "--target-tag", "b", "--class", "tag"]) + run(["investigate", "policy", "get", "--category", "c", "--priority", "4", "--target-tag", "a", "--target-tag", "b", "--class", "tag", + "--template", "t"]) sdk.get_effective_policy.assert_called_with( - category="c", priority=4, severity=None, target_tags=["a", "b"], action_class="tag") + category="c", priority=4, severity=None, target_tags=["a", "b"], action_class="tag", template="t") run(["investigate", "policy", "status"]) sdk.get_autonomy.assert_called_once() run(["investigate", "policy", "pause"]) diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index c80a061e..6576b36f 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -87,6 +87,11 @@ def test_effective_policy_sends_class_not_class_underscore(self, inv, mock_org): query = dict(_request_args(mock_org)[2]["query_params"]) assert query == {"category": "c", "priority": "3", "target_tags": "a,b", "class": "contain.host"} + def test_effective_policy_sends_the_template(self, inv, mock_org): + mock_org.client.request.return_value = {} + inv.get_effective_policy(template="t") + assert dict(_request_args(mock_org)[2]["query_params"]) == {"template": "t"} + def test_list_runs_sends_only_set_filters(self, inv, mock_org): mock_org.client.request.return_value = {"runs": []} inv.list_runs(state="running", limit=20, cursor="cur") From 178a9fdc1c75e6b2aad5c4054e5335f54b8eb3a6 Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 17:28:44 -0800 Subject: [PATCH 08/14] Investigation CLI: tests and docs for report.summary sentences of both shapes Engine v0.1.4 lets a template's report.summary sentence be a string or {text, when}. The SDK and CLI treat a template as an opaque mapping, so no code change is needed; tests now pin that save, get, install-defaults and the YAML round trip keep both shapes unchanged. Co-Authored-By: Claude Sonnet 5.5 --- doc/cli/investigation-engine.md | 1 + tests/unit/test_sdk_investigation.py | 47 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md index f51376bc..41cc56ea 100644 --- a/doc/cli/investigation-engine.md +++ b/doc/cli/investigation-engine.md @@ -59,6 +59,7 @@ Rules the CLI enforces: - A record is never overwritten without `--overwrite`, and a draft **never** replaces an active template (save under a new name; a person decides whether it replaces the active one). - A draft runs only through `templates test` or a run a person starts, with response actions capped at `suggest`. A run started by a D&R rule or a case never uses a draft. - Nothing is saved when validation reports an error. +- The CLI never rewrites a template's content beyond `status`: a `report.summary` sentence can be a plain string or `{text, when}` (written only when the condition holds), and both are stored and printed as written. ## Reference data diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index 6576b36f..18b89e57 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -51,6 +51,16 @@ def _not_found(): VALID = {"valid": True, "errors": [], "warnings": [{"code": "unused_fact", "path": "facts.x", "message": "m"}]} INVALID = {"valid": False, "errors": [{"code": "unknown_field", "path": "facts.a", "message": "m", "suggestion": "b"}], "warnings": []} +SUMMARY = { + "true_positive": ["The message is a phish.", {"text": "The link {link.url} is known bad.", "when": "link.known_bad"}], + "false_positive": [{"text": "Long history.", "when": "history.days > 30"}], +} +SUMMARY_SRC = ( + "description: test plan\nreport:\n summary:\n true_positive:\n - The message is a phish.\n" + " - text: The link {link.url} is known bad.\n when: link.known_bad\n" + " false_positive:\n - text: Long history.\n when: history.days > 30\n" +) + SRC = "description: test plan\nfacts:\n a:\n op: entity.get\n args: {entity_id: $entity}\n" @@ -356,6 +366,16 @@ def test_dump_keeps_key_order(self): text = dump_template({"zebra": 1, "alpha": 2}) assert text.index("zebra") < text.index("alpha") + def test_report_summary_sentences_of_both_shapes_round_trip(self): + # Engine v0.1.4: a sentence is a string or {text, when}; the SDK treats the template as an opaque mapping. + data = {"report": {"summary": { + "true_positive": ["Plain: {a.b} seen.", {"text": "Only on a hit: {a.b}", "when": "a.b == true and not c.d"}], + "false_positive": [{"text": "Plain object", "when": ""}, "yes"], + }}} + text = dump_template(data) + assert parse_template_source(text) == data + assert parse_template_source(SUMMARY_SRC)["report"]["summary"] == SUMMARY + @pytest.mark.parametrize("src,match", [ ("", "empty"), ("- a\n- b\n", "mapping"), @@ -522,6 +542,33 @@ def test_a_write_that_failed_and_did_not_land_is_an_error(self, inv, mock_org, h inv.save_template("my-plan", SRC) +class TestSummarySentences: + def test_get_template_returns_both_shapes_in_the_record_and_the_yaml(self, inv, hive): + hive.get.return_value = HiveRecord(name="p", data={"description": "d", "report": {"summary": SUMMARY}}, etag="e") + view = inv.get_template("p") + assert view["data"]["report"]["summary"] == SUMMARY + assert parse_template_source(view["yaml"])["report"]["summary"] == SUMMARY + + def test_save_validates_and_stores_the_sentences_unchanged(self, inv, mock_org, hive): + mock_org.client.request.return_value = VALID + hive.get.side_effect = [_not_found(), HiveRecord(name="p", data={"status": "draft"}, etag="e")] + assert inv.save_template("p", SUMMARY_SRC)["saved"] is True + sent = json.loads(mock_org.client.request.call_args.kwargs["raw_body"])["yaml"] + assert parse_template_source(sent)["report"]["summary"] == SUMMARY + assert hive.set.call_args.args[0].data["report"]["summary"] == SUMMARY + + def test_install_defaults_writes_a_library_template_with_both_shapes(self, inv, mock_org, hive): + def request(verb, path, **kw): + if path.endswith("authoring-kit"): + return {"library": [{"name": "alpha", "version": 3, "status": "active", "yaml": SUMMARY_SRC}]} + return VALID + mock_org.client.request.side_effect = request + hive.get.side_effect = [_not_found(), HiveRecord(name="alpha", data={"status": "draft"}, etag="e")] + out = inv.install_default_templates() + assert [w["name"] for w in out["written"]] == ["alpha"] + assert hive.set.call_args.args[0].data["report"]["summary"] == SUMMARY + + class TestTemplateRecordPermissions: def test_a_refused_hive_operation_names_the_investigation_plan_permissions(self, inv, hive): hive.get.side_effect = error_from_status_code(400, {"error": "lc_error_code:UNAUTHORIZED - nope"}) From aef84c4f92748766aa7dab6e810c3d2c7b9a2094 Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 18:18:38 -0800 Subject: [PATCH 09/14] Investigation CLI: read and write scalars exactly as the engine's YAML parser does Unquoted scalars under a fact's args follow gopkg.in/yaml.v3 (012, 1_000, 0o17, 1e3, timestamps), other template fields keep the author's text except the numeric and boolean ones, mapping keys are text. A fixture generated from the engine's own parser pins every edge spelling. The template dumper quotes every string the engine would read as something else, and save/promote validate text proven to read back as the stored record. NaN/Infinity are refused as invalid_input. The docs state who a run executes as when the policy names a secret identity. Co-Authored-By: Claude Sonnet 5.5 --- doc/cli/investigation-engine.md | 10 +- limacharlie/commands/investigate.py | 11 +- limacharlie/sdk/investigation.py | 420 ++++++- .../unit/fixtures/investigation_scalars.json | 1089 +++++++++++++++++ tests/unit/test_cli_investigate.py | 17 + tests/unit/test_sdk_investigation.py | 154 ++- 6 files changed, 1630 insertions(+), 71 deletions(-) create mode 100644 tests/unit/fixtures/investigation_scalars.json diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md index 41cc56ea..e449c54d 100644 --- a/doc/cli/investigation-engine.md +++ b/doc/cli/investigation-engine.md @@ -22,7 +22,7 @@ All the listed permissions are required. | `policy pause` / `policy resume` | `org.conf.set` | | Template records (`investigation_plan` hive) | `investigation_plan.get`, `investigation_plan.set`, `investigation_plan.del` (explicit grants; no default role has them) | -Runs, waves and tests execute with **your own token**, so the operators enforce your permissions. +Waves and template tests always execute with **your own token**, so the operators enforce your permissions. A run started with `runs start` executes as the identity the organization's `investigation_policy` names for its template: your token when the policy names none, otherwise the Hive secret the policy points to, in which case the operators enforce that secret's permissions and not yours. An active template can therefore reach data or take actions beyond your own permissions. Before you start one, check `limacharlie investigate policy get --template NAME`: its `identity` shows the default identity, the allowed ones and where they come from. A finished run's `identity` field (`kind` caller or secret) records which one it used. Runs started by a detection rule or a case never have a caller and always use the policy's secret identity. ## AI agent workflow @@ -195,7 +195,13 @@ In a structured output format (`--output json`, the default when piped) a failur Every other failure is written the same way: a Hive or API error carries its `message`, HTTP `status` and the backend's code as `error_code` (the `lc_error_code` of a Hive answer, for example a refused promote), and a refused input file (unreadable, not YAML/JSON, wrong shape) carries `errors: [{code: invalid_input, path: --file, message}]` and exits `4`. -Wave files, sample files and templates are read with YAML 1.2 scalar rules, so `off`, `yes`, `no` and an unquoted date such as `2026-11-10` are sent as the text you wrote, not as a boolean or a date. +Templates, wave files and sample files are read the way the engine's YAML library reads them, not with YAML 1.1 rules, so `off`, `yes`, `no` and an unquoted date such as `2026-11-10` are text, not a boolean or a date. What an unquoted scalar means depends on where it is: + +- In a template, under a fact's `args`, a number is read as the engine reads it: `012` is 10, `1_000` is 1000, `0o17` and `0x1F` are 15 and 31, `1e3` is the float 1000, `.inf` and `.nan` are floats (refused, see below), and an unquoted date or timestamp is stored as its UTC RFC 3339 text (`2026-11-10` becomes `2026-11-10T00:00:00Z`). Quote a value (`"012"`, `"1e3"`) to keep it as text. +- Elsewhere in a template (`description`, `applies_to`, `inputs`, ...) a scalar is text exactly as written, except the numeric and boolean fields (`version`, `prior`, the weights, `predict`, `case` on a fact). Mapping keys are always text. +- In a wave or samples file every unquoted scalar is read like an argument, except that a date stays the text you wrote. + +`templates get --yaml` quotes every string that would otherwise read back as something else, and `templates save` / `promote` validate exactly the text they store. A value JSON cannot carry (`.nan`, `.inf`, `NaN`, `Infinity`) is refused as `invalid_input` before anything is sent. ## SDK diff --git a/limacharlie/commands/investigate.py b/limacharlie/commands/investigate.py index a9569e9b..b644175a 100644 --- a/limacharlie/commands/investigate.py +++ b/limacharlie/commands/investigate.py @@ -466,7 +466,7 @@ def _input_error(hint: str, message: str) -> InvestigationError: def _load_document(path: str, hint: str) -> Any: - """Read a YAML or JSON file; scalars keep the meaning they have in YAML 1.2 (``off`` is text).""" + """Read a YAML or JSON file; unquoted scalars are read as the engine reads them (``off`` is text, ``012`` is 10).""" try: with open(path, "r", encoding="utf-8") as f: content = f.read() @@ -499,12 +499,17 @@ def _parse_kv(pairs: tuple[str, ...], hint: str) -> dict[str, str]: return out +def _reject_constant(name: str) -> Any: + """``json.loads`` hook: Python reads NaN and Infinity, but JSON (and the engine) has no such numbers.""" + raise ValueError(f"{name} is not a JSON number") + + def _json_object(text: str | None, hint: str) -> dict[str, Any] | None: if text is None: return None try: - value = json.loads(text) - except json.JSONDecodeError as exc: + value = json.loads(text, parse_constant=_reject_constant) + except ValueError as exc: # JSONDecodeError is one; so is a refused NaN/Infinity raise _input_error(hint, f"invalid JSON: {exc}") from exc if not isinstance(value, dict): raise _input_error(hint, "must be a JSON object") diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index 76e75c88..53c74cae 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -33,6 +33,8 @@ from __future__ import annotations +import base64 +import datetime import functools import json import re @@ -248,69 +250,322 @@ def _check_wait(wait: int) -> int: return wait -class _TemplateLoader(yaml.SafeLoader): - """YAML loader for template sources. - - The engine reads templates with YAML 1.2 rules. PyYAML follows 1.1, which - would turn ``yes``/``no``/``on``/``off`` into booleans and an unquoted - date into a ``datetime``, so a template would be stored as something other - than what its author wrote. This loader keeps both as text and accepts - only ``true``/``false`` as booleans. Numbers follow the YAML 1.2 core - schema; one written the 1.1 way (``012``, ``1_000``, ``10:30``) stays text. - """ - - +_STR_TAG = "tag:yaml.org,2002:str" _BOOL_TAG = "tag:yaml.org,2002:bool" _INT_TAG = "tag:yaml.org,2002:int" _FLOAT_TAG = "tag:yaml.org,2002:float" +_NULL_TAG = "tag:yaml.org,2002:null" _TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" -_TemplateLoader.yaml_implicit_resolvers = { - ch: [(tag, rx) for tag, rx in resolvers if tag not in (_BOOL_TAG, _INT_TAG, _FLOAT_TAG, _TIMESTAMP_TAG)] - for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() -} -# YAML 1.2 core schema. A number written with a leading zero, an underscore or -# a colon (`012`, `1_000`, `10:30`) is text here, as the author wrote it, -# rather than the number YAML 1.1 reads it as. -_TemplateLoader.add_implicit_resolver( - _BOOL_TAG, re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$"), list("tTfF"), -) -_TemplateLoader.add_implicit_resolver( - _INT_TAG, re.compile(r"^(?:[-+]?(?:0|[1-9][0-9]*)|0o[0-7]+|0x[0-9a-fA-F]+)$"), list("-+0123456789"), -) -_TemplateLoader.add_implicit_resolver( - _FLOAT_TAG, - re.compile(r"^(?:[-+]?(?:[0-9]+\.[0-9]*|\.[0-9]+|[0-9]+(?=[eE]))(?:[eE][-+]?[0-9]+)?|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$"), - list("-+0123456789."), +_BINARY_TAG = "tag:yaml.org,2002:binary" +_MERGE_TAG = "tag:yaml.org,2002:merge" +# Tag of an unquoted, untagged scalar until its context says what it is. +_PLAIN_TAG = "tag:limacharlie.io,2026:plain" + +_INT64_MIN, _INT64_MAX, _UINT64_MAX = -(1 << 63), (1 << 63) - 1, (1 << 64) - 1 +_YAML_STYLE_FLOAT_RE = re.compile(r"^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$") +_DOT_FLOAT_RE = re.compile(r"^\.[0-9]+([eE][-+]?[0-9]+)?$") +_TS_DATE = r"([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})" +_TS_CLOCK = r"([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})(?:\.([0-9]+))?" +# The four layouts yaml.v3 tries, in the order it tries them. +_TIMESTAMP_RES = ( + re.compile(rf"^{_TS_DATE}[Tt]{_TS_CLOCK}(Z|[+-][0-9]{{2}}:[0-9]{{2}})$"), + re.compile(rf"^{_TS_DATE} {_TS_CLOCK}()$"), + re.compile(rf"^{_TS_DATE}()()()()()$"), ) +_RESOLVE_MAP: dict[str, tuple[str, Any]] = {} +for _words, _tag, _val in ( + (("true", "True", "TRUE"), _BOOL_TAG, True), + (("false", "False", "FALSE"), _BOOL_TAG, False), + (("", "~", "null", "Null", "NULL"), _NULL_TAG, None), + ((".nan", ".NaN", ".NAN"), _FLOAT_TAG, float("nan")), + ((".inf", ".Inf", ".INF", "+.inf", "+.Inf", "+.INF"), _FLOAT_TAG, float("inf")), + (("-.inf", "-.Inf", "-.INF"), _FLOAT_TAG, float("-inf")), +): + for _w in _words: + _RESOLVE_MAP[_w] = (_tag, _val) +_BOOL_WORDS_1_1 = { + "y": True, "Y": True, "yes": True, "Yes": True, "YES": True, "on": True, "On": True, "ON": True, + "n": False, "N": False, "no": False, "No": False, "NO": False, "off": False, "Off": False, "OFF": False, +} + + +def _go_parse_uint(s: str) -> int | None: + """Go ``strconv.ParseUint(s, 0, 64)`` (without the underscore rules: callers strip them).""" + if not s: + return None + base = 10 + if s[0] == "0": + if len(s) >= 3 and s[1] in "bB": + base, s = 2, s[2:] + elif len(s) >= 3 and s[1] in "oO": + base, s = 8, s[2:] + elif len(s) >= 3 and s[1] in "xX": + base, s = 16, s[2:] + else: + base, s = 8, s[1:] + n = 0 + for ch in s: + d = "0123456789abcdef".find(ch.lower()) if ch.isascii() else -1 + if d < 0 or d >= base: + return None + n = n * base + d + if n > _UINT64_MAX: + return None + return n + + +def _go_parse_int(s: str) -> int | None: + """Go ``strconv.ParseInt(s, 0, 64)``.""" + neg = s[:1] == "-" + if s[:1] in ("+", "-"): + s = s[1:] + n = _go_parse_uint(s) + if n is None: + return None + n = -n if neg else n + return n if _INT64_MIN <= n <= _INT64_MAX else None + + +def _go_float(s: str) -> float | None: + """The float for a decimal spelling Go's ParseFloat accepts; None when it overflows.""" + value = float(s) + return None if value in (float("inf"), float("-inf")) else value + + +def _parse_timestamp(s: str) -> str | None: + """A timestamp as yaml.v3 reads it, rendered as the UTC RFC 3339 text the engine would store. + + Only the layouts yaml.v3 knows are timestamps: ``2006-1-2``, + ``2006-1-2T15:4:5.999Z07:00`` (a lower-case ``t`` too) and + ``2006-1-2 15:4:5.999`` with no zone. Anything Go's time parser would + reject (month 13, February 30th, hour 25) is text. + """ + for rx in _TIMESTAMP_RES: + m = rx.match(s) + if m: + break + else: + return None + year, month, day = int(m[1]), int(m[2]), int(m[3]) + hour, minute, second = (int(m[4]), int(m[5]), int(m[6])) if m[4] else (0, 0, 0) + shift = 400 if year < 400 else 0 # datetime has no year 0; the calendar repeats every 400 years + try: + local = datetime.datetime(year + shift, month, day, hour, minute, second) + except ValueError: + return None + zone = m[8] or "Z" + offset = 0 + if zone != "Z": + if int(zone[1:3]) > 24 or int(zone[4:6]) > 59: + return None + offset = (int(zone[1:3]) * 60 + int(zone[4:6])) * (1 if zone[0] == "+" else -1) + try: + utc = local - datetime.timedelta(minutes=offset) + except OverflowError: + return None + frac = (m[7] or "")[:9].rstrip("0") + if utc.year - shift < 0: + return None + stamp = f"{utc.year - shift:04d}-{utc.month:02d}-{utc.day:02d}T{utc.hour:02d}:{utc.minute:02d}:{utc.second:02d}" + return stamp + (f".{frac}" if frac else "") + "Z" + + +def _resolve_scalar(text: str, tag: str = "") -> tuple[str, Any]: + """Resolve a scalar the way the engine's YAML library (gopkg.in/yaml.v3) does. + + This is a port of its ``resolve`` and is checked against the engine's own + parser by ``tests/unit/fixtures/investigation_scalars.json``. ``tag`` is + ``""`` for a plain scalar or an explicit ``!!int``-style tag. Returns + ``(tag, value)``; a timestamp is returned as its UTC RFC 3339 text, the + form the engine stores it in. + + Raises: + ValueError: If an explicit tag does not fit the text. + """ + kind, value = _natural_scalar(text, tag) + if tag in ("", _STR_TAG, _BINARY_TAG) or kind == tag: + return kind, value + if tag == _FLOAT_TAG and kind == _INT_TAG: + return _FLOAT_TAG, float(value) + raise ValueError(f"cannot decode {kind.rsplit(':', 1)[-1]} `{text}` as a {tag.rsplit(':', 1)[-1]}") + + +def _natural_scalar(text: str, tag: str) -> tuple[str, Any]: + if tag in (_STR_TAG, _BINARY_TAG): + return _STR_TAG, text + first = text[:1] + if first == "": + hint = "N" + elif first in "+-": + hint = "S" + elif first in "0123456789": + hint = "D" + elif first in "yYnNtTfFoO~": + hint = "M" + elif first == ".": + hint = "." + else: + return _STR_TAG, text + if text in _RESOLVE_MAP: + return _RESOLVE_MAP[text] + if hint == ".": + if _DOT_FLOAT_RE.match(text): + f = _go_float(text) + if f is not None: + return _FLOAT_TAG, f + elif hint in ("D", "S"): + if tag in ("", _TIMESTAMP_TAG): + ts = _parse_timestamp(text) + if ts is not None: + return _TIMESTAMP_TAG, ts + plain = text.replace("_", "") + n = _go_parse_int(plain) + if n is None and plain[0] not in "+-": + n = _go_parse_uint(plain) + if n is not None: + return _INT_TAG, n + if _YAML_STYLE_FLOAT_RE.match(plain): + f = _go_float(plain) + if f is not None: + return _FLOAT_TAG, f + return _STR_TAG, text -def _construct_core_int(loader: yaml.SafeLoader, node: yaml.Node) -> int: - value = str(loader.construct_scalar(node)) - if value[:2] == "0o": - return int(value[2:], 8) - return int(value, 0) if value[:2] == "0x" else int(value) +class _TemplateLoader(yaml.SafeLoader): + """YAML loader for template sources and other documents sent to the engine. + + The engine reads YAML with gopkg.in/yaml.v3, not with PyYAML's YAML 1.1 + rules, and what a scalar means depends on where it sits: + + * Under a fact's ``args`` (and in any document that is not a template) an + unquoted scalar is resolved by yaml.v3's rules: ``yes``/``off`` are text, + ``012`` is 10, ``1_000`` is 1000, ``0o17`` is 15, ``1e3`` is a float, a + date is a timestamp (kept as its UTC RFC 3339 text, which is what the + stored record holds). + * Everywhere else the engine reads into text fields, so the author's spelling + is kept: ``description: 1e3`` stays ``"1e3"``. A few fields are numbers or + booleans (``version``, ``prior``, weights, ``predict``, ``case``) and + are read as such. + + A quoted scalar is always text. A mapping key is always text. + """ + + def resolve(self, kind: Any, value: Any, implicit: Any) -> str: + if kind is yaml.ScalarNode and implicit[0]: + return _PLAIN_TAG + return super().resolve(kind, value, implicit) + + +def _construct_resolved(tag: str) -> Any: + def construct(loader: yaml.SafeLoader, node: yaml.Node) -> Any: + text = str(loader.construct_scalar(node)) + try: + return _resolve_scalar(text, tag)[1] + except ValueError as exc: + raise yaml.constructor.ConstructorError(None, None, str(exc), node.start_mark) from exc + return construct -_TemplateLoader.add_constructor(_INT_TAG, _construct_core_int) +def _construct_binary(loader: yaml.SafeLoader, node: yaml.Node) -> str: + try: + return base64.b64decode(str(loader.construct_scalar(node)), validate=False).decode("utf-8") + except (ValueError, UnicodeDecodeError) as exc: + raise yaml.constructor.ConstructorError(None, None, f"cannot read !!binary as text: {exc}", node.start_mark) from exc + + +for _t in (_BOOL_TAG, _INT_TAG, _FLOAT_TAG, _NULL_TAG, _TIMESTAMP_TAG): + _TemplateLoader.add_constructor(_t, _construct_resolved(_t)) +_TemplateLoader.add_constructor(_PLAIN_TAG, _construct_resolved(_STR_TAG)) +_TemplateLoader.add_constructor(_BINARY_TAG, _construct_binary) + +# Template fields the engine reads as numbers or booleans, by path ("*" is any +# list index or map key). Every other scalar outside a fact's args is text. +_NUMBER_PATHS = { + ("version",), ("prior",), ("evidence", "*", "weight"), + ("signals", "*", "malicious_weight"), ("signals", "*", "benign_weight"), +} +_BOOL_PATHS = {("facts", "*", "predict"), ("facts", "*", "case")} + + +def _path_matches(path: tuple[str, ...], patterns: set[tuple[str, ...]]) -> bool: + return any( + len(p) == len(path) and all(a == "*" or a == b for a, b in zip(p, path)) for p in patterns + ) -def _load_yaml(source: str, allow_aliases: bool) -> Any: - """Parse YAML with the engine's YAML 1.2 scalar rules; raise ``yaml.YAMLError``.""" +def _retag(node: yaml.Node, path: tuple[str, ...], any_ctx: bool, seen: set[int], document: bool = False) -> None: + """Decide what each unquoted scalar of the document is, from where it sits. + + ``document`` is a file the CLI turns into JSON itself (a wave, samples): + the engine never reads it as YAML, so a date keeps the author's spelling. + """ + if id(node) in seen: + return + seen.add(id(node)) + if isinstance(node, yaml.ScalarNode): + if node.tag != _PLAIN_TAG: + return + if any_ctx: + node.tag = _resolve_scalar(node.value)[0] + if document and node.tag == _TIMESTAMP_TAG: + node.tag = _STR_TAG + elif _path_matches(path, _NUMBER_PATHS): + tag = _resolve_scalar(node.value)[0] + node.tag = tag if tag in (_INT_TAG, _FLOAT_TAG) else _STR_TAG + elif _path_matches(path, _BOOL_PATHS): + if node.value in _BOOL_WORDS_1_1: + node.tag, node.value = _BOOL_TAG, "true" if _BOOL_WORDS_1_1[node.value] else "false" + else: + tag = _resolve_scalar(node.value)[0] + node.tag = tag if tag == _BOOL_TAG else _STR_TAG + else: + node.tag = _STR_TAG + return + if isinstance(node, yaml.SequenceNode): + for item in node.value: + _retag(item, path + ("*",), any_ctx, seen, document) + return + if isinstance(node, yaml.MappingNode): + for key, value in node.value: + if isinstance(key, yaml.ScalarNode) and key.tag == _PLAIN_TAG: + key.tag = _MERGE_TAG if key.value == "<<" else _STR_TAG + name = key.value if isinstance(key, yaml.ScalarNode) else "*" + child_any = any_ctx or (len(path) == 2 and path[0] == "facts" and name == "args") + _retag(value, path + (name,), child_any, seen, document) + + +def _load_yaml(source: str, allow_aliases: bool, document: bool) -> Any: + """Parse YAML with the engine's scalar rules; raise ``yaml.YAMLError``.""" if not allow_aliases: for event in yaml.parse(source, Loader=_TemplateLoader): if isinstance(event, yaml.AliasEvent): raise ValidationError( "YAML anchors and aliases are not supported in a template; write the value out" ) - return yaml.load(source, Loader=_TemplateLoader) # noqa: S506 - SafeLoader subclass + loader = _TemplateLoader(source) + try: + node = loader.get_single_node() + if node is None: + return None + _retag(node, (), document, set(), document) + return loader.construct_document(node) + finally: + loader.dispose() + + +def _check_json_carries(doc: Any, what: str) -> None: + try: + json.dumps(doc, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValidationError(f"{what} holds a value JSON cannot carry: {exc}") from exc def parse_document(source: str, what: str = "the document") -> Any: """Parse a YAML or JSON document into plain JSON-compatible values. - Scalars follow the same YAML 1.2 rules as a template: ``off``/``yes``/``no`` - stay text and an unquoted date stays a string, so every value is sent to - the engine as it was written. + Unquoted scalars are read the way the engine's YAML library reads a value + it holds as ``any``: ``off``/``yes``/``no`` stay text, ``012`` is 10, + ``1_000`` is 1000 and ``0o17`` is 15. A date stays the text it was written as. Args: source: The YAML or JSON text. @@ -321,21 +576,18 @@ def parse_document(source: str, what: str = "the document") -> Any: Raises: ValidationError: If the text is not YAML or JSON, or holds a value - JSON cannot carry (a binary or set tag, for example). + JSON cannot carry (a set tag, ``.nan`` or ``.inf``, for example). """ if not isinstance(source, str) or not source.strip(): raise ValidationError(f"{what} is empty") try: - doc = _load_yaml(source, allow_aliases=True) + doc = _load_yaml(source, allow_aliases=True, document=True) except yaml.YAMLError as exc: try: doc = json.loads(source) except ValueError: raise ValidationError(f"{what} is neither valid YAML nor JSON: {exc}") from exc - try: - json.dumps(doc) - except (TypeError, ValueError) as exc: - raise ValidationError(f"{what} holds a value JSON cannot carry: {exc}") from exc + _check_json_carries(doc, what) return doc @@ -349,30 +601,87 @@ def parse_template_source(source: str) -> dict[str, Any]: dict: The template as plain JSON-compatible values. Raises: - ValidationError: If the source is empty, not YAML, not a mapping, or - uses YAML anchors or aliases (a template has no use for them). + ValidationError: If the source is empty, not YAML, not a mapping, + uses YAML anchors or aliases (a template has no use for them) or + holds a value JSON cannot carry (``.nan``, ``.inf``). """ if not isinstance(source, str) or not source.strip(): raise ValidationError("the template is empty") try: - doc = _load_yaml(source, allow_aliases=False) + doc = _load_yaml(source, allow_aliases=False, document=False) except yaml.YAMLError as exc: raise ValidationError(f"the template is not valid YAML: {exc}") from exc if not isinstance(doc, dict): raise ValidationError("the template must be a YAML mapping (description, applies_to, facts, ...)") + _check_json_carries(doc, "the template") return doc +# A string the engine's parser could read as something else when it is not +# quoted: numbers (any base, with underscores, signs or exponents), dates, +# ``.inf``/``.nan``, and the null, boolean and YAML 1.1 boolean words. A superset +# of what yaml.v3 resolves: quoting a string that did not need it changes nothing. +_NEEDS_QUOTES_RE = re.compile( + r"^(?:[-+]?\.?[0-9]\S*$|[-+]?\.(?:inf|nan)|(?:~|null|true|false|y|n|yes|no|on|off|<<)$)", + re.IGNORECASE, +) + + +class _TemplateDumper(yaml.SafeDumper): + """SafeDumper that quotes every string the engine could read as a non-string. + + PyYAML quotes only what YAML 1.1 reads as a non-string. ``"1e3"`` and + ``"0o17"`` are text there but numbers to the engine, so they are quoted + here. Every other value is written as ``yaml.safe_dump`` writes it. + """ + + def ignore_aliases(self, data: Any) -> bool: + return True + + +def _represent_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode: + quote = bool(_NEEDS_QUOTES_RE.match(data)) or _resolve_scalar(data)[0] != _STR_TAG + return dumper.represent_scalar(_STR_TAG, data, style='"' if quote else None) + + +_TemplateDumper.add_representer(str, _represent_str) + + def dump_template(data: dict[str, Any]) -> str: """Render a template mapping as YAML, in the key order it was written. + Every string reads back as the same string in the engine, whatever it + looks like (``"1e3"``, ``"0o17"``, ``"on"``, ``"2026-11-10"`` are quoted). + Args: data: The template mapping. Returns: str: YAML text the engine accepts. """ - return yaml.safe_dump(data, sort_keys=False, default_flow_style=False, allow_unicode=True) + return yaml.dump( + data, Dumper=_TemplateDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, + ) + + +def _rendered_template(name: str, data: dict[str, Any]) -> str: + """The YAML to validate for a template record, proven to be exactly that record. + + Validation sees the rendered text, the hive keeps ``data``. If reading the + text back does not give ``data`` (a number in a text field, for example) + the engine would be validating a different template than the one stored. + """ + text = dump_template(data) + try: + same = json.dumps(parse_template_source(text)) == json.dumps(data) + except ValidationError: + same = False + if not same: + raise ValidationError( + f"template {name!r} cannot be rendered as YAML that reads back identically (a number where the " + "engine reads text, or a value YAML cannot spell); fix the record before validating it" + ) + return text class Investigation: @@ -407,7 +716,10 @@ def _call( """ kwargs: dict[str, Any] = {"query_params": query or None, "timeout": timeout} if body is not None: - kwargs["raw_body"] = json.dumps(body).encode() + try: + kwargs["raw_body"] = json.dumps(body, allow_nan=False).encode() + except ValueError as exc: + raise ValidationError(f"the request holds a value JSON cannot carry: {exc}") from exc kwargs["content_type"] = "application/json" if write: kwargs["retry_gateway_timeout"] = False @@ -1084,7 +1396,7 @@ def save_template( final_status = STATUS_ACTIVE if explicit_active else STATUS_DRAFT data["status"] = final_status - report = self.validate_plan(dump_template(data)) + report = self.validate_plan(_rendered_template(name, data)) if report.get("valid") is not True or report.get("errors"): return { "saved": False, @@ -1191,7 +1503,7 @@ def promote_template(self, name: str) -> dict[str, Any]: if (data.get("status") or STATUS_DRAFT) == STATUS_ACTIVE: return {"promoted": False, "name": name, "status": STATUS_ACTIVE, "reason": "the template is already active"} data["status"] = STATUS_ACTIVE - report = self.validate_plan(dump_template(data)) + report = self.validate_plan(_rendered_template(name, data)) if report.get("valid") is not True or report.get("errors"): return {"promoted": False, "name": name, "status": STATUS_DRAFT, "reason": "the template does not validate", "validation": report} diff --git a/tests/unit/fixtures/investigation_scalars.json b/tests/unit/fixtures/investigation_scalars.json new file mode 100644 index 00000000..110c98a3 --- /dev/null +++ b/tests/unit/fixtures/investigation_scalars.json @@ -0,0 +1,1089 @@ +{ + "about": "How the Investigation Engine's template parser (gopkg.in/yaml.v3 v3.0.1, Template parse) reads each scalar spelling given as an operator argument (facts..args.value). Generated by a small Go program that feeds every spelling to the engine's own parser; do not edit by hand. kind is the Go type the argument got: null, bool, int, float, string, timestamp (value is UTC RFC 3339), or error. Float values: nan, +inf, -inf or the %g text.", + "cases": [ + { + "src": "012", + "kind": "int", + "value": "10" + }, + { + "src": "0o12", + "kind": "int", + "value": "10" + }, + { + "src": "0O12", + "kind": "int", + "value": "10" + }, + { + "src": "0b101", + "kind": "int", + "value": "5" + }, + { + "src": "0B101", + "kind": "int", + "value": "5" + }, + { + "src": "0x1F", + "kind": "int", + "value": "31" + }, + { + "src": "0X1f", + "kind": "int", + "value": "31" + }, + { + "src": "1_000", + "kind": "int", + "value": "1000" + }, + { + "src": "1__0", + "kind": "int", + "value": "10" + }, + { + "src": "_1", + "kind": "string", + "value": "_1" + }, + { + "src": "1_", + "kind": "int", + "value": "1" + }, + { + "src": "0_7", + "kind": "int", + "value": "7" + }, + { + "src": "0x1_F", + "kind": "int", + "value": "31" + }, + { + "src": "0b1_0", + "kind": "int", + "value": "2" + }, + { + "src": "0o1_7", + "kind": "int", + "value": "15" + }, + { + "src": "1e3", + "kind": "float", + "value": "1000" + }, + { + "src": "1E3", + "kind": "float", + "value": "1000" + }, + { + "src": "1e+3", + "kind": "float", + "value": "1000" + }, + { + "src": "1E+3", + "kind": "float", + "value": "1000" + }, + { + "src": "1e-3", + "kind": "float", + "value": "0.001" + }, + { + "src": "1.5e3", + "kind": "float", + "value": "1500" + }, + { + "src": "1.e3", + "kind": "float", + "value": "1000" + }, + { + "src": ".5", + "kind": "float", + "value": "0.5" + }, + { + "src": "5.", + "kind": "float", + "value": "5" + }, + { + "src": "+1", + "kind": "int", + "value": "1" + }, + { + "src": "-1", + "kind": "int", + "value": "-1" + }, + { + "src": "+.5", + "kind": "float", + "value": "0.5" + }, + { + "src": "-.5", + "kind": "float", + "value": "-0.5" + }, + { + "src": "+5.", + "kind": "float", + "value": "5" + }, + { + "src": "1e", + "kind": "string", + "value": "1e" + }, + { + "src": "1e+", + "kind": "string", + "value": "1e+" + }, + { + "src": "e3", + "kind": "string", + "value": "e3" + }, + { + "src": ".e3", + "kind": "string", + "value": ".e3" + }, + { + "src": ".inf", + "kind": "float", + "value": "+inf" + }, + { + "src": "-.inf", + "kind": "float", + "value": "-inf" + }, + { + "src": "+.inf", + "kind": "float", + "value": "+inf" + }, + { + "src": ".Inf", + "kind": "float", + "value": "+inf" + }, + { + "src": "-.Inf", + "kind": "float", + "value": "-inf" + }, + { + "src": "+.Inf", + "kind": "float", + "value": "+inf" + }, + { + "src": ".INF", + "kind": "float", + "value": "+inf" + }, + { + "src": "-.INF", + "kind": "float", + "value": "-inf" + }, + { + "src": ".nan", + "kind": "float", + "value": "nan" + }, + { + "src": ".NaN", + "kind": "float", + "value": "nan" + }, + { + "src": ".NAN", + "kind": "float", + "value": "nan" + }, + { + "src": ".nAn", + "kind": "string", + "value": ".nAn" + }, + { + "src": "inf", + "kind": "string", + "value": "inf" + }, + { + "src": "-inf", + "kind": "string", + "value": "-inf" + }, + { + "src": "+inf", + "kind": "string", + "value": "+inf" + }, + { + "src": "nan", + "kind": "string", + "value": "nan" + }, + { + "src": "NaN", + "kind": "string", + "value": "NaN" + }, + { + "src": "Infinity", + "kind": "string", + "value": "Infinity" + }, + { + "src": "-Infinity", + "kind": "string", + "value": "-Infinity" + }, + { + "src": "yes", + "kind": "string", + "value": "yes" + }, + { + "src": "no", + "kind": "string", + "value": "no" + }, + { + "src": "on", + "kind": "string", + "value": "on" + }, + { + "src": "off", + "kind": "string", + "value": "off" + }, + { + "src": "y", + "kind": "string", + "value": "y" + }, + { + "src": "n", + "kind": "string", + "value": "n" + }, + { + "src": "Y", + "kind": "string", + "value": "Y" + }, + { + "src": "N", + "kind": "string", + "value": "N" + }, + { + "src": "Yes", + "kind": "string", + "value": "Yes" + }, + { + "src": "No", + "kind": "string", + "value": "No" + }, + { + "src": "YES", + "kind": "string", + "value": "YES" + }, + { + "src": "NO", + "kind": "string", + "value": "NO" + }, + { + "src": "On", + "kind": "string", + "value": "On" + }, + { + "src": "Off", + "kind": "string", + "value": "Off" + }, + { + "src": "ON", + "kind": "string", + "value": "ON" + }, + { + "src": "OFF", + "kind": "string", + "value": "OFF" + }, + { + "src": "true", + "kind": "bool", + "value": "true" + }, + { + "src": "false", + "kind": "bool", + "value": "false" + }, + { + "src": "True", + "kind": "bool", + "value": "true" + }, + { + "src": "False", + "kind": "bool", + "value": "false" + }, + { + "src": "TRUE", + "kind": "bool", + "value": "true" + }, + { + "src": "FALSE", + "kind": "bool", + "value": "false" + }, + { + "src": "tRue", + "kind": "string", + "value": "tRue" + }, + { + "src": "null", + "kind": "null" + }, + { + "src": "Null", + "kind": "null" + }, + { + "src": "NULL", + "kind": "null" + }, + { + "src": "nUll", + "kind": "string", + "value": "nUll" + }, + { + "src": "~", + "kind": "null" + }, + { + "src": "10:30", + "kind": "string", + "value": "10:30" + }, + { + "src": "1:2:3", + "kind": "string", + "value": "1:2:3" + }, + { + "src": "01:02:03", + "kind": "string", + "value": "01:02:03" + }, + { + "src": "190:20:30", + "kind": "string", + "value": "190:20:30" + }, + { + "src": "0.1.2", + "kind": "string", + "value": "0.1.2" + }, + { + "src": "1,000", + "kind": "string", + "value": "1,000" + }, + { + "src": "0", + "kind": "int", + "value": "0" + }, + { + "src": "-0", + "kind": "int", + "value": "0" + }, + { + "src": "+0", + "kind": "int", + "value": "0" + }, + { + "src": "00", + "kind": "int", + "value": "0" + }, + { + "src": "007", + "kind": "int", + "value": "7" + }, + { + "src": "08", + "kind": "float", + "value": "8" + }, + { + "src": "09", + "kind": "float", + "value": "9" + }, + { + "src": "0.0", + "kind": "float", + "value": "0" + }, + { + "src": "-0.0", + "kind": "float", + "value": "-0" + }, + { + "src": "1.0", + "kind": "float", + "value": "1" + }, + { + "src": "100", + "kind": "int", + "value": "100" + }, + { + "src": "-100", + "kind": "int", + "value": "-100" + }, + { + "src": "+100", + "kind": "int", + "value": "100" + }, + { + "src": "9223372036854775807", + "kind": "int", + "value": "9223372036854775807" + }, + { + "src": "9223372036854775808", + "kind": "int", + "value": "9223372036854775808" + }, + { + "src": "18446744073709551615", + "kind": "int", + "value": "18446744073709551615" + }, + { + "src": "18446744073709551616", + "kind": "float", + "value": "1.8446744073709552e+19" + }, + { + "src": "-9223372036854775808", + "kind": "int", + "value": "-9223372036854775808" + }, + { + "src": "-9223372036854775809", + "kind": "float", + "value": "-9.223372036854776e+18" + }, + { + "src": "1e400", + "kind": "string", + "value": "1e400" + }, + { + "src": "-1e400", + "kind": "string", + "value": "-1e400" + }, + { + "src": "1e-400", + "kind": "float", + "value": "0" + }, + { + "src": "0x", + "kind": "string", + "value": "0x" + }, + { + "src": "0o", + "kind": "string", + "value": "0o" + }, + { + "src": "0b", + "kind": "string", + "value": "0b" + }, + { + "src": "0o8", + "kind": "string", + "value": "0o8" + }, + { + "src": "0b2", + "kind": "string", + "value": "0b2" + }, + { + "src": "0xG", + "kind": "string", + "value": "0xG" + }, + { + "src": "0x7FFFFFFFFFFFFFFF", + "kind": "int", + "value": "9223372036854775807" + }, + { + "src": "0xFFFFFFFFFFFFFFFF", + "kind": "int", + "value": "18446744073709551615" + }, + { + "src": "0x10000000000000000", + "kind": "string", + "value": "0x10000000000000000" + }, + { + "src": "0o777777777777777777777", + "kind": "int", + "value": "9223372036854775807" + }, + { + "src": "-0x1F", + "kind": "int", + "value": "-31" + }, + { + "src": "+0x1F", + "kind": "int", + "value": "31" + }, + { + "src": "+0o17", + "kind": "int", + "value": "15" + }, + { + "src": "-0o17", + "kind": "int", + "value": "-15" + }, + { + "src": "-0b11", + "kind": "int", + "value": "-3" + }, + { + "src": "1.7976931348623157e309", + "kind": "string", + "value": "1.7976931348623157e309" + }, + { + "src": "2026-11-10", + "kind": "timestamp", + "value": "2026-11-10T00:00:00Z" + }, + { + "src": "2026-1-1", + "kind": "timestamp", + "value": "2026-01-01T00:00:00Z" + }, + { + "src": "2026-11-10T10:00:00Z", + "kind": "timestamp", + "value": "2026-11-10T10:00:00Z" + }, + { + "src": "2026-11-10t10:00:00Z", + "kind": "timestamp", + "value": "2026-11-10T10:00:00Z" + }, + { + "src": "10:00:00", + "kind": "string", + "value": "10:00:00" + }, + { + "src": "2026-11-10T10:00:00.5-08:00", + "kind": "timestamp", + "value": "2026-11-10T18:00:00.5Z" + }, + { + "src": "2026-13-45", + "kind": "string", + "value": "2026-13-45" + }, + { + "src": "2026-11-10T25:00:00Z", + "kind": "string", + "value": "2026-11-10T25:00:00Z" + }, + { + "src": "=", + "kind": "string", + "value": "=" + }, + { + "src": "<<", + "kind": "string", + "value": "<<" + }, + { + "src": "-", + "kind": "error", + "error": "template oracle: yaml: line 5: block sequence entries are not allowed in this context" + }, + { + "src": "?", + "kind": "error", + "error": "template oracle: yaml: line 5: mapping keys are not allowed in this context" + }, + { + "src": "!", + "kind": "null" + }, + { + "src": "&", + "kind": "error", + "error": "template oracle: yaml: line 5: did not find expected alphabetic or numeric character" + }, + { + "src": "*", + "kind": "error", + "error": "template oracle: yaml: line 5: did not find expected alphabetic or numeric character" + }, + { + "src": "@", + "kind": "error", + "error": "template oracle: yaml: line 5: found character that cannot start any token" + }, + { + "src": "a", + "kind": "string", + "value": "a" + }, + { + "src": "abc", + "kind": "string", + "value": "abc" + }, + { + "src": "1a", + "kind": "string", + "value": "1a" + }, + { + "src": "a1", + "kind": "string", + "value": "a1" + }, + { + "src": "1.2.3", + "kind": "string", + "value": "1.2.3" + }, + { + "src": "1e3x", + "kind": "string", + "value": "1e3x" + }, + { + "src": "0e0", + "kind": "float", + "value": "0" + }, + { + "src": "00.5", + "kind": "float", + "value": "0.5" + }, + { + "src": "007.5", + "kind": "float", + "value": "7.5" + }, + { + "src": "1_0.5", + "kind": "float", + "value": "10.5" + }, + { + "src": "1.5_0", + "kind": "float", + "value": "1.5" + }, + { + "src": "-.inf.", + "kind": "string", + "value": "-.inf." + }, + { + "src": ".infinity", + "kind": "string", + "value": ".infinity" + }, + { + "src": "NaN.", + "kind": "string", + "value": "NaN." + }, + { + "src": "True1", + "kind": "string", + "value": "True1" + }, + { + "src": "y/n", + "kind": "string", + "value": "y/n" + }, + { + "src": "ok", + "kind": "string", + "value": "ok" + }, + { + "src": "~~", + "kind": "string", + "value": "~~" + }, + { + "src": "nulls", + "kind": "string", + "value": "nulls" + }, + { + "src": "Nullx", + "kind": "string", + "value": "Nullx" + }, + { + "src": "", + "kind": "null" + }, + { + "src": " ", + "kind": "null" + }, + { + "src": "\"1e3\"", + "kind": "string", + "value": "1e3" + }, + { + "src": "'1e3'", + "kind": "string", + "value": "1e3" + }, + { + "src": "\"yes\"", + "kind": "string", + "value": "yes" + }, + { + "src": "'0o17'", + "kind": "string", + "value": "0o17" + }, + { + "src": "\"012\"", + "kind": "string", + "value": "012" + }, + { + "src": "\"~\"", + "kind": "string", + "value": "~" + }, + { + "src": "\"\"", + "kind": "string" + }, + { + "src": "''", + "kind": "string" + }, + { + "src": "\"null\"", + "kind": "string", + "value": "null" + }, + { + "src": "!!str 1", + "kind": "string", + "value": "1" + }, + { + "src": "!!int 3", + "kind": "int", + "value": "3" + }, + { + "src": "!!int \"3\"", + "kind": "int", + "value": "3" + }, + { + "src": "!!float 1", + "kind": "float", + "value": "1" + }, + { + "src": "!!bool true", + "kind": "bool", + "value": "true" + }, + { + "src": "!!null ''", + "kind": "null" + }, + { + "src": "!!str ''", + "kind": "string" + }, + { + "src": "!!binary aGk=", + "kind": "string", + "value": "hi" + }, + { + "src": "!!timestamp 2026-11-10", + "kind": "timestamp", + "value": "2026-11-10T00:00:00Z" + }, + { + "src": "!!int 0x1F", + "kind": "int", + "value": "31" + }, + { + "src": "!!int 012", + "kind": "int", + "value": "10" + }, + { + "src": "!!float .inf", + "kind": "float", + "value": "+inf" + }, + { + "src": "!!str 2026-11-10", + "kind": "string", + "value": "2026-11-10" + }, + { + "src": "!!bool yes", + "kind": "error", + "error": "template oracle: yaml: cannot decode !!str `yes` as a !!bool" + }, + { + "src": "!!int abc", + "kind": "error", + "error": "template oracle: yaml: cannot decode !!str `abc` as a !!int" + }, + { + "src": "&a 1", + "kind": "int", + "value": "1" + }, + { + "src": "1 # c", + "kind": "int", + "value": "1" + }, + { + "src": "1e3 # c", + "kind": "float", + "value": "1000" + }, + { + "src": "'a b'", + "kind": "string", + "value": "a b" + }, + { + "src": "a b", + "kind": "string", + "value": "a b" + }, + { + "src": "a: b", + "kind": "error", + "error": "template oracle: yaml: line 5: mapping values are not allowed in this context" + }, + { + "src": "1 2", + "kind": "string", + "value": "1 2" + }, + { + "src": "- 1", + "kind": "error", + "error": "template oracle: yaml: line 5: block sequence entries are not allowed in this context" + }, + { + "src": "[1]", + "kind": "other:[]interface {}", + "value": "[1]" + }, + { + "src": "{a: 1}", + "kind": "other:map[string]interface {}", + "value": "map[a:1]" + }, + { + "src": "0e", + "kind": "string", + "value": "0e" + }, + { + "src": "00x1", + "kind": "string", + "value": "00x1" + }, + { + "src": "0b102", + "kind": "string", + "value": "0b102" + }, + { + "src": "1.5.2", + "kind": "string", + "value": "1.5.2" + }, + { + "src": "+", + "kind": "string", + "value": "+" + }, + { + "src": "+-1", + "kind": "string", + "value": "+-1" + }, + { + "src": "--1", + "kind": "string", + "value": "--1" + }, + { + "src": "1e3.5", + "kind": "string", + "value": "1e3.5" + }, + { + "src": "0x1.8p1", + "kind": "string", + "value": "0x1.8p1" + }, + { + "src": "٣", + "kind": "string", + "value": "٣" + }, + { + "src": "1 ", + "kind": "string", + "value": "1 " + }, + { + "src": "2026-02-29", + "kind": "string", + "value": "2026-02-29" + }, + { + "src": "2026-02-28", + "kind": "timestamp", + "value": "2026-02-28T00:00:00Z" + }, + { + "src": "2026-11-10T10:00:00+24:00", + "kind": "timestamp", + "value": "2026-11-09T10:00:00Z" + }, + { + "src": "2026-11-10T10:00:00+05:30", + "kind": "timestamp", + "value": "2026-11-10T04:30:00Z" + }, + { + "src": "2026-11-10T10:00:00.123456789012Z", + "kind": "timestamp", + "value": "2026-11-10T10:00:00.123456789Z" + }, + { + "src": "2026-11-10 10:00:00.5", + "kind": "timestamp", + "value": "2026-11-10T10:00:00.5Z" + }, + { + "src": "2026-11-10 10:00:00Z", + "kind": "string", + "value": "2026-11-10 10:00:00Z" + }, + { + "src": "2026-11-10T10:00:00", + "kind": "string", + "value": "2026-11-10T10:00:00" + }, + { + "src": "0000-01-01", + "kind": "timestamp", + "value": "0000-01-01T00:00:00Z" + }, + { + "src": "2026-11-10T9:5:7Z", + "kind": "timestamp", + "value": "2026-11-10T09:05:07Z" + }, + { + "src": "12345-01-01", + "kind": "string", + "value": "12345-01-01" + }, + { + "src": "2026-11-10T10:00Z", + "kind": "string", + "value": "2026-11-10T10:00Z" + } + ] +} diff --git a/tests/unit/test_cli_investigate.py b/tests/unit/test_cli_investigate.py index ca878458..37113966 100644 --- a/tests/unit/test_cli_investigate.py +++ b/tests/unit/test_cli_investigate.py @@ -578,6 +578,23 @@ def test_an_unparseable_wave_file_is_a_coded_json_error(self, sdk, tmp_path): assert doc["errors"][0]["code"] == "invalid_input" and doc["errors"][0]["path"] == "--file" sdk.run_wave.assert_not_called() + def test_a_nan_in_a_wave_file_is_a_coded_json_error(self, sdk, tmp_path): + f = tmp_path / "wave.yaml" + f.write_text("inputs: {x: .nan}\nsteps: []\n") + result = run(["investigate", "wave", "--file", str(f)]) + assert result.exit_code == 4 + error = json.loads(result.stderr)["errors"][0] + assert error["code"] == "invalid_input" and error["path"] == "--file" and "JSON cannot carry" in error["message"] + sdk.run_wave.assert_not_called() + + @pytest.mark.parametrize("value", ["NaN", "Infinity", "-Infinity"]) + def test_a_nonfinite_number_in_inputs_json_is_a_coded_json_error(self, sdk, value): + result = run(["investigate", "runs", "start", "--template", "my-plan", "--detection-id", "d1", "--inputs-json", '{"x": %s}' % value]) + assert result.exit_code == 4 + error = json.loads(result.stderr)["errors"][0] + assert error["code"] == "invalid_input" and error["path"] == "--inputs-json" + sdk.start_run.assert_not_called() + def test_a_wrong_shape_is_a_coded_json_error(self, sdk, tmp_path): f = tmp_path / "samples.yaml" f.write_text("just: a mapping\n") diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index 18b89e57..b6c1c435 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -1,6 +1,7 @@ """Tests for limacharlie.sdk.investigation (Investigation Engine).""" import json +from pathlib import Path from unittest.mock import MagicMock, patch import pytest @@ -347,21 +348,113 @@ def test_authentication_errors_pass_through_untouched(self, inv, mock_org): # Template source # --------------------------------------------------------------------------- +_SCALARS = json.loads((Path(__file__).parent / "fixtures" / "investigation_scalars.json").read_text())["cases"] +_PLAIN_SCALARS = [ + c for c in _SCALARS + if c["kind"] != "error" and not c["kind"].startswith("other") + and c["src"] == c["src"].strip() and c["src"] and c["src"][0] not in "'\"!&*#" and " #" not in c["src"] +] + + +def _args_value(src: str) -> str: + return "facts:\n x:\n op: plan.pick\n args:\n value: " + src + "\n" + + class TestTemplateSource: - def test_yaml_1_1_surprises_stay_text(self): - data = parse_template_source("a: yes\nb: off\nc: 2026-11-10\nd: true\ne: [on]\nf: 10:30\ng: 012\nh: 1_000\n") - assert data == {"a": "yes", "b": "off", "c": "2026-11-10", "d": True, "e": ["on"], - "f": "10:30", "g": "012", "h": "1_000"} + @pytest.mark.parametrize("case", _SCALARS, ids=[c["src"] or "" for c in _SCALARS]) + def test_args_scalars_are_read_as_the_engine_reads_them(self, case): + """Every spelling gives what the engine's own parser gave it (fixture generated from that parser).""" + src, kind, want = case["src"], case["kind"], case.get("value", "") + text = _args_value(src) + if kind == "error": + with pytest.raises(ValidationError): + parse_template_source(text) + return + if kind == "float" and want in ("nan", "+inf", "-inf"): + with pytest.raises(ValidationError, match="JSON cannot carry"): + parse_template_source(text) + return + got = parse_template_source(text)["facts"]["x"]["args"]["value"] + if kind == "null": + assert got is None + elif kind == "bool": + assert got is (want == "true") + elif kind == "int": + assert type(got) is int and got == int(want) + elif kind == "float": + assert type(got) is float and got == float(want) + elif kind in ("string", "timestamp"): + assert type(got) is str and got == want + elif kind == "other:[]interface {}": + assert isinstance(got, list) + else: + assert isinstance(got, dict) + + def test_text_fields_keep_the_authors_spelling(self): + data = parse_template_source( + "description: 1e3\nstatus: 012\napplies_to: {category: [on, 0o17, 2026-11-10]}\n" + "inputs: {a: yes, 1e3: ~}\n" + ) + assert data == { + "description": "1e3", "status": "012", + "applies_to": {"category": ["on", "0o17", "2026-11-10"]}, + "inputs": {"a": "yes", "1e3": "~"}, + } - def test_yaml_1_2_numbers_are_numbers(self): - data = parse_template_source("a: 12\nb: -3\nc: 1.5\nd: 1e3\ne: 0x1f\nf: 0o17\ng: 0\nh: .5\ni: null\n") - assert data == {"a": 12, "b": -3, "c": 1.5, "d": 1000.0, "e": 31, "f": 15, "g": 0, "h": 0.5, "i": None} - assert type(data["a"]) is int + def test_numeric_and_boolean_fields_are_read_as_such(self): + data = parse_template_source( + "version: 3\nprior: .5\nevidence: [{when: x, weight: -1e0}]\n" + "signals: {s: {malicious_weight: 2, benign_weight: 0x10}}\n" + "facts: {a: {op: o, predict: yes, case: true}, b: {op: o, predict: off, case: False}}\n" + ) + assert data["version"] == 3 and type(data["version"]) is int + assert data["prior"] == 0.5 + assert data["evidence"] == [{"when": "x", "weight": -1.0}] + assert data["signals"]["s"] == {"malicious_weight": 2, "benign_weight": 16} + assert data["facts"]["a"]["predict"] is True and data["facts"]["a"]["case"] is True + assert data["facts"]["b"]["predict"] is False and data["facts"]["b"]["case"] is False + + def test_report_case_is_text_while_a_fact_case_is_a_boolean(self): + data = parse_template_source("report: {case: create}\nfacts: {a: {op: o, case: true}}\n") + assert data["report"]["case"] == "create" and data["facts"]["a"]["case"] is True + + def test_quoted_and_tagged_scalars(self): + data = parse_template_source( + _args_value('"1e3"') + "").copy() + assert data["facts"]["x"]["args"]["value"] == "1e3" + got = parse_template_source( + "facts:\n x:\n op: o\n args: {a: !!str 12, b: !!int '0x1f', c: !!float 1, d: '~', e: !!binary aGk=}\n" + )["facts"]["x"]["args"] + assert got == {"a": "12", "b": 31, "c": 1.0, "d": "~", "e": "hi"} + + def test_a_tag_that_does_not_fit_is_refused(self): + with pytest.raises(ValidationError, match="cannot decode"): + parse_template_source(_args_value("!!int abc")) + + def test_nested_args_are_read_like_top_level_args(self): + got = parse_template_source( + "facts:\n x:\n op: o\n args:\n list: [012, 1_000, 1e3, off]\n deep: {k: 0o17}\n" + )["facts"]["x"]["args"] + assert got == {"list": [10, 1000, 1000.0, "off"], "deep": {"k": 15}} def test_dump_round_trips_text_that_looks_like_something_else(self): data = {"a": "yes", "c": "2026-11-10", "n": "0308"} assert parse_template_source(dump_template(data)) == data + @pytest.mark.parametrize("case", _PLAIN_SCALARS, ids=[c["src"] or "" for c in _PLAIN_SCALARS]) + def test_dump_quotes_every_string_the_engine_would_read_as_something_else(self, case): + """A string the engine reads as a non-string when bare must be written quoted.""" + src = case["src"] + text = dump_template({"facts": {"x": {"op": "plan.pick", "args": {"value": src}}}}) + assert parse_template_source(text)["facts"]["x"]["args"]["value"] == src + if case["kind"] != "string": + assert f"value: {src}\n" not in text, f"{src!r} would be read as {case['kind']}: {text}" + + @pytest.mark.parametrize("src", ["1e3", "0o17", "1E+3", "1e-3", "0x1F", "012", "1_000", ".5", "5.", "+1", ".inf", "-.INF", ".nan", "~", "null", "True", "yes", "off", "2026-11-10", "10:30"]) + def test_dump_of_quoted_numeric_strings_keeps_the_type(self, src): + data = {"description": src, "facts": {"x": {"op": "o", "args": {"v": src, "l": [src]}}}} + assert json.dumps(parse_template_source(dump_template(data))) == json.dumps(data) + def test_dump_keeps_key_order(self): text = dump_template({"zebra": 1, "alpha": 2}) assert text.index("zebra") < text.index("alpha") @@ -413,6 +506,25 @@ def test_validates_the_exact_text_that_is_stored(self, inv, mock_org, hive): sent = json.loads(mock_org.client.request.call_args.kwargs["raw_body"])["yaml"] assert parse_template_source(sent)["status"] == "draft" + def test_quoted_numeric_looking_strings_are_validated_as_stored(self, inv, mock_org, hive): + src = ('description: "1e3"\nfacts:\n a:\n op: plan.pick\n' + ' args: {value: "1e3", hex: "0o17", num: 1e3, octal: 012}\n') + mock_org.client.request.return_value = VALID + hive.get.side_effect = [_not_found(), self._stored("draft")] + inv.save_template("my-plan", src) + stored = hive.set.call_args.args[0].data + assert stored["description"] == "1e3" + assert stored["facts"]["a"]["args"] == {"value": "1e3", "hex": "0o17", "num": 1000.0, "octal": 10} + sent = json.loads(mock_org.client.request.call_args.kwargs["raw_body"])["yaml"] + assert json.dumps(parse_template_source(sent)) == json.dumps(stored) + assert '"1e3"' in sent and '"0o17"' in sent # still strings when the engine reads the text it validated + + def test_a_nan_in_the_source_is_refused_before_anything_is_sent(self, inv, mock_org, hive): + with pytest.raises(ValidationError, match="JSON cannot carry"): + inv.save_template("my-plan", "facts: {a: {op: plan.pick, args: {value: .nan}}}\n") + mock_org.client.request.assert_not_called() + hive.set.assert_not_called() + def test_a_template_cannot_activate_itself(self, inv, mock_org, hive): out = inv.save_template("my-plan", SRC + "status: active\n") assert out["saved"] is False and "cannot activate itself" in out["reason"] @@ -609,6 +721,14 @@ def test_promotes_with_the_etag_that_was_read(self, inv, mock_org, hive): sent = parse_template_source(json.loads(mock_org.client.request.call_args.kwargs["raw_body"])["yaml"]) assert sent["status"] == "active" # what is validated is what is stored + def test_a_record_that_does_not_read_back_identically_is_not_validated_as_something_else(self, inv, mock_org, hive): + # A number where the engine reads text: the rendered YAML would read back as text, a different template. + hive.get.return_value = HiveRecord(name="p", data={"description": 5, "status": "draft"}, etag="e1") + with pytest.raises(ValidationError, match="reads back identically"): + inv.promote_template("p") + mock_org.client.request.assert_not_called() + hive.set.assert_not_called() + def test_an_invalid_draft_stays_a_draft(self, inv, mock_org, hive): mock_org.client.request.return_value = INVALID hive.get.return_value = HiveRecord(name="p", data={"status": "draft"}, etag="e1") @@ -693,8 +813,8 @@ def test_active_status_is_written_only_when_asked(self, inv, mock_org, hive): class TestParseDocument: def test_yaml_11_words_and_dates_are_sent_as_written(self): - doc = parse_document("a: off\nb: yes\nc: no\nd: 2026-11-10\ne: true\nf: 012\n") - assert doc == {"a": "off", "b": "yes", "c": "no", "d": "2026-11-10", "e": True, "f": "012"} + doc = parse_document("a: off\nb: yes\nc: no\nd: 2026-11-10\ne: true\nf: 012\ng: 1_000\nh: 1e3\n") + assert doc == {"a": "off", "b": "yes", "c": "no", "d": "2026-11-10", "e": True, "f": 10, "g": 1000, "h": 1000.0} json.dumps(doc) def test_json_is_accepted(self): @@ -711,6 +831,16 @@ def test_empty_is_a_validation_error(self): with pytest.raises(ValidationError): parse_document(" \n") - def test_values_json_cannot_carry_are_refused(self): + @pytest.mark.parametrize("src", ["a: .nan\n", "a: [.inf]\n", "a: {b: -.inf}\n"]) + def test_values_json_cannot_carry_are_refused(self, src): + with pytest.raises(ValidationError, match="JSON cannot carry"): + parse_document(src) + + def test_a_nan_input_never_reaches_the_wire(self, inv, mock_org): + with pytest.raises(ValidationError, match="JSON cannot carry"): + inv._call("POST", "waves", body={"inputs": {"x": float("nan")}}, write=True) + mock_org.client.request.assert_not_called() + + def test_a_set_tag_is_refused(self): with pytest.raises(ValidationError): - parse_document("a: !!binary aGk=\n") + parse_document("a: !!set {x}\n") From 5f77b1f39f6c04f821a4a277e5a00b2cd74f3203 Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 18:21:38 -0800 Subject: [PATCH 10/14] Investigation CLI: pin the typed template fields to the engine's types with a generated fixture Co-Authored-By: Claude Sonnet 5.5 --- .../fixtures/investigation_typed_fields.json | 33 +++++++++++++++++++ tests/unit/test_sdk_investigation.py | 10 ++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/unit/fixtures/investigation_typed_fields.json diff --git a/tests/unit/fixtures/investigation_typed_fields.json b/tests/unit/fixtures/investigation_typed_fields.json new file mode 100644 index 00000000..69d1ef15 --- /dev/null +++ b/tests/unit/fixtures/investigation_typed_fields.json @@ -0,0 +1,33 @@ +{ + "about": "Fields of a template the engine reads as a number or a boolean (every other scalar outside a fact's args is text), by YAML path; * is a list index or a map key. Generated from the engine's template types; do not edit by hand.", + "fields": [ + { + "kind": "number", + "path": "evidence.*.weight" + }, + { + "kind": "bool", + "path": "facts.*.case" + }, + { + "kind": "bool", + "path": "facts.*.predict" + }, + { + "kind": "number", + "path": "prior" + }, + { + "kind": "number", + "path": "signals.*.benign_weight" + }, + { + "kind": "number", + "path": "signals.*.malicious_weight" + }, + { + "kind": "number", + "path": "version" + } + ] +} diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index b6c1c435..243c767e 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -390,6 +390,16 @@ def test_args_scalars_are_read_as_the_engine_reads_them(self, case): else: assert isinstance(got, dict) + def test_the_typed_field_lists_cover_every_non_text_field_of_the_engine(self): + """A field the engine adds as a number or boolean must be added to the SDK's lists, or it would be stored as text.""" + fields = json.loads((Path(__file__).parent / "fixtures" / "investigation_typed_fields.json").read_text())["fields"] + sdk_paths = { + "number": {".".join(p) for p in inv_mod._NUMBER_PATHS}, + "bool": {".".join(p) for p in inv_mod._BOOL_PATHS}, + } + assert {f["path"] for f in fields if f["kind"] == "number"} == sdk_paths["number"] + assert {f["path"] for f in fields if f["kind"] == "bool"} == sdk_paths["bool"] + def test_text_fields_keep_the_authors_spelling(self): data = parse_template_source( "description: 1e3\nstatus: 012\napplies_to: {category: [on, 0o17, 2026-11-10]}\n" From 85824baaa274ab5f1473ab07fe9bc048fe1ab6b0 Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 19:03:11 -0800 Subject: [PATCH 11/14] Explain read-only live sensor identity permissions Co-Authored-By: Claude Sonnet 5.5 --- doc/cli/investigation-engine.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md index e449c54d..3e21a874 100644 --- a/doc/cli/investigation-engine.md +++ b/doc/cli/investigation-engine.md @@ -24,6 +24,14 @@ All the listed permissions are required. Waves and template tests always execute with **your own token**, so the operators enforce your permissions. A run started with `runs start` executes as the identity the organization's `investigation_policy` names for its template: your token when the policy names none, otherwise the Hive secret the policy points to, in which case the operators enforce that secret's permissions and not yours. An active template can therefore reach data or take actions beyond your own permissions. Before you start one, check `limacharlie investigate policy get --template NAME`: its `identity` shows the default identity, the allowed ones and where they come from. A finished run's `identity` field (`kind` caller or secret) records which one it used. Runs started by a detection rule or a case never have a caller and always use the policy's secret identity. +Live sensor metadata operators (`sensor.read.*`) need `sensor.task.read` (or the +broader `sensor.task`) plus `output.set` on the execution identity. Explicitly grant +`sensor.task.read` for an observe-only identity; no default or preset includes it. +It authorizes only allowlisted metadata commands when the deployment enables +`INVESTIGATION_ENGINE_ENABLED=true` or `1`. File contents, mutations and unknown +commands are refused server-side; with the gate off it grants no tasking. Keep +observe-only policies at `suggest`. Action permissions above remain unchanged. + ## AI agent workflow Hand an agent this loop. It never activates anything on its own: a person does the last step. From b0b05c8bef4b8276eaaa6008b2cc4c56fa31cb7c Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 19:03:10 -0800 Subject: [PATCH 12/14] Add investigation run deletion to the SDK and CLI Expose DELETE for runs with validated IDs, explicit CLI confirmation, discovery and documentation. Preserve API errors and disable gateway timeout retries for deletion writes. Co-Authored-By: Claude Sonnet 5.5 --- doc/cli/investigation-engine.md | 5 ++++- limacharlie/commands/investigate.py | 27 +++++++++++++++++++++-- limacharlie/discovery.py | 2 +- limacharlie/sdk/investigation.py | 21 +++++++++++++++--- tests/unit/test_cli_investigate.py | 32 ++++++++++++++++++++++++++++ tests/unit/test_sdk_investigation.py | 27 +++++++++++++++++++++++ 6 files changed, 107 insertions(+), 7 deletions(-) diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md index 3e21a874..dae09acb 100644 --- a/doc/cli/investigation-engine.md +++ b/doc/cli/investigation-engine.md @@ -17,7 +17,7 @@ All the listed permissions are required. | Operation | Permissions | |---|---| | Reads: runs, templates, operators, packs, kit, policy, test results, the approvals inbox | `investigation.get` + `insight.evt.get` | -| Writes: start a run, wave, cancel, test a template, decide an action | `investigation.set` + `insight.evt.get` | +| Writes: start a run, wave, cancel, delete a run, test a template, decide an action | `investigation.set` + `insight.evt.get` | | Approve / deny an action | the writes above + the permission of the action's class (for example `sensor.task`, `mailsec.act`, `sensor.tag`) | | `policy pause` / `policy resume` | `org.conf.set` | | Template records (`investigation_plan` hive) | `investigation_plan.get`, `investigation_plan.set`, `investigation_plan.del` (explicit grants; no default role has them) | @@ -135,10 +135,13 @@ limacharlie investigate runs events RUN_ID [--after SEQ] [--wait SECONDS] [--lim limacharlie investigate runs steps RUN_ID [--values] limacharlie investigate runs case RUN_ID limacharlie investigate runs cancel RUN_ID [--revert-leases] +limacharlie investigate runs delete RUN_ID --confirm ``` `--auto` sends no plan, which is how the engine is asked to pick the **active** template whose `applies_to` matches the trigger (a draft is never picked; the engine refuses `plan.auto` itself for a person's run). `applies_to` keys are `detect_cat`, `severity`, `tag` and `source`; the first three are matched against `--category`, `--severity` and `--tag` (a start from the CLI always has `source: api`), so pass the alert's attributes. When no template matches, the start is refused with the reason. `--category`, `--severity`, `--priority` and `--tag` also select the run-scoped policy records. `--urgency` defaults to a value derived from `--priority` (8+ critical, 6+ high, 3+ medium, else low) or `--severity`, and medium when neither is given. A start is sent once and never retried automatically; `--idempotency-key` makes a repeated start return the same run (`joined: true`). +`runs delete` requires explicit `--confirm` and returns the engine's deletion response. It needs `investigation.set` + `insight.evt.get`. Deletion requests are sent once and are not automatically replayed after a gateway timeout. + `runs list` filters: `--state` (queued, running, decided, completed, cancelled, failed, budget_exhausted), `--verdict` (true_positive, false_positive, inconclusive, undecided), `--trigger` (the kind of trigger: detection, message, entity, case, question) and `--started-by` (an identity such as `user:UID` or `apikey:ID`, or a user's email or a key's name). `--mode` is `full` by default (triage, scoping and containment); `triage` stops at the verdict and costs less, `scope` is for scoping an already-triaged alert. (`templates test` defaults to `triage`.) diff --git a/limacharlie/commands/investigate.py b/limacharlie/commands/investigate.py index b644175a..4ab928ec 100644 --- a/limacharlie/commands/investigate.py +++ b/limacharlie/commands/investigate.py @@ -66,7 +66,7 @@ Questions without a template: use `investigate packs` then `investigate wave`. Permissions: reads need investigation.get + insight.evt.get, writes (start, -wave, test, cancel, decide) investigation.set + insight.evt.get, pause/resume +wave, test, cancel, delete, decide) investigation.set + insight.evt.get, pause/resume org.conf.set, template records investigation_plan.get/.set/.del. """ @@ -293,6 +293,14 @@ limacharlie investigate runs cancel """ +_EXPLAIN_RUNS_DELETE = """\ +Delete a run. Requires investigation.set + insight.evt.get and --confirm. The request is sent +once; a gateway timeout is not automatically retried. + +Example: + limacharlie investigate runs delete --confirm +""" + _EXPLAIN_WAVE = """\ Hand the engine a batch of questions at once and get compact, typed answers. The engine runs every ready step in parallel with YOUR token (no template, no @@ -385,6 +393,7 @@ "investigate.runs.steps": _EXPLAIN_RUNS_STEPS, "investigate.runs.case": _EXPLAIN_RUNS_CASE, "investigate.runs.cancel": _EXPLAIN_RUNS_CANCEL, + "investigate.runs.delete": _EXPLAIN_RUNS_DELETE, "investigate.wave": _EXPLAIN_WAVE, "investigate.actions.list": _EXPLAIN_ACTIONS_LIST, "investigate.actions.approve": _EXPLAIN_ACTIONS_APPROVE, @@ -876,7 +885,7 @@ def templates_install_defaults(ctx, names, overwrite, status) -> None: @group.group("runs") def runs_group() -> None: - """Investigation runs: start, get, list, events, steps, case, cancel.""" + """Investigation runs: start, get, list, events, steps, case, cancel, delete.""" @runs_group.command("start") @@ -1065,6 +1074,20 @@ def runs_cancel(ctx, run_id, revert_leases) -> None: command_output(ctx, _get_investigation(ctx).cancel_run(run_id, revert_leases=revert_leases)) +@runs_group.command("delete") +@click.argument("run_id") +@click.option("--confirm", is_flag=True, default=False, help="Confirm deletion (required).") +@pass_context +@_engine_errors +def runs_delete(ctx, run_id, confirm) -> None: + """Delete a run (requires --confirm).""" + if not confirm: + raise click.UsageError("Destructive operation requires --confirm flag.") + result = _get_investigation(ctx).delete_run(run_id) + note(ctx, f"Run '{run_id}' deleted.") + command_output(ctx, result) + + # --------------------------------------------------------------------------- # wave # --------------------------------------------------------------------------- diff --git a/limacharlie/discovery.py b/limacharlie/discovery.py index ffe95c25..2fa4d8b4 100644 --- a/limacharlie/discovery.py +++ b/limacharlie/discovery.py @@ -223,7 +223,7 @@ "investigate templates delete", "investigate templates install-defaults", "investigate runs start", "investigate runs get", "investigate runs list", "investigate runs events", "investigate runs steps", "investigate runs case", - "investigate runs cancel", + "investigate runs cancel", "investigate runs delete", "investigate wave", "investigate actions list", "investigate actions approve", "investigate actions deny", "investigate policy get", "investigate policy status", diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index 53c74cae..f3fa163a 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -3,7 +3,7 @@ Wraps the ``/investigate/{oid}/...`` REST routes served by the API gateway: the authoring kit, the typed operator and fact-pack catalogues, plan validation and dry runs on past alerts, investigation runs (start, read, -list, events, cancel), waves, the approvals inbox and the autonomy switch. +list, events, cancel, delete), waves, the approvals inbox and the autonomy switch. The feature is experimental and gated per deployment. Where it is off, every route is an unknown path (HTTP 404 with no engine error code); where the @@ -14,7 +14,7 @@ - reads (runs, templates, operators, packs, kit, policy, test results, the approvals inbox): ``investigation.get`` + ``insight.evt.get`` -- writes (start a run, waves, cancel, test a plan, decide an action): +- writes (start a run, waves, cancel, delete, test a plan, decide an action): ``investigation.set`` + ``insight.evt.get``; deciding an action also needs the permission of the action's own class (for example ``sensor.task``) - pause / resume of automatic actions: ``org.conf.set`` @@ -146,7 +146,7 @@ def _issue_line(issue: Any) -> str: _PERMISSION_HINT = ( - "Reads need investigation.get + insight.evt.get, writes (start, wave, cancel, test, decide) need " + "Reads need investigation.get + insight.evt.get, writes (start, wave, cancel, delete, test, decide) need " "investigation.set + insight.evt.get, pause/resume needs org.conf.set, and template records need " "investigation_plan.get / investigation_plan.set. Deciding an action also needs the permission of its class." ) @@ -1180,6 +1180,21 @@ def cancel_run(self, run_id: str, revert_leases: bool = False) -> dict[str, Any] body = {"revert_leases": True} if revert_leases else {} return self._call("POST", f"runs/{_quote(_check_id('run id', run_id), safe='')}/cancel", body=body, write=True) + def delete_run(self, run_id: str) -> dict[str, Any]: + """Delete a run (requires investigation.set and insight.evt.get). + + Args: + run_id: The run id. + + Returns: + dict: The engine's deletion response. + + Raises: + ValidationError: If the run id is invalid. + InvestigationError: If the API refuses deletion or the request fails. + """ + return self._call("DELETE", f"runs/{_quote(_check_id('run id', run_id), safe='')}", write=True) + # ------------------------------------------------------------------ # Waves # ------------------------------------------------------------------ diff --git a/tests/unit/test_cli_investigate.py b/tests/unit/test_cli_investigate.py index 37113966..fc960d61 100644 --- a/tests/unit/test_cli_investigate.py +++ b/tests/unit/test_cli_investigate.py @@ -427,6 +427,38 @@ def events(run_id, **kw): result = run(["investigate", "runs", "events", "r1", "--follow"], fmt="table") assert result.stdout.strip() == '1\t2026-11-10T00:00:00Z\tstep\t{"fact":"a"}' + def test_delete_requires_confirmation_before_constructing_sdk(self, sdk): + result = run(["investigate", "runs", "delete", "r1"]) + assert result.exit_code == 2 + assert "--confirm" in result.stderr + sdk.delete_run.assert_not_called() + from limacharlie.commands.investigate import Investigation + Investigation.assert_not_called() + + def test_delete_passes_run_id_and_prints_response(self, sdk): + sdk.delete_run.return_value = {"deleted": True} + result = run(["investigate", "runs", "delete", "run:1", "--confirm"]) + assert result.exit_code == 0, result.stderr + assert out_json(result) == {"deleted": True} + sdk.delete_run.assert_called_once_with("run:1") + sdk.cancel_run.assert_not_called() + + def test_delete_error_uses_normal_engine_rendering(self, sdk): + err = issue_error() + sdk.delete_run.side_effect = err + result = run(["investigate", "runs", "delete", "r1", "--confirm"]) + assert result.exit_code == err.exit_code + assert result.stdout == "" + assert json.loads(result.stderr)["error_code"] == err.error_code + sdk.delete_run.assert_called_once_with("r1") + + def test_delete_is_discoverable_and_explained(self): + from limacharlie.discovery import PROFILES + assert "investigate runs delete" in PROFILES["investigation_engine"]["commands"] + result = CliRunner().invoke(cli, ["investigate", "runs", "delete", "--ai-help"]) + assert result.exit_code == 0 + assert "investigation.set" in result.output and "--confirm" in result.output + def test_get_steps_case_cancel(self, sdk): sdk.get_run.return_value = {"run_id": "r1"} run(["investigate", "runs", "get", "r1"]) diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index 243c767e..99905f25 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -148,6 +148,33 @@ def test_run_id_is_escaped_and_validated(self, inv, mock_org): with pytest.raises(ValidationError): inv.get_run(bad) + def test_delete_run_route_response_and_no_retry(self, inv, mock_org): + response = {"deleted": True} + mock_org.client.request.return_value = response + assert inv.delete_run("run:1.a-b") == response + mock_org.client.request.assert_called_once_with( + "DELETE", "investigate/oid-1/runs/run%3A1.a-b", + query_params=None, timeout=None, retry_gateway_timeout=False, + ) + + @pytest.mark.parametrize("run_id", ["..", ".", "a/b", "", "x" * 129]) + def test_delete_run_rejects_invalid_ids_before_request(self, inv, mock_org, run_id): + with pytest.raises(ValidationError): + inv.delete_run(run_id) + mock_org.client.request.assert_not_called() + + @pytest.mark.parametrize("status, code", [(403, "forbidden"), (404, "not_found"), (409, "conflict"), (504, "timeout")]) + def test_delete_run_preserves_errors_without_replay(self, inv, mock_org, status, code): + body = {"error": "deletion refused", "error_code": code} + mock_org.client.request.side_effect = error_from_status_code(status, body) + with pytest.raises(InvestigationError) as ei: + inv.delete_run("r1") + assert ei.value.status_code == status + assert ei.value.error_code == code + assert ei.value.to_dict()["error"] == "deletion refused" + assert mock_org.client.request.call_count == 1 + assert _request_args(mock_org)[2]["retry_gateway_timeout"] is False + def test_events_bounds_and_params(self, inv, mock_org): mock_org.client.request.return_value = {"events": [], "next": 4, "done": False} inv.get_run_events("r1", after=4, wait=10, limit=500) From 3cc81aff45f183d8a4265e8ceedc966f8fb33c3e Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 19:08:18 -0800 Subject: [PATCH 13/14] Investigation CLI: say when the API does not serve run deletion; drop a server setting name from the docs Co-Authored-By: Claude Sonnet 5.5 --- doc/cli/investigation-engine.md | 6 +++--- limacharlie/sdk/investigation.py | 14 +++++++++++++- tests/unit/test_sdk_investigation.py | 16 +++++++++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md index dae09acb..d658e131 100644 --- a/doc/cli/investigation-engine.md +++ b/doc/cli/investigation-engine.md @@ -27,8 +27,8 @@ Waves and template tests always execute with **your own token**, so the operator Live sensor metadata operators (`sensor.read.*`) need `sensor.task.read` (or the broader `sensor.task`) plus `output.set` on the execution identity. Explicitly grant `sensor.task.read` for an observe-only identity; no default or preset includes it. -It authorizes only allowlisted metadata commands when the deployment enables -`INVESTIGATION_ENGINE_ENABLED=true` or `1`. File contents, mutations and unknown +It authorizes only allowlisted metadata commands, and only when the Investigation +Engine is enabled for the deployment. File contents, mutations and unknown commands are refused server-side; with the gate off it grants no tasking. Keep observe-only policies at `suggest`. Action permissions above remain unchanged. @@ -140,7 +140,7 @@ limacharlie investigate runs delete RUN_ID --confirm `--auto` sends no plan, which is how the engine is asked to pick the **active** template whose `applies_to` matches the trigger (a draft is never picked; the engine refuses `plan.auto` itself for a person's run). `applies_to` keys are `detect_cat`, `severity`, `tag` and `source`; the first three are matched against `--category`, `--severity` and `--tag` (a start from the CLI always has `source: api`), so pass the alert's attributes. When no template matches, the start is refused with the reason. `--category`, `--severity`, `--priority` and `--tag` also select the run-scoped policy records. `--urgency` defaults to a value derived from `--priority` (8+ critical, 6+ high, 3+ medium, else low) or `--severity`, and medium when neither is given. A start is sent once and never retried automatically; `--idempotency-key` makes a repeated start return the same run (`joined: true`). -`runs delete` requires explicit `--confirm` and returns the engine's deletion response. It needs `investigation.set` + `insight.evt.get`. Deletion requests are sent once and are not automatically replayed after a gateway timeout. +`runs delete` requires explicit `--confirm` and returns the engine's deletion response. It needs `investigation.set` + `insight.evt.get`. Deletion requests are sent once and are not automatically replayed after a gateway timeout. The API deployment must serve run deletion, which is newer than the other run routes; an older deployment answers 404 without an engine error code, and the CLI says so. `runs list` filters: `--state` (queued, running, decided, completed, cancelled, failed, budget_exhausted), `--verdict` (true_positive, false_positive, inconclusive, undecided), `--trigger` (the kind of trigger: detection, message, entity, case, question) and `--started-by` (an identity such as `user:UID` or `apikey:ID`, or a user's email or a key's name). diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index f3fa163a..bcc4b1b8 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -204,6 +204,11 @@ def _wrap_error(exc: LimaCharlieError) -> LimaCharlieError: ) +_DELETE_ROUTE_HINT = ( + "A 404 with no engine error code means this API deployment does not serve run deletion: the " + "Investigation Engine is not enabled for it, or it is a version that predates `runs delete`." +) + _TEMPLATE_PERMISSION_HINT = ( "Template records are the hive 'investigation_plan': reading needs investigation_plan.get, writing " "investigation_plan.set, deleting investigation_plan.del. They are explicit grants that no default role has." @@ -1193,7 +1198,14 @@ def delete_run(self, run_id: str) -> dict[str, Any]: ValidationError: If the run id is invalid. InvestigationError: If the API refuses deletion or the request fails. """ - return self._call("DELETE", f"runs/{_quote(_check_id('run id', run_id), safe='')}", write=True) + try: + return self._call("DELETE", f"runs/{_quote(_check_id('run id', run_id), safe='')}", write=True) + except InvestigationError as exc: + if exc.status_code == 404 and not exc.error_code: + # Without an engine code the API did not serve the route at all: either the feature is + # off or this API deployment predates run deletion. + exc.suggestion = _DELETE_ROUTE_HINT + raise # ------------------------------------------------------------------ # Waves diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index 99905f25..df872b45 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -172,9 +172,23 @@ def test_delete_run_preserves_errors_without_replay(self, inv, mock_org, status, assert ei.value.status_code == status assert ei.value.error_code == code assert ei.value.to_dict()["error"] == "deletion refused" - assert mock_org.client.request.call_count == 1 assert _request_args(mock_org)[2]["retry_gateway_timeout"] is False + def test_delete_run_404_without_engine_code_says_the_route_is_not_served(self, inv, mock_org): + mock_org.client.request.side_effect = error_from_status_code(404, {"error": "not found"}) + with pytest.raises(InvestigationError) as ei: + inv.delete_run("r1") + assert ei.value.status_code == 404 + assert "does not serve run deletion" in ei.value.suggestion + + def test_delete_run_404_with_engine_code_keeps_the_engine_message(self, inv, mock_org): + body = {"error": "no such run", "error_code": "not_found"} + mock_org.client.request.side_effect = error_from_status_code(404, body) + with pytest.raises(InvestigationError) as ei: + inv.delete_run("r1") + assert ei.value.error_code == "not_found" + assert "does not serve" not in (ei.value.suggestion or "") + def test_events_bounds_and_params(self, inv, mock_org): mock_org.client.request.return_value = {"events": [], "next": 4, "done": False} inv.get_run_events("r1", after=4, wait=10, limit=500) From 78344085ca5056cab9ff7d7b996f82aed065cc3e Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Tue, 10 Nov 2026 19:10:29 -0800 Subject: [PATCH 14/14] Investigation CLI: rebuild the run-deletion 404 error so the message carries one hint Co-Authored-By: Claude Sonnet 5.5 --- limacharlie/sdk/investigation.py | 11 ++++++++++- tests/unit/test_sdk_investigation.py | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index bcc4b1b8..3f935c4f 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -1204,7 +1204,16 @@ def delete_run(self, run_id: str) -> dict[str, Any]: if exc.status_code == 404 and not exc.error_code: # Without an engine code the API did not serve the route at all: either the feature is # off or this API deployment predates run deletion. - exc.suggestion = _DELETE_ROUTE_HINT + # Rebuilt, not patched: the message text embeds the suggestion it was built with. + raise InvestigationError( + exc.raw_message, + status_code=exc.status_code, + issues=exc.issues, + warnings=exc.warnings, + response_body=exc.response_body, + suggestion=_DELETE_ROUTE_HINT, + exit_code=exc.exit_code, + ) from exc raise # ------------------------------------------------------------------ diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index df872b45..37db432a 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -180,6 +180,13 @@ def test_delete_run_404_without_engine_code_says_the_route_is_not_served(self, i inv.delete_run("r1") assert ei.value.status_code == 404 assert "does not serve run deletion" in ei.value.suggestion + # What the user sees carries the delete hint only, once. + assert str(ei.value).count("Suggestion:") == 1 + assert "does not serve run deletion" in str(ei.value) + assert "experimental and gated" not in str(ei.value) + doc = ei.value.to_dict() + assert doc["error"] == "not found" + assert "does not serve run deletion" in doc["suggestion"] def test_delete_run_404_with_engine_code_keeps_the_engine_message(self, inv, mock_org): body = {"error": "no such run", "error_code": "not_found"}