diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f4e96d6..a3dad375 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +### Sync — `sync push` no longer turns quoted strings into numbers + +- `sync push` (and `Configs.push` / `Configs.push_from_file`) stored some + quoted strings as numbers: `'0308'`, `'0129'` and zero-padded IDs containing + an 8 or 9 became 308, 129, ..., `'1e3'` became 1000 and `'0o17'` became 15. + A D&R rule matching such a value then silently stopped matching. PyYAML + (YAML 1.1) wrote these strings without quotes, and the server reads the + config with YAML 1.2 rules, under which they are numbers. Every string in a + pushed config is now written double-quoted, which is a string under any YAML + version. Numbers, booleans, nulls, dates and binary values are written as + before. +- Values already stored with the wrong type are corrected by pushing the + config again with this version. + ### Adapters — Palo Alto SaaS Security (`prisma_saas`) - `cloud-adapter` and `external-adapter` help now document the `prisma_saas` diff --git a/limacharlie/sdk/configs.py b/limacharlie/sdk/configs.py index fc99440f..01f071a8 100644 --- a/limacharlie/sdk/configs.py +++ b/limacharlie/sdk/configs.py @@ -18,6 +18,38 @@ class ConfigException(Exception): pass +class _QuotedStrDumper(yaml.SafeDumper): + """SafeDumper that writes every string as a double-quoted scalar. + + PyYAML implements YAML 1.1 and leaves a string unquoted whenever YAML 1.1 + would read it back as a string. ext-infrastructure parses the pushed + config with gopkg.in/yaml.v3, which applies YAML 1.2 rules, where some of + those bare strings are numbers: ``0308``, ``0129``, ``1e3`` and ``0o17`` + would be stored in the hive as 308, 129, 1000 and 15. A double-quoted + scalar is a string under every YAML version. Every other type is written + exactly as ``yaml.safe_dump`` writes it. + """ + + +def _represent_quoted_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') + + +_QuotedStrDumper.add_representer(str, _represent_quoted_str) + + +def _dump_push_config(config: dict[str, Any]) -> str: + """Serialize a config for ext-infrastructure's ``push`` action. + + Args: + config: Dict of configuration data to push. + + Returns: + str: The config as a YAML document with every string double-quoted. + """ + return yaml.dump(config, Dumper=_QuotedStrDumper, version=(1, 1)) + + class Configs: """Infrastructure-as-Code: fetch and push org configuration. @@ -151,7 +183,7 @@ def push(self, config: dict[str, Any], is_force: bool = False, is_dry_run: bool if sync_hives is None: sync_hives = {} - final_config = yaml.safe_dump(config, version=(1, 1)) + final_config = _dump_push_config(config) from .extensions import Extensions ext = Extensions(self._org) diff --git a/tests/unit/test_sdk_configs.py b/tests/unit/test_sdk_configs.py index d6501a45..cb5a69ac 100644 --- a/tests/unit/test_sdk_configs.py +++ b/tests/unit/test_sdk_configs.py @@ -1,5 +1,9 @@ """Tests for limacharlie.sdk.configs module.""" +import base64 +import datetime +import gzip +import json import os import tempfile from unittest.mock import MagicMock @@ -127,6 +131,157 @@ def test_push_returns_backend_errors(self, configs, mock_org): assert "bad-output" in errors[0] +def _sent_push_config(mock_org): + """Return the config text the SDK sent to ext-infrastructure's push action.""" + params = mock_org.client.request.call_args.kwargs["params"] + assert params["action"] == "push" + data = json.loads(gzip.decompress(base64.b64decode(params["gzdata"]))) + return data["config"] + + +def _scalar_nodes(node): + """Yield every scalar node of a composed YAML document, keys included.""" + if isinstance(node, yaml.ScalarNode): + yield node + elif isinstance(node, yaml.SequenceNode): + for item in node.value: + yield from _scalar_nodes(item) + elif isinstance(node, yaml.MappingNode): + for k, v in node.value: + yield from _scalar_nodes(k) + yield from _scalar_nodes(v) + + +# Strings that PyYAML (YAML 1.1) leaves unquoted but that a YAML 1.2 reader, +# such as ext-infrastructure's gopkg.in/yaml.v3, reads as numbers. +YAML12_NUMBER_LOOKALIKES = [ + "0308", "0129", "0389", "08", "09", + "00000000000000000000000000000308", + "1e3", "2e5", "1E10", "-1e3", + "0o17", "0o0", +] + + +class TestConfigsPushStringTyping: + """Strings must reach ext-infrastructure as strings under YAML 1.1 and 1.2.""" + + @pytest.mark.parametrize("value", YAML12_NUMBER_LOOKALIKES) + def test_number_lookalike_is_sent_double_quoted(self, configs, mock_org, value): + mock_org.client.request.return_value = {"data": {"ops": []}} + config = {"version": 3, "hives": {"dr-general": {"r1": {"data": { + "detect": {"event": "NEW_PROCESS", "op": "is", "path": "event/COMMAND_LINE", "value": value}, + "respond": [{"action": "report", "name": "r1"}], + }}}}} + configs.push(config, sync_hives={"dr-general": True}) + + sent = _sent_push_config(mock_org) + assert f'"value": "{value}"' in sent + assert yaml.safe_load(sent) == config + + def test_every_string_is_double_quoted_and_other_types_are_plain(self, configs, mock_org): + mock_org.client.request.return_value = {"data": {"ops": []}} + strings = YAML12_NUMBER_LOOKALIKES + [ + "0307", "0123", "true", "yes", "off", "null", "~", "", " ", ".inf", ".nan", + "1.0", "1_000", "0x1F", "0b101", "12:30", "2026-01-01", "a: b", "- a", "#c", + "&a", "*a", "!tag", "|", ">", "%x", "@x", "'single'", '"double"', "back\\slash", + "line1\nline2", "trailing\n", "tab\there", " lead", "trail ", "\u00e9t\u00e9", + "\U0001F600", "\x7f", "\x85", "\u2028", "\ufeff", "\x00", "\x1b", + "x" * 3000, " ".join(["word"] * 500), + ] + config = {"version": 3, "hives": {"lookup": {"l1": { + "usr_mtd": {"enabled": True, "expiry": 0, "tags": ["0308"], "comment": "1e3"}, + "data": { + "values": strings, + "keyed": {s: i for i, s in enumerate(strings)}, + "long-key-" + "k" * 2000: "v", + "ints": [0, -5, 2**40, 2**63 - 1], + "floats": [0.5, 1.0, -2.25, 1e20, 1e-05], + "bools": [True, False], + "nothing": None, + "nested": [{"a": [{"b": s}]} for s in strings[:5]], + }, + }}}} + configs.push(config, sync_hives={"lookup": True}) + + sent = _sent_push_config(mock_org) + assert yaml.safe_load(sent) == config + nodes = list(_scalar_nodes(yaml.compose(sent))) + str_nodes = [n for n in nodes if n.tag == "tag:yaml.org,2002:str"] + other_nodes = [n for n in nodes if n.tag != "tag:yaml.org,2002:str"] + assert len(str_nodes) > len(strings) * 2 + assert {n.style for n in str_nodes} == {'"'} + assert other_nodes + assert {n.style for n in other_nodes} == {None} + + def test_non_string_scalars_are_written_as_before(self, configs, mock_org): + mock_org.client.request.return_value = {"data": {"ops": []}} + config = {"version": 3, "hives": {"lookup": {"l1": {"data": { + "ints": [0, 7, -5, 2**63 - 1], + "floats": [0.5, 1e20, float("inf")], + "bools": [True, False], + "nothing": None, + "day": datetime.date(2026, 1, 1), + "when": datetime.datetime(2026, 1, 1, 10, 0, 0), + "raw": b"\x00\x01", + }}}}} + configs.push(config, sync_hives={"lookup": True}) + + def non_str(text): + return [(n.tag, n.value, n.style) for n in _scalar_nodes(yaml.compose(text)) + if n.tag != "tag:yaml.org,2002:str"] + + sent = _sent_push_config(mock_org) + assert non_str(sent) == non_str(yaml.safe_dump(config, version=(1, 1))) + assert yaml.safe_load(sent) == config + + def test_push_from_file_keeps_quoted_values_as_strings(self, configs, mock_org): + mock_org.client.request.return_value = {"data": {"ops": []}} + content = """version: 3 +hives: + dr-general: + zero-padded-ids: + usr_mtd: + enabled: false + data: + detect: + op: or + rules: + - op: is + path: event/COMMAND_LINE + value: '0389' + - op: is + path: event/FILE_PATH + value: '1e3' + respond: + - action: report + name: zero-padded-ids + tests: + non_match: + - - event: + A: '00000000000000000000000000000307' + B: '00000000000000000000000000000308' + C: '0123' + D: '0129' + E: '0o17' + N: 42 + routing: + event_type: NEW_PROCESS + this: '00000000000000000000000000000309' +""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "config.yaml") + with open(path, "w") as f: + f.write(content) + configs.push_from_file(path, sync_hives={"dr-general": True}) + + sent = _sent_push_config(mock_org) + assert yaml.safe_load(sent) == yaml.safe_load(content) + for value in ("0389", "1e3", "00000000000000000000000000000308", "0129", "0o17", + "00000000000000000000000000000309"): + assert f'"{value}"' in sent + assert '"N": 42' in sent + + class TestConfigsLoadEffectiveConfig: def test_load_simple_config(self, configs): with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: