diff --git a/CHANGELOG.md b/CHANGELOG.md index 15884dc..cd9bd2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Tests: no invented numbers + +- `drivers/tests/lua_harness/no_invented_numbers.lua` runs a driver twice with different registers, config and clock, then makes every read fail and every Modbus reply short. A number the driver still emits that is equal in both runs and was not read is invented: 0 W, 0 A, 0 % SoC, 50 Hz. A failed read must leave the value nil, so the host sends null. Sourceful's driver registry runs the same file on every publish. +- `drivers/tests/test_no_invented_numbers.py` ratchets against `invented-number-baseline.json`: 44 of 91 drivers carry this debt today. A count that rises fails, a driver not listed must be clean, and a count that falls fails until the file is updated. `make invented-number-report ID=` prints the fields for one driver. + ## tesla_vehicle 0.2.5 - Recover missing, partial and old SoC with a telemetry-only wake and read. diff --git a/Makefile b/Makefile index 6e3feff..d57746e 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ KIND ?= meter LEVEL ?= patch .PHONY: bootstrap new-driver test-driver check boundary \ - refused-write-report absent-register-report \ + refused-write-report absent-register-report invented-number-report \ sync-manifests bump-driver history ftw-baseline ftw-baseline-report \ host-api site watch-upstream-docs @@ -40,6 +40,12 @@ refused-write-report: test -n "$(ID)" ./lua55 drivers/tests/lua_harness/refused_write_probe.lua . "drivers/lua/$(ID).lua" +# Which fields a driver still fills with a number once its device stops +# answering. A failed read has to stay nil, so the host sends null, not 0. +invented-number-report: + test -n "$(ID)" + ./lua55 drivers/tests/lua_harness/no_invented_numbers.lua drivers/tests/lua_harness "drivers/lua/$(ID).lua" + bootstrap: uv sync --frozen --extra package --extra dev diff --git a/drivers/tests/invented-number-baseline.json b/drivers/tests/invented-number-baseline.json new file mode 100644 index 0000000..3cb282b --- /dev/null +++ b/drivers/tests/invented-number-baseline.json @@ -0,0 +1,59 @@ +{ + "_comment": [ + "Fields a driver emits a number for after its device stopped answering,", + "a number that came from no read, config or clock (0 W, 50 Hz, 0 % SoC).", + "", + "This file records debt that already shipped: 44 of the 91 drivers when", + "drivers/tests/test_no_invented_numbers.py was added. The test ratchets:", + "a count that rises fails, a driver not listed must be clean, and a count", + "that falls fails until this file is updated. Shrink it. Never grow it.", + "", + "make invented-number-report ID= prints the fields for one driver." + ], + "drivers": { + "abb_meter": 13, + "abb_terra": 8, + "acrel": 13, + "acuvim": 8, + "alfen": 8, + "alphaess": 10, + "atmoce": 13, + "carlo_gavazzi": 13, + "chint": 13, + "circutor": 13, + "ctek": 9, + "ctek_hybrid": 9, + "ctek_v2": 9, + "deye": 28, + "easee": 4, + "etrel": 8, + "ferroamp_modbus": 18, + "fronius": 11, + "goe": 8, + "goodwe": 24, + "growatt": 20, + "hello": 3, + "janitza": 13, + "keba": 8, + "kostal": 22, + "kstar": 9, + "mennekes": 8, + "schneider_meter": 13, + "schrack_ev": 8, + "siemens_pac": 13, + "sigenergy": 5, + "sma": 28, + "sma_pv": 21, + "socomec": 13, + "sofar": 21, + "solaredge": 20, + "solax": 15, + "solinteg": 27, + "solis": 27, + "solis_string": 21, + "sungrow": 20, + "varta": 5, + "victron": 10, + "wallbox": 2 + } +} diff --git a/drivers/tests/lua_harness/no_invented_numbers.lua b/drivers/tests/lua_harness/no_invented_numbers.lua new file mode 100644 index 0000000..352ea7e --- /dev/null +++ b/drivers/tests/lua_harness/no_invented_numbers.lua @@ -0,0 +1,343 @@ +-- no_invented_numbers.lua -- "null, never 0": a driver must not invent numbers. +-- +-- Usage: lua55 no_invented_numbers.lua +-- Prints one JSON object (see `result` below) on stdout. +-- +-- The rule: once the device stops answering, every measurement a driver +-- emits must be nil. A number the driver emits anyway can only come from +-- one of three inputs -- an earlier reading, the driver config, or the +-- clock -- or it is invented (0 W, 50 Hz, a hard-coded SoC ...). +-- +-- How it is measured. The driver runs twice, A and B. The runs differ in +-- every input a driver can legitimately use: +-- * register values (every register answers FILL_A / FILL_B in phase 1), +-- * numeric config values (CONFIG_A / CONFIG_B), +-- * the clock (host.now_ms / host.millis start at different bases). +-- Each run: driver_init(config), one healthy poll (phase 1: reads answer), +-- then POLLS_FAILING polls where every read fails (phase 2: modbus_read, +-- http_get, writes raise; MQTT, serial and P1 deliver nothing), then +-- POLLS_FAILING polls where every Modbus reply is short (phase 3: half of +-- the requested registers, so a decode of a missing register is nil and a +-- `decode(...) or 0` default shows up as a number that did not change). +-- +-- "Emitting a number" means: a finite Lua number anywhere inside a table +-- passed to host.emit(der_type, payload) during phase 2 or 3, including nested +-- tables and arrays (e.g. mppts[1].V). Strings, booleans, nil, NaN and +-- infinities are not numbers here. host.set_rated_w / set_make / set_sn +-- and logs are not emits and are not checked. +-- +-- A phase-2 number at a payload path is +-- * INVENTED (a violation) when the same value appears at that path in +-- phase 2 (or 3) of both runs and the driver did not read it in phase 1: +-- it changed with none of the inputs, so it came from none of them; +-- * a STALE replay (phase 2 only; reported, not a violation) when it +-- equals what that path carried in phase 1 of the same run and that +-- phase-1 value differed between the runs: read once, now repeated; +-- * DERIVED otherwise (it differs between the runs: config, clock, or +-- the registers a short reply did deliver). +-- Allowed constants: the data model's default SoC window +-- (min_soc_fract 0.05, max_soc_fract 1.0, srcful-data-models 2.2.0), which +-- a battery reports when it cannot read its own. +-- +-- A driver whose driver_init fails in this probe is NOT MEASURED (it would +-- not be polled by a host either); the result says so. + +local harness_dir = arg[1] +local driver_path = arg[2] +if not harness_dir or not driver_path then + io.stderr:write("usage: lua55 no_invented_numbers.lua \n") + os.exit(2) +end + +dofile(harness_dir .. "/host_mock.lua") -- global `host`: decoders, json, logging + +local POLLS_FAILING = 3 + +local FILL_A, FILL_B = 37, 61 -- register value every read returns in phase 1 +local CLOCK_A, CLOCK_B = 1767225600000, 1767312000000 -- now_ms bases (one day apart) +local MILLIS_A, MILLIS_B = 1000, 7777000 -- millis bases + +local function config_for(run) + local a = (run == "A") + return { + -- strings and connection parameters: the same in both runs + sn = "PROBE-001", type = "lua", host = "127.0.0.1", port = 502, + unit_id = 1, slave_id = 1, serial = "PROBE123", gateway_serial = "GW-PROBE", + url = "http://127.0.0.1", topic = "probe", serial_port = "/dev/ttyUSB0", + baud_rate = 9600, encryption_key = "00112233445566778899aabbccddeeff", + auth_key = "00112233445566778899aabbccddeeff", ders = {}, + -- numeric config the registry drivers read: different per run + battery_rated_w = a and 10000 or 7300, + battery_capacity_wh = a and 13500 or 9100, + battery_max_c_rate = a and 0.5 or 0.7, + ffr_slew_rate_pct_per_s = a and 10 or 17, + power_scale = a and 1 or 2, + nominal_w = a and 10000 or 7300, + rated_w = a and 10000 or 7300, + } +end + +--------------------------------------------------------------------------- +-- Probe host: failure model, clock, and the functions host_mock lacks +--------------------------------------------------------------------------- + +local state = { phase = 0, poll = 0, fill = 0, clock = 0, millis = 0, emits = {} } +local unknown_host = {} + +local function deepcopy(v, seen) + if type(v) ~= "table" then return v end + seen = seen or {} + if seen[v] then return seen[v] end + local out = {} + seen[v] = out + for k, x in pairs(v) do out[deepcopy(k, seen)] = deepcopy(x, seen) end + return out +end + +local function fail(what) + error(what .. ": timeout (probe: the device does not answer)", 2) +end + +host.emit = function(der_type, data) + table.insert(state.emits, { + phase = state.phase, poll = state.poll, + der = tostring(der_type), data = deepcopy(data), + }) + return true +end + +host.modbus_read = function(addr, count, kind) + if state.phase == 2 then fail("modbus_read") end + local n = count or 1 + if state.phase == 3 then n = n // 2 end -- a short reply: half the registers + local regs = {} + for i = 0, n - 1 do + local a = addr + i + if a == 40000 then regs[i + 1] = 0x5375 -- SunSpec "Su" + elseif a == 40001 then regs[i + 1] = 0x6e53 -- SunSpec "nS" + else regs[i + 1] = state.fill end + end + return regs +end + +local function write_ok() if state.phase == 2 then fail("write") end return true end +host.modbus_write = write_ok +host.modbus_write_multiple = write_ok +host.modbus_write_multi = write_ok +host.write = write_ok +host.write_fc06 = write_ok +host.write_registers = write_ok + +host.http_get = function() fail("http_get") end +host.http_post = function() fail("http_post") end +host.http_patch = function() fail("http_patch") end +host.mqtt_messages = function() return {} end +host.serial_read = function() return nil end +host.serial_available = function() return 0 end +host.p1_telegram = function() return nil end + +host.now_ms = function() state.clock = state.clock + 50; return state.clock end +host.millis = function() state.millis = state.millis + 50; return state.millis end +host.sleep = function() end +host.control_mode = function() return "" end +host.bus_parked = function() return false end +host.set_model = function() end +host.set_rated_w = function() end +host.set_warmup_s = function() end +host.aes_gcm_decrypt = function() return nil, "probe: no key material" end +host.decode_f32_be = host.decode_f32_be or host.decode_f32 +host.decode_string = host.decode_string or function(regs) + if type(regs) ~= "table" then return "" end + local chars = {} + for _, r in ipairs(regs) do + chars[#chars + 1] = string.char((r >> 8) & 0xFF, r & 0xFF) + end + return (table.concat(chars):gsub("%z+$", "")) +end + +-- Any other host function: record it and return nil, so an unknown helper +-- does not abort the probe. +setmetatable(host, { __index = function(_, k) + if type(k) == "string" and k:sub(1, 1) == "_" then return nil end -- host_mock internals + unknown_host[tostring(k)] = true + return function() return nil end +end }) + +--------------------------------------------------------------------------- +-- Running a driver in a sandbox +--------------------------------------------------------------------------- + +local function read_file(path) + local f = assert(io.open(path, "rb")) + local s = f:read("a") + f:close() + return s +end + +local SOURCE = read_file(driver_path) + +local function sandbox() + local env = { + host = host, string = string, table = table, math = math, utf8 = utf8, + pairs = pairs, ipairs = ipairs, next = next, select = select, + type = type, tostring = tostring, tonumber = tonumber, + pcall = pcall, xpcall = xpcall, error = error, assert = assert, + rawget = rawget, rawset = rawset, rawequal = rawequal, rawlen = rawlen, + setmetatable = setmetatable, getmetatable = getmetatable, + unpack = table.unpack, print = function() end, + } + env._G = env + return env +end + +local function run(name) + state.fill = (name == "A") and FILL_A or FILL_B + state.clock = (name == "A") and CLOCK_A or CLOCK_B + state.millis = (name == "A") and MILLIS_A or MILLIS_B + state.emits = {} + state.phase, state.poll = 0, 0 + host._emitted, host._calls, host._logs = {}, {}, {} + + local env = sandbox() + local chunk, err = load(SOURCE, "=driver", "t", env) + if not chunk then return { ok = false, reason = "load: " .. tostring(err) } end + local ok, lerr = pcall(chunk) + if not ok then return { ok = false, reason = "load: " .. tostring(lerr) } end + if type(env.driver_init) ~= "function" or type(env.driver_poll) ~= "function" then + return { ok = false, reason = "driver_init/driver_poll not defined" } + end + + state.phase = 1 + local iok, iret = pcall(env.driver_init, config_for(name)) + if not iok then return { ok = false, reason = "driver_init failed: " .. tostring(iret) } end + if iret == false then return { ok = false, reason = "driver_init returned false" } end + + local errors = {} + state.poll = 1 + local pok, perr = pcall(env.driver_poll) + if not pok then errors[#errors + 1] = "healthy poll: " .. tostring(perr) end + + for _, phase in ipairs({ 2, 3 }) do + state.phase = phase + for i = 1, POLLS_FAILING do + state.poll = i + local fok, ferr = pcall(env.driver_poll) + if not fok then + errors[#errors + 1] = "phase " .. phase .. " poll " .. i .. ": " .. tostring(ferr) + end + end + end + return { ok = true, emits = state.emits, errors = errors } +end + +--------------------------------------------------------------------------- +-- Collect numbers per payload path and classify +--------------------------------------------------------------------------- + +local function finite(v) + return type(v) == "number" and v == v and v ~= math.huge and v ~= -math.huge +end + +local function walk(prefix, v, out) + if type(v) == "table" then + local keys = {} + for k in pairs(v) do keys[#keys + 1] = k end + table.sort(keys, function(x, y) return tostring(x) < tostring(y) end) + for _, k in ipairs(keys) do + local seg = (type(k) == "number") and ("[" .. k .. "]") or ("." .. tostring(k)) + walk(prefix .. seg, v[k], out) + end + elseif finite(v) then + out[#out + 1] = { path = prefix, value = v } + end +end + +-- numbers[phase][path] = { [value] = first poll it appeared in } +local function numbers(emits) + local by = { [1] = {}, [2] = {}, [3] = {} } + for _, e in ipairs(emits) do + local leaves = {} + walk(e.der, e.data, leaves) + for _, l in ipairs(leaves) do + local t = by[e.phase] + if t then + t[l.path] = t[l.path] or {} + if t[l.path][l.value] == nil then t[l.path][l.value] = e.poll end + end + end + end + return by +end + +local result = { + measured = false, reason = nil, + invented = {}, stale = {}, + unknown_host_functions = {}, errors = {}, +} + +local A = run("A") +local B = run("B") + +if not A.ok or not B.ok then + result.reason = (not A.ok) and A.reason or B.reason +else + result.measured = true + for _, e in ipairs(A.errors) do result.errors[#result.errors + 1] = "A " .. e end + for _, e in ipairs(B.errors) do result.errors[#result.errors + 1] = "B " .. e end + local na, nb = numbers(A.emits), numbers(B.emits) + local DEFAULTS = { min_soc_fract = 0.05, max_soc_fract = 1.0 } + local function is_default(field, v) + local d = DEFAULTS[field] + return d ~= nil and math.abs(v - d) < 1e-6 + end + local function split(p) + local i = p:find("[%.%[]") + if not i then return p, "" end + local field = p:sub(i) + if field:sub(1, 1) == "." then field = field:sub(2) end + return p:sub(1, i - 1), field + end + local seen = {} + for _, phase in ipairs({ 2, 3 }) do + local paths = {} + for p in pairs(na[phase]) do paths[#paths + 1] = p end + table.sort(paths) + for _, p in ipairs(paths) do + local vb = nb[phase][p] or {} + local healthyA, healthyB = na[1][p] or {}, nb[1][p] or {} + -- phase-1 values that changed with the register fill were read + local healthy_was_read = next(healthyA) ~= nil and next(healthyB) ~= nil + if healthy_was_read then + for v in pairs(healthyA) do + if healthyB[v] ~= nil then healthy_was_read = false end + end + end + local der, field = split(p) + for v, poll in pairs(na[phase][p]) do + local key = phase .. "|" .. p .. "|" .. tostring(v) + if not seen[key] and not is_default(field, v) then + seen[key] = true + local row = { der = der, field = field, value = v, poll = poll, phase = phase } + -- phase 3 re-reads the delivered half: a phase-1 value there + -- is a fresh read, not a replay, so only phase 2 is stale + if vb[v] ~= nil then + if healthy_was_read and healthyA[v] ~= nil then + if phase == 2 then result.stale[#result.stale + 1] = row end + else + result.invented[#result.invented + 1] = row + end + elseif phase == 2 and healthyA[v] ~= nil then + result.stale[#result.stale + 1] = row + end + end + end + end + end +end + +for k in pairs(unknown_host) do + result.unknown_host_functions[#result.unknown_host_functions + 1] = k +end +table.sort(result.unknown_host_functions) + +io.write(host.json_encode(result), "\n") diff --git a/drivers/tests/test_no_invented_numbers.py b/drivers/tests/test_no_invented_numbers.py new file mode 100644 index 0000000..5270af4 --- /dev/null +++ b/drivers/tests/test_no_invented_numbers.py @@ -0,0 +1,99 @@ +"""A device that stops answering gets nil from its driver, never a number. + +`local w = 0` before a read, or `decode(...) or 0` after one, turns a failed +read into a measurement: 0 W, 0 A, 0 % SoC, 50 Hz. Nothing downstream can tell +it from a real reading. A meter at 0 W looks like a site in balance; a battery +at 0 % looks empty and ready to charge. A missing value has to stay missing, so +the host sends null. + +`lua_harness/no_invented_numbers.lua` measures it. It runs a driver twice, with +different register values, numeric config and clock, then makes every read +fail, then makes every Modbus reply short. A number the driver still emits +that is equal in both runs, and was not read, came from none of its inputs: it +was invented. The data model's default SoC window (0.05 / 1.0) is the one +constant allowed. Sourceful's driver registry runs the same file on every +publish and refuses a driver that fails it. + +## Why a baseline + +When the probe was first run here, 44 of 91 drivers invented at least one +number. As in `test_absent_register_settles.py`, the counts sit in +`invented-number-baseline.json` and this test ratchets: + +* a driver whose count goes **up** fails; +* a driver **absent from the baseline** must be clean, so new drivers get the + rule in full; +* a driver whose count goes **down** also fails, asking for the baseline to be + updated, so the debt cannot be re-borrowed. + +Shrink the baseline. Never grow it. +""" + +import json +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +LUA = ROOT / "lua55" +HARNESS = ROOT / "drivers" / "tests" / "lua_harness" +PROBE = HARNESS / "no_invented_numbers.lua" +DRIVERS = ROOT / "drivers" / "lua" +BASELINE = Path(__file__).parent / "invented-number-baseline.json" + +pytestmark = pytest.mark.skipif( + not LUA.exists(), reason="run make check to build ./lua55") + + +def load_baseline() -> dict: + return json.loads(BASELINE.read_text())["drivers"] + + +def probe(driver: str) -> tuple[list[str], str | None]: + """Return the fields this driver invents a number for, or why it was not measured.""" + result = subprocess.run( + [str(LUA), str(PROBE), str(HARNESS), str(DRIVERS / f"{driver}.lua")], + capture_output=True, text=True, cwd=ROOT) + assert result.returncode == 0, result.stdout + result.stderr + report = json.loads(result.stdout) + if not report["measured"]: + return [], report["reason"] + return sorted({f"{row['der']}.{row['field']}" for row in report["invented"]}), None + + +def driver_names() -> list[str]: + return sorted(p.stem for p in DRIVERS.glob("*.lua")) + + +@pytest.mark.parametrize("driver", driver_names()) +def test_invented_number_debt_does_not_grow(driver: str) -> None: + baseline = load_baseline() + invented, not_measured = probe(driver) + if not_measured is not None: + # driver_init failed with probe config, so a host would not poll it either. + assert driver not in baseline, ( + f"{driver} is in the baseline but can no longer be measured " + f"({not_measured}). Remove it from {BASELINE.name}.") + return + + expected = baseline.get(driver, 0) + found = len(invented) + + if driver not in baseline: + assert found == 0, ( + f"{driver} emits numbers it never read once the device stops " + f"answering: {invented}. Leave a value nil when its read failed " + f"or came back short, so the host sends null instead of a fake 0.") + return + + assert found <= expected, ( + f"{driver} went from {expected} to {found} fields it invents a " + f"number for: {invented}.") + + assert found == expected, ( + f"{driver} is down to {found} from {expected}. Set it to {found} in " + f"{BASELINE.name} so the debt cannot be re-borrowed." + if found else + f"{driver} is clean now. Remove it from {BASELINE.name} so it is held " + f"to the rule in full.")