From 152205cc6f0ddd59a81b19aa08c74d17bd5bb2ca Mon Sep 17 00:00:00 2001 From: fengzero Date: Mon, 9 Nov 2026 07:42:32 +0000 Subject: [PATCH 1/2] export driver tags with unit --- plugins/restful/adapter_handle.c | 2 + plugins/restful/global_config_handle.c | 7 ++- tests/ft/http_api/test_http_api.py | 71 ++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/plugins/restful/adapter_handle.c b/plugins/restful/adapter_handle.c index 465d2bcbb..0a7bc9683 100644 --- a/plugins/restful/adapter_handle.c +++ b/plugins/restful/adapter_handle.c @@ -550,6 +550,7 @@ static int json_to_gdatatags(neu_json_gtag_array_t *gtag_array, gdatatags[i].tags[j].bias = gtag_array->gtags[i].tags[j].bias; gdatatags[i].tags[j].address = gtag_array->gtags[i].tags[j].address; gdatatags[i].tags[j].name = gtag_array->gtags[i].tags[j].name; + gdatatags[i].tags[j].unit = gtag_array->gtags[i].tags[j].unit; if (gtag_array->gtags[i].tags[j].description != NULL) { gdatatags[i].tags[j].description = gtag_array->gtags[i].tags[j].description; @@ -560,6 +561,7 @@ static int json_to_gdatatags(neu_json_gtag_array_t *gtag_array, gtag_array->gtags[i].tags[j].address = NULL; // moved gtag_array->gtags[i].tags[j].name = NULL; // moved gtag_array->gtags[i].tags[j].description = NULL; // moved + gtag_array->gtags[i].tags[j].unit = NULL; // moved } } diff --git a/plugins/restful/global_config_handle.c b/plugins/restful/global_config_handle.c index 152b7ea88..49aed814b 100644 --- a/plugins/restful/global_config_handle.c +++ b/plugins/restful/global_config_handle.c @@ -1228,10 +1228,12 @@ static int get_driver_tags_resp(context_t *ctx, neu_resp_get_tag_t *tags) gtag->tags[index].precision = tag->precision; gtag->tags[index].decimal = tag->decimal; gtag->tags[index].bias = tag->bias; + gtag->tags[index].unit = tag->unit; gtag->tags[index].t = NEU_JSON_UNDEFINE; tag->name = NULL; // moved tag->address = NULL; // moved tag->description = NULL; // moved + tag->unit = NULL; // moved } end: @@ -1500,9 +1502,10 @@ static int add_tag(context_t *ctx, neu_json_add_tags_req_t *data) cmd.tags[i].name = strdup(data->tags[i].name); cmd.tags[i].description = strdup(data->tags[i].description ? data->tags[i].description : ""); + cmd.tags[i].unit = strdup(data->tags[i].unit ? data->tags[i].unit : ""); if (NULL == cmd.tags[i].address || NULL == cmd.tags[i].name || - NULL == cmd.tags[i].description) { + NULL == cmd.tags[i].description || NULL == cmd.tags[i].unit) { ret = NEU_ERR_EINTERNAL; goto error; } @@ -1516,7 +1519,7 @@ static int add_tag(context_t *ctx, neu_json_add_tags_req_t *data) return ret; error: - for (int j = 0; j < i && j < cmd.n_tag; ++j) { + for (int j = 0; j < cmd.n_tag; ++j) { neu_tag_fini(&cmd.tags[j]); } free(cmd.tags); diff --git a/tests/ft/http_api/test_http_api.py b/tests/ft/http_api/test_http_api.py index db1cce083..91af4f6b8 100644 --- a/tests/ft/http_api/test_http_api.py +++ b/tests/ft/http_api/test_http_api.py @@ -292,6 +292,77 @@ def test_get_global_config(self): for node in global_config['nodes']: api.del_node(node['name']) + @pytest.mark.parametrize("unit", ["°C", "", None], ids=["nonempty", "empty", "missing"]) + def test_global_config_tag_unit_roundtrip(self, unit): + driver = copy.deepcopy(driver_1) + group = driver["groups"][0] + tags = group["tags"] + if unit is None: + tags[0].pop("unit") + else: + tags[0]["unit"] = unit + tags[1].pop("unit") + expected_tags = copy.deepcopy(tags) + for tag in expected_tags: + tag.setdefault("unit", "") + + data = { + "nodes": [{"name": driver["name"], "plugin": driver["plugin"]}], + "groups": [{"driver": driver["name"], "group": group["group"], + "interval": group["interval"], "tag_count": len(tags)}], + "tags": [{"driver": driver["name"], "group": group["group"], "tags": tags}], + "subscriptions": [], + "settings": [] + } + try: + for _ in range(2): + response = api.put_global_config(json=data) + assert 200 == response.status_code + assert error.NEU_ERR_SUCCESS == response.json()["error"] + + response = api.get_tags(driver["name"], group["group"]) + assert 200 == response.status_code + assert expected_tags == response.json()["tags"] + + response = api.get_global_config() + assert 200 == response.status_code + data = response.json() + exported = next(g for g in data["tags"] + if g["driver"] == driver["name"] and g["group"] == group["group"]) + assert expected_tags == exported["tags"] + finally: + api.del_node(driver["name"]) + + @pytest.mark.parametrize("unit", ["°C", "", None], ids=["nonempty", "empty", "missing"]) + def test_driver_tag_unit_roundtrip(self, unit): + driver = copy.deepcopy(driver_1) + tags = driver["groups"][0]["tags"] + if unit is None: + tags[0].pop("unit") + else: + tags[0]["unit"] = unit + tags[1].pop("unit") + expected = copy.deepcopy(driver) + for tag in expected["groups"][0]["tags"]: + tag.setdefault("unit", "") + + try: + for _ in range(2): + response = api.put_drivers([driver]) + assert 200 == response.status_code + assert error.NEU_ERR_SUCCESS == response.json()["error"] + + response = api.get_tags(driver["name"], driver["groups"][0]["group"]) + assert 200 == response.status_code + assert expected["groups"][0]["tags"] == response.json()["tags"] + + response = api.get_driver(driver["name"]) + assert 200 == response.status_code + assert [expected] == response.json()["nodes"] + driver = response.json()["nodes"][0] + finally: + api.del_node(driver["name"]) + @description(given="running neuron", when="put drivers with too long node name", then="should fail") def test_put_drivers_with_long_node_name(self): response = api.put_driver( From da2ca562cb32620fcc2a18787b3a68c79dad090e Mon Sep 17 00:00:00 2001 From: fengzero Date: Tue, 10 Nov 2026 08:03:58 +0000 Subject: [PATCH 2/2] api: get tags with quality & timestamp --- CMakeLists.txt | 1 + docs/api/cn/http.md | 117 ++++++++++++ docs/api/english/http.md | 117 ++++++++++++ plugins/restful/global_config_handle.c | 1 + plugins/restful/read_response.c | 80 ++++++++ plugins/restful/read_response.h | 17 ++ plugins/restful/rw_handle.c | 38 +++- tests/ft/auto_type_http.py | 96 ++++++++-- tests/plugins/auto_type/auto_type.c | 18 +- tests/ut/CMakeLists.txt | 10 + tests/ut/http_read_attrs_test.cc | 255 +++++++++++++++++++++++++ 11 files changed, 724 insertions(+), 26 deletions(-) create mode 100644 plugins/restful/read_response.c create mode 100644 plugins/restful/read_response.h create mode 100644 tests/ut/http_read_attrs_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index bc5a61717..be63c8b1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -172,6 +172,7 @@ set(NEURON_SOURCES plugins/restful/metric_handle.c plugins/restful/normal_handle.c plugins/restful/rw_handle.c + plugins/restful/read_response.c plugins/restful/adapter_handle.c plugins/restful/datatag_handle.c plugins/restful/global_config_handle.c diff --git a/docs/api/cn/http.md b/docs/api/cn/http.md index a9777bede..3ede6496f 100644 --- a/docs/api/cn/http.md +++ b/docs/api/cn/http.md @@ -1235,6 +1235,123 @@ HTTP 200 响应： } ``` +## 读取点位值 + +*POST* /api/v2/read + +### 请求头部 + +**Content-Type** application/json + +**Authorization** Bearer \ + +### 请求体 + +```json +{ + "node": "modbus-tcp-1", + "group": "group-1", + "sync": false +} +``` + +`sync` 为可选布尔值，默认为 `false`，读取缓存；`true` 请求同步读取。 + +### 响应（200 OK） + +```json +{ + "tags": [ + { + "name": "temperature", + "value": 25.5, + "transferPrecision": 2, + "timestamp": 1700000000000, + "quality": 192, + "source_timestamp": 1700000000123, + "server_timestamp": 1700000000456 + }, + { + "name": "pressure", + "error": 3002, + "quality": 0 + } + ] +} +``` + +成功点位返回 `value`，错误点位返回 `error`；浮点值附带 `transferPrecision`。以下采样属性对成功和错误点位均可用，位于点位对象顶层： + +| 字段 | 类型 | 说明 | +| --- | --- | --- | +| timestamp | 整数（int64） | 驱动提供的原有点位时间戳，原样传递，不转换单位 | +| quality | 整数（int32） | 驱动提供的质量值 | +| source_timestamp | 整数（int64） | 源时间戳，Unix epoch 毫秒 | +| server_timestamp | 整数（int64） | 服务器时间戳，Unix epoch 毫秒 | + +四个字段独立可选：仅在当前样本具有对应属性时返回，显式 `0` 会保留；缺失时省略，不返回 `null`，不生成时间戳，也不使用其他时间戳替代。 + +原有点位元数据（metas）仍在点位对象顶层展开。如果与采样属性同名，存在的采样属性覆盖同名元数据；采样属性缺失时保留原元数据。 + +## 分页读取点位值 + +*POST* /api/v2/read/paginate + +请求头部与读取点位值接口相同。 + +### 请求体 + +```json +{ + "node": "modbus-tcp-1", + "group": "group-1", + "sync": false, + "query": { + "currentPage": 1, + "pageSize": 10, + "isError": false + } +} +``` + +`query` 可选。分页参数位于 `query` 中，`currentPage` 从 1 开始；`isError: true` 仅返回错误点位，`false` 不启用错误筛选。 + +### 响应（200 OK） + +```json +{ + "meta": { + "currentPage": 1, + "pageSize": 10, + "total": 1 + }, + "items": [ + { + "name": "temperature", + "type": 10, + "address": "1!400001", + "attribute": 1, + "description": "", + "precision": 2, + "decimal": 0, + "bias": 0, + "unit": "°C", + "value": 25.5, + "transferPrecision": 2, + "timestamp": 1700000000000, + "quality": 192, + "source_timestamp": 1700000000123, + "server_timestamp": 1700000000456, + "attributes": {} + } + ] +} +``` + +`items[]` 中四个可选采样属性的规则与 `/api/v2/read` 相同。原有 metas 放在 `attributes` 对象中，与顶层采样属性独立；`attributes` 不是采样属性容器。`type` 仍为配置类型（AUTO 为 `0`），不因运行时识别而改变。空结果或超出范围的页返回空 `items` 数组。 + +这些采样属性属于读取结果，`GET /api/v2/tags` 的配置响应不增加这些字段。 + ## 添加插件 *POST* /api/v2/plugin diff --git a/docs/api/english/http.md b/docs/api/english/http.md index c26cc8a7f..9894f029e 100644 --- a/docs/api/english/http.md +++ b/docs/api/english/http.md @@ -1216,6 +1216,123 @@ To update both group name and interval: } ``` +## Read Tag Values + +*POST* /api/v2/read + +### Request Headers + +**Content-Type** application/json + +**Authorization** Bearer \ + +### Body + +```json +{ + "node": "modbus-tcp-1", + "group": "group-1", + "sync": false +} +``` + +`sync` is an optional boolean, defaulting to `false` to read cached values. Set it to `true` to request a synchronous read. + +### Response (200 OK) + +```json +{ + "tags": [ + { + "name": "temperature", + "value": 25.5, + "transferPrecision": 2, + "timestamp": 1700000000000, + "quality": 192, + "source_timestamp": 1700000000123, + "server_timestamp": 1700000000456 + }, + { + "name": "pressure", + "error": 3002, + "quality": 0 + } + ] +} +``` + +A successful tag returns `value`; an error tag returns `error`. Floating-point values also include `transferPrecision`. Both successful and error samples may include these top-level sample attributes: + +| Field | Type | Description | +| --- | --- | --- | +| timestamp | Integer (int64) | Existing driver-provided tag timestamp, passed through without unit conversion | +| quality | Integer (int32) | Driver-provided quality value | +| source_timestamp | Integer (int64) | Source timestamp in Unix epoch milliseconds | +| server_timestamp | Integer (int64) | Server timestamp in Unix epoch milliseconds | + +The four fields are independently optional. A field is returned only when the current sample supplies it; explicit `0` is preserved. Missing fields are omitted, not returned as `null`. No timestamps are generated or substituted from another timestamp field. + +Existing tag metadata (metas) remains flattened into the tag object. A present sample attribute replaces a metadata field with the same name; when the sample attribute is absent, the original metadata field is preserved. + +## Read Tag Values with Pagination + +*POST* /api/v2/read/paginate + +Request headers are the same as for reading tag values. + +### Body + +```json +{ + "node": "modbus-tcp-1", + "group": "group-1", + "sync": false, + "query": { + "currentPage": 1, + "pageSize": 10, + "isError": false + } +} +``` + +`query` is optional. Pagination parameters belong inside `query`, and `currentPage` starts at 1. Set `isError` to `true` to return only error tags; `false` does not enable error filtering. + +### Response (200 OK) + +```json +{ + "meta": { + "currentPage": 1, + "pageSize": 10, + "total": 1 + }, + "items": [ + { + "name": "temperature", + "type": 10, + "address": "1!400001", + "attribute": 1, + "description": "", + "precision": 2, + "decimal": 0, + "bias": 0, + "unit": "°C", + "value": 25.5, + "transferPrecision": 2, + "timestamp": 1700000000000, + "quality": 192, + "source_timestamp": 1700000000123, + "server_timestamp": 1700000000456, + "attributes": {} + } + ] +} +``` + +The four optional sample attributes in `items[]` follow the same rules as `/api/v2/read`. Existing metas remain inside `attributes`, separate from the top-level sample attributes; `attributes` is not a sample-attribute container. `type` remains the configured type (including `0` for AUTO), not the inferred runtime type. Empty results and out-of-range pages return an empty `items` array. + +These sample attributes belong to read responses; the configuration response from `GET /api/v2/tags` is unchanged. + ## Add Plugin *POST* /api/v2/plugin diff --git a/plugins/restful/global_config_handle.c b/plugins/restful/global_config_handle.c index 49aed814b..7467fcd53 100644 --- a/plugins/restful/global_config_handle.c +++ b/plugins/restful/global_config_handle.c @@ -1148,6 +1148,7 @@ static int get_tags_resp(context_t *ctx, neu_resp_get_tag_t *tags) tags_res.tags[index].precision = tag->precision; tags_res.tags[index].decimal = tag->decimal; tags_res.tags[index].bias = tag->bias; + tags_res.tags[index].unit = tag->unit; tags_res.tags[index].t = NEU_JSON_UNDEFINE; } diff --git a/plugins/restful/read_response.c b/plugins/restful/read_response.c new file mode 100644 index 000000000..c4d45bf4d --- /dev/null +++ b/plugins/restful/read_response.c @@ -0,0 +1,80 @@ +#include "msg.h" + +#include "read_response.h" + +static int encode_attrs(json_t *object, const neu_tag_sample_attrs_t *attrs) +{ + const struct { + const char *name; + bool present; + int64_t value; + } fields[] = { + { "timestamp", attrs->has_timestamp, attrs->timestamp }, + { "quality", attrs->has_quality, attrs->quality }, + { "source_timestamp", attrs->has_source_timestamp, + attrs->source_timestamp }, + { "server_timestamp", attrs->has_server_timestamp, + attrs->server_timestamp }, + }; + + for (size_t i = 0; i < sizeof(fields) / sizeof(fields[0]); i++) { + if (fields[i].present && + json_object_set_new(object, fields[i].name, + json_integer(fields[i].value)) != 0) { + return -1; + } + } + return 0; +} + +int neu_http_encode_read_response(void *json_object, void *param) +{ + neu_http_read_response_t *ctx = param; + json_t * object = json_object; + if (ctx == NULL || ctx->response == NULL) { + return -1; + } + + int count = ctx->paginate + ? ((neu_json_read_paginate_resp_t *) ctx->response)->n_tag + : ((neu_json_read_resp_t *) ctx->response)->n_tag; + unsigned int length = ctx->tags ? utarray_len(ctx->tags) : 0; + if (!json_is_object(object) || count < 0 || + (count > 0 && + (ctx->offset > length || + (unsigned int) count > length - ctx->offset))) { + if (ctx->paginate) { + neu_json_read_paginate_resp_t *response = ctx->response; + for (int i = 0; response->tags && i < count; i++) { + free(response->tags[i].datatag.name); + free(response->tags[i].datatag.address); + free(response->tags[i].datatag.description); + free(response->tags[i].datatag.unit); + } + } + return -1; + } + + int ret = ctx->paginate + ? neu_json_encode_read_paginate_resp(json_object, ctx->response) + : neu_json_encode_read_resp(json_object, ctx->response); + if (ret != 0) { + return ret; + } + + json_t *items = + json_object_get(json_object, ctx->paginate ? "items" : "tags"); + if (!json_is_array(items) || json_array_size(items) != (size_t) count) { + return -1; + } + for (int i = 0; i < count; i++) { + void *tag = utarray_eltptr(ctx->tags, ctx->offset + (unsigned int) i); + const neu_tag_sample_attrs_t *attrs = ctx->paginate + ? &((neu_resp_tag_value_meta_paginate_t *) tag)->value.attrs + : &((neu_resp_tag_value_meta_t *) tag)->value.attrs; + if (encode_attrs(json_array_get(items, i), attrs) != 0) { + return -1; + } + } + return 0; +} diff --git a/plugins/restful/read_response.h b/plugins/restful/read_response.h new file mode 100644 index 000000000..bb0b03293 --- /dev/null +++ b/plugins/restful/read_response.h @@ -0,0 +1,17 @@ +#ifndef _NEU_REST_READ_RESPONSE_H_ +#define _NEU_REST_READ_RESPONSE_H_ + +#include + +#include "utils/utarray.h" + +typedef struct { + void * response; + UT_array * tags; + unsigned int offset; + bool paginate; +} neu_http_read_response_t; + +int neu_http_encode_read_response(void *json_object, void *param); + +#endif diff --git a/plugins/restful/rw_handle.c b/plugins/restful/rw_handle.c index 42e7ea661..f61a1a903 100644 --- a/plugins/restful/rw_handle.c +++ b/plugins/restful/rw_handle.c @@ -27,6 +27,7 @@ #include "utils/http.h" #include "utils/time.h" +#include "read_response.h" #include "rw_handle.h" #include "otel/otel_manager.h" @@ -866,13 +867,22 @@ void handle_read_resp(nng_aio *aio, neu_resp_read_group_t *resp) index += 1; } - neu_json_encode_by_fn(&api_res, neu_json_encode_read_resp, &result); + neu_http_read_response_t context = { .response = &api_res, + .tags = resp->tags }; + int ret = + neu_json_encode_by_fn(&context, neu_http_encode_read_response, &result); for (int i = 0; i < api_res.n_tag; i++) { if (api_res.tags[i].n_meta > 0) { free(api_res.tags[i].metas); } } - neu_http_ok(aio, result); + if (ret != 0 || result == NULL) { + NEU_JSON_RESPONSE_ERROR(NEU_ERR_EINTERNAL, { + neu_http_response(aio, NEU_ERR_EINTERNAL, result_error); + }); + } else { + neu_http_ok(aio, result); + } free(api_res.tags); free(result); } @@ -880,9 +890,10 @@ void handle_read_resp(nng_aio *aio, neu_resp_read_group_t *resp) void handle_read_paginate_resp(nng_aio * aio, neu_resp_read_group_paginate_t *resp) { - neu_json_read_paginate_resp_t api_res = { 0 }; - char * result = NULL; - unsigned int total_tags = resp->total_count; + neu_json_read_paginate_resp_t api_res = { 0 }; + char * result = NULL; + unsigned int total_tags = resp->total_count; + unsigned int source_offset = 0; if (resp->is_error) { UT_array *filtered_tags; @@ -933,6 +944,7 @@ void handle_read_paginate_resp(nng_aio * aio, unsigned int start_index = (resp->current_page - 1) * resp->page_size; unsigned int end_index = start_index + resp->page_size; + source_offset = start_index; if (start_index >= total_tags) { api_res.n_tag = 0; @@ -1000,8 +1012,12 @@ void handle_read_paginate_resp(nng_aio * aio, api_res.meta.page_size = resp->page_size; api_res.meta.total = total_tags; - neu_json_encode_by_fn(&api_res, neu_json_encode_read_paginate_resp, - &result); + neu_http_read_response_t context = { .response = &api_res, + .tags = resp->tags, + .offset = source_offset, + .paginate = true }; + int ret = + neu_json_encode_by_fn(&context, neu_http_encode_read_response, &result); for (unsigned int i = 0; i < utarray_len(resp->tags); ++i) { neu_resp_tag_value_meta_paginate_t *final_tag_value; @@ -1020,7 +1036,13 @@ void handle_read_paginate_resp(nng_aio * aio, free(api_res.tags[i].metas); } } - neu_http_ok(aio, result); + if (ret != 0 || result == NULL) { + NEU_JSON_RESPONSE_ERROR(NEU_ERR_EINTERNAL, { + neu_http_response(aio, NEU_ERR_EINTERNAL, result_error); + }); + } else { + neu_http_ok(aio, result); + } free(api_res.tags); free(result); } diff --git a/tests/ft/auto_type_http.py b/tests/ft/auto_type_http.py index 7e36c3840..bcc9e2f3b 100644 --- a/tests/ft/auto_type_http.py +++ b/tests/ft/auto_type_http.py @@ -9,8 +9,9 @@ Mock params replace the previous settings: hold (default true), concrete numeric NEU_TYPE type (default INT16), value (default 123), error (default 0), update_mode -(update/meta/trace/im/im_f_m), pointer (STRING/BYTES wrapper), optional quality and -timestamp, record_path (append JSONL writes: tag, actual type, union value, path). +(update/meta/trace/im/im_f_m), pointer (STRING/BYTES wrapper), optional quality, +timestamp, source_timestamp, server_timestamp, record_path (append JSONL writes: +tag, actual type, union value, path). All readable tags receive the selected sample; group_sync uses the same controls. """ @@ -33,6 +34,7 @@ REPO = Path(__file__).resolve().parents[2] BUILD = REPO / "build" KEEP_RUNTIME = False +SAMPLE_ATTRS = ("timestamp", "quality", "source_timestamp", "server_timestamp") class AutoTypeHTTP(unittest.TestCase): @@ -177,23 +179,27 @@ def tag(name="a", **fields): return dict({"name": name, "type": 0, "address": "sample", "attribute": 3, "precision": 0, "decimal": 0}, **fields) - def read(self, name="a", *, group="g", paginate=False, sync=False): - response = self.http("POST", "/read/paginate" if paginate else "/read", - {"node": self.node, "group": group, "sync": sync}) + def read(self, name="a", *, group="g", paginate=False, sync=False, query=None): + payload = {"node": self.node, "group": group, "sync": sync} + if query is not None: + payload["query"] = query + response = self.http("POST", "/read/paginate" if paginate else "/read", payload) return next(tag for tag in response["items" if paginate else "tags"] if tag["name"] == name) - def await_sample(self, value=None, error=None, **kwargs): + def await_sample(self, value=None, error=None, attrs=None, **kwargs): deadline = time.monotonic() + 8 actual = None while time.monotonic() < deadline: actual = self.read(**kwargs) - if error is not None and actual.get("error") == error: - return actual - if error is None and "error" not in actual and actual.get("value") == value: + sample_attrs = {name: actual[name] for name in SAMPLE_ATTRS if name in actual} + matches = (actual.get("error") == error if error is not None else + "error" not in actual and actual.get("value") == value) + if matches and (attrs is None or sample_attrs == attrs): return actual time.sleep(0.05) - self.fail(f"sample did not become value={value!r}, error={error!r}: {actual}") + self.fail(f"sample did not become value={value!r}, error={error!r}, " + f"attrs={attrs!r}: {actual}") def write(self, value, *, name="a", error=0): status, response = self.request("POST", "/write", { @@ -265,13 +271,79 @@ def test_all_update_callbacks(self): name = f"mode{index}" self.add_tags(self.tag(name)) self.await_sample(error=3011, name=name) - self.control(update_mode=mode) - self.await_sample(123, name=name) + attrs = {"timestamp": 1700000000000, "quality": 192, + "source_timestamp": 1700000000123, + "server_timestamp": 1700000000456} + self.control(update_mode=mode, **attrs) + self.await_sample(123, name=name, attrs=attrs) + self.await_sample(123, name=name, attrs=attrs, paginate=True) self.control(type=10, value=1.5, update_mode=mode) self.await_sample(error=3007, name=name) self.control(value=125, update_mode=mode) self.await_sample(125, name=name) + def test_optional_sample_attrs(self): + self.add_tags(self.tag(), self.tag("explicit", type=3)) + fields = {"timestamp": 1700000000000, "quality": 192, + "source_timestamp": 1700000000123, "server_timestamp": 1700000000456} + for flags in range(16): + attrs = {key: fields[key] for index, key in enumerate(SAMPLE_ATTRS) + if flags & (1 << index)} + with self.subTest(flags=flags): + self.control(**attrs) + for name in ("a", "explicit"): + for paginate in (False, True): + self.await_sample(123, name=name, attrs=attrs, paginate=paginate) + self.await_sample(123, attrs=attrs, sync=True) + zeros = dict.fromkeys(SAMPLE_ATTRS, 0) + self.control(**zeros) + for paginate in (False, True): + self.await_sample(123, attrs=zeros, paginate=paginate) + self.control(error=3002, **fields) + for paginate in (False, True): + self.await_sample(error=3002, attrs=fields, paginate=paginate) + self.control(error=3002) + for paginate in (False, True): + self.await_sample(error=3002, attrs={}, paginate=paginate) + self.control() + for paginate in (False, True): + self.await_sample(123, attrs={}, paginate=paginate) + + def test_sample_attrs_pagination(self): + self.add_tags(self.tag("a"), self.tag("b"), self.tag("c")) + attrs = {"quality": 0, "source_timestamp": 1700000000123} + self.control(**attrs) + self.await_sample(123, attrs=attrs) + self.control(hold=True) + request = {"node": self.node, "group": "g"} + full = self.http("POST", "/read/paginate", request)["items"] + for index, expected in enumerate(full): + page = self.http("POST", "/read/paginate", dict(request, query={ + "currentPage": index + 1, "pageSize": 1})) + self.assertEqual([expected["name"]], [tag["name"] for tag in page["items"]]) + self.assertEqual(attrs, {key: page["items"][0][key] for key in SAMPLE_ATTRS + if key in page["items"][0]}) + self.control(error=3002, **attrs) + for name in ("a", "b", "c"): + self.await_sample(error=3002, attrs=attrs, name=name) + self.control(hold=True) + errors = self.http("POST", "/read/paginate", dict(request, query={"isError": True})) + self.assertEqual(3, errors["meta"]["total"]) + for index, expected in enumerate(errors["items"]): + page = self.http("POST", "/read/paginate", dict(request, query={ + "isError": True, "currentPage": index + 1, "pageSize": 1})) + self.assertEqual([expected], page["items"]) + for is_error in (False, True): + empty = self.http("POST", "/read/paginate", dict(request, query={ + "isError": is_error, "currentPage": 8, "pageSize": 1})) + self.assertEqual([], empty["items"]) + self.control() + for name in ("a", "b", "c"): + self.await_sample(123, attrs={}, name=name) + empty = self.http("POST", "/read/paginate", dict(request, query={"isError": True})) + self.assertEqual([], empty["items"]) + self.assertEqual(0, empty["meta"]["total"]) + def test_scaled_value_and_byte_order(self): self.add_tags(self.tag(decimal=0.5, bias=1), self.tag("big", address="sample#B")) self.control(value=256) diff --git a/tests/plugins/auto_type/auto_type.c b/tests/plugins/auto_type/auto_type.c index 8a02fefca..134123914 100644 --- a/tests/plugins/auto_type/auto_type.c +++ b/tests/plugins/auto_type/auto_type.c @@ -105,6 +105,18 @@ static neu_dvalue_t sample(json_t *settings) if (!s) { s = "sample"; } + json_t *quality = json_object_get(settings, "quality"); + json_t *timestamp = json_object_get(settings, "timestamp"); + json_t *source_timestamp = json_object_get(settings, "source_timestamp"); + json_t *server_timestamp = json_object_get(settings, "server_timestamp"); + v.attrs.has_quality = quality != NULL; + v.attrs.quality = json_integer_value(quality); + v.attrs.has_timestamp = timestamp != NULL; + v.attrs.timestamp = json_integer_value(timestamp); + v.attrs.has_source_timestamp = source_timestamp != NULL; + v.attrs.source_timestamp = json_integer_value(source_timestamp); + v.attrs.has_server_timestamp = server_timestamp != NULL; + v.attrs.server_timestamp = json_integer_value(server_timestamp); if (error) { v.type = NEU_TYPE_ERROR; v.value.i32 = error; @@ -184,12 +196,6 @@ static neu_dvalue_t sample(json_t *settings) default: break; } - json_t *quality = json_object_get(settings, "quality"); - json_t *timestamp = json_object_get(settings, "timestamp"); - v.attrs.has_quality = quality != NULL; - v.attrs.quality = json_integer_value(quality); - v.attrs.has_timestamp = timestamp != NULL; - v.attrs.timestamp = json_integer_value(timestamp); return v; } diff --git a/tests/ut/CMakeLists.txt b/tests/ut/CMakeLists.txt index a52746315..f6eaed39f 100644 --- a/tests/ut/CMakeLists.txt +++ b/tests/ut/CMakeLists.txt @@ -129,7 +129,17 @@ target_compile_options(driver_auto_test PRIVATE -ffunction-sections -fdata-secti target_link_libraries(driver_auto_test neuron-base gtest_main gtest pthread jansson m -Wl,--gc-sections) +add_executable(http_read_attrs_test http_read_attrs_test.cc + ${CMAKE_SOURCE_DIR}/plugins/restful/read_response.c) +target_include_directories(http_read_attrs_test PRIVATE + ${CMAKE_SOURCE_DIR}/src + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/plugins +) +target_link_libraries(http_read_attrs_test neuron-base gtest_main gtest pthread jansson) + include(GoogleTest) +gtest_discover_tests(http_read_attrs_test) gtest_discover_tests(driver_auto_test) gtest_discover_tests(json_test) gtest_discover_tests(http_test) diff --git a/tests/ut/http_read_attrs_test.cc b/tests/ut/http_read_attrs_test.cc new file mode 100644 index 000000000..b5be5e690 --- /dev/null +++ b/tests/ut/http_read_attrs_test.cc @@ -0,0 +1,255 @@ +#include + +#include +#include +#include + +extern "C" { +#include "msg.h" +#include "restful/read_response.h" +#include "utils/log.h" + +zlog_category_t *neuron = nullptr; +} + +using Json = std::unique_ptr; + +static Json encode_sample(bool paginate, neu_tag_sample_attrs_t attrs, + bool error = false, bool legacy = false) +{ + UT_array *source; + UT_icd icd = { paginate ? sizeof(neu_resp_tag_value_meta_paginate_t) + : sizeof(neu_resp_tag_value_meta_t), + nullptr, nullptr, nullptr }; + utarray_new(source, &icd); + neu_resp_tag_value_meta_t tag = {}; + neu_resp_tag_value_meta_paginate_t paginated_tag = {}; + tag.value.attrs = paginated_tag.value.attrs = attrs; + if (paginate) { + utarray_push_back(source, &paginated_tag); + } else { + utarray_push_back(source, &tag); + } + + neu_json_tag_meta_t metas[2] = {}; + metas[0].name = const_cast("timestamp"); + metas[0].t = NEU_JSON_INT; + metas[0].value.val_int = 11; + metas[1].name = const_cast("quality"); + metas[1].t = NEU_JSON_INT; + metas[1].value.val_int = 22; + + neu_json_read_resp_tag_t item = {}; + neu_json_read_paginate_resp_tag_t page_item = {}; + item.name = page_item.name = const_cast("a"); + item.error = page_item.error = error ? 3002 : 0; + item.t = page_item.t = NEU_JSON_DOUBLE; + item.value.val_double = page_item.value.val_double = 12.5; + item.precision = page_item.precision = 2; + item.n_meta = page_item.n_meta = legacy ? 2 : 0; + item.metas = page_item.metas = metas; + neu_json_read_resp_t response = { 1, &item }; + neu_json_read_paginate_resp_t page = {}; + page.n_tag = 1; + page.tags = &page_item; + page.meta.current_page = 2; + page.meta.page_size = 1; + page.meta.total = 3; + neu_http_read_response_t context = {}; + context.response = paginate ? (void *) &page : (void *) &response; + context.tags = source; + context.paginate = paginate; + Json root(json_object(), json_decref); + EXPECT_EQ(0, neu_http_encode_read_response(root.get(), &context)); + utarray_free(source); + return root; +} + +static json_t *first_item(const Json &root, bool paginate) +{ + return json_array_get( + json_object_get(root.get(), paginate ? "items" : "tags"), 0); +} + +TEST(HttpReadAttrs, IndependentOptionalFields) +{ + const char *names[] = { "timestamp", "quality", "source_timestamp", + "server_timestamp" }; + for (bool paginate : { false, true }) { + for (int flags = 0; flags < 16; ++flags) { + SCOPED_TRACE(::testing::Message() << paginate << ":" << flags); + neu_tag_sample_attrs_t attrs = {}; + attrs.timestamp = std::numeric_limits::min(); + attrs.quality = std::numeric_limits::max(); + attrs.source_timestamp = 0; + attrs.server_timestamp = std::numeric_limits::max(); + attrs.has_timestamp = flags & 1; + attrs.has_quality = flags & 2; + attrs.has_source_timestamp = flags & 4; + attrs.has_server_timestamp = flags & 8; + auto root = encode_sample(paginate, attrs); + auto item = first_item(root, paginate); + ASSERT_TRUE(json_is_object(item)); + const int64_t expected[] = { attrs.timestamp, attrs.quality, + attrs.source_timestamp, + attrs.server_timestamp }; + for (int i = 0; i < 4; ++i) { + auto field = json_object_get(item, names[i]); + if (flags & (1 << i)) { + ASSERT_TRUE(json_is_integer(field)); + EXPECT_EQ(expected[i], json_integer_value(field)); + } else { + EXPECT_EQ(nullptr, field); + } + } + EXPECT_DOUBLE_EQ(12.5, + json_number_value(json_object_get(item, "value"))); + EXPECT_EQ( + 2, + json_integer_value(json_object_get(item, "transferPrecision"))); + EXPECT_EQ(nullptr, json_object_get(item, "error")); + if (paginate) { + EXPECT_TRUE( + json_is_object(json_object_get(item, "attributes"))); + EXPECT_EQ(3, + json_integer_value(json_object_get( + json_object_get(root.get(), "meta"), "total"))); + } + } + } +} + +TEST(HttpReadAttrs, ExplicitZerosAndErrorSamples) +{ + neu_tag_sample_attrs_t attrs = {}; + attrs.has_timestamp = attrs.has_quality = true; + attrs.has_source_timestamp = attrs.has_server_timestamp = true; + for (bool paginate : { false, true }) { + auto root = encode_sample(paginate, attrs, true); + auto item = first_item(root, paginate); + for (const char *name : { "timestamp", "quality", "source_timestamp", + "server_timestamp" }) { + ASSERT_TRUE(json_is_integer(json_object_get(item, name))); + EXPECT_EQ(0, json_integer_value(json_object_get(item, name))); + } + EXPECT_EQ(3002, json_integer_value(json_object_get(item, "error"))); + EXPECT_EQ(nullptr, json_object_get(item, "value")); + EXPECT_EQ(nullptr, json_object_get(item, "transferPrecision")); + } +} + +TEST(HttpReadAttrs, LegacyMetadataCollisionAndNesting) +{ + for (bool paginate : { false, true }) { + for (bool present : { false, true }) { + neu_tag_sample_attrs_t attrs = {}; + attrs.has_timestamp = attrs.has_quality = present; + attrs.timestamp = 100; + attrs.quality = 192; + auto root = encode_sample(paginate, attrs, false, true); + auto item = first_item(root, paginate); + auto legacy = paginate ? json_object_get(item, "attributes") : item; + EXPECT_EQ(paginate || !present ? 11 : 100, + json_integer_value(json_object_get(legacy, "timestamp"))); + EXPECT_EQ(paginate || !present ? 22 : 192, + json_integer_value(json_object_get(legacy, "quality"))); + if (paginate) { + EXPECT_EQ(present, + json_object_get(item, "timestamp") != nullptr); + if (present) { + EXPECT_EQ( + 100, + json_integer_value(json_object_get(item, "timestamp"))); + } + } + char *encoded = json_dumps(root.get(), 0); + Json checked(json_loads(encoded, JSON_REJECT_DUPLICATES, nullptr), + json_decref); + free(encoded); + EXPECT_NE(nullptr, checked.get()); + } + } +} + +TEST(HttpReadAttrs, FilteredPageUsesSourceOffset) +{ + UT_array *source; + utarray_new(source, neu_resp_tag_value_meta_paginate_icd()); + for (int i = 0; i < 3; ++i) { + neu_resp_tag_value_meta_paginate_t tag = {}; + tag.value.attrs.has_quality = true; + tag.value.attrs.quality = 100 + i; + utarray_push_back(source, &tag); + } + neu_json_read_paginate_resp_tag_t items[2] = {}; + items[0].name = const_cast("b"); + items[1].name = const_cast("c"); + for (auto &item : items) { + item.error = 3002; + } + neu_json_read_paginate_resp_t response = {}; + response.n_tag = 2; + response.tags = items; + neu_http_read_response_t context = { &response, source, 1, true }; + Json root(json_object(), json_decref); + ASSERT_EQ(0, neu_http_encode_read_response(root.get(), &context)); + auto output = json_object_get(root.get(), "items"); + EXPECT_EQ(101, + json_integer_value( + json_object_get(json_array_get(output, 0), "quality"))); + EXPECT_EQ(102, + json_integer_value( + json_object_get(json_array_get(output, 1), "quality"))); + utarray_free(source); +} + +TEST(HttpReadAttrs, RejectsInvalidContextsAndReleasesPageConfiguration) +{ + Json root(json_object(), json_decref); + EXPECT_EQ(-1, neu_http_encode_read_response(root.get(), nullptr)); + neu_http_read_response_t empty = {}; + EXPECT_EQ(-1, neu_http_encode_read_response(root.get(), &empty)); + + for (bool invalid_object : { false, true }) { + UT_array *source; + utarray_new(source, neu_resp_tag_value_meta_paginate_icd()); + if (invalid_object) { + neu_resp_tag_value_meta_paginate_t tag = {}; + utarray_push_back(source, &tag); + } + neu_json_read_paginate_resp_tag_t item = {}; + item.datatag.name = strdup("a"); + item.datatag.address = strdup("sample"); + item.datatag.description = strdup(""); + item.datatag.unit = strdup("C"); + neu_json_read_paginate_resp_t response = {}; + response.n_tag = 1; + response.tags = &item; + neu_http_read_response_t context = { &response, source, 0, true }; + EXPECT_EQ(-1, + neu_http_encode_read_response( + invalid_object ? nullptr : root.get(), &context)); + utarray_free(source); + } +} + +TEST(HttpReadAttrs, EmptyPagesAndInvalidSourceBounds) +{ + for (bool paginate : { false, true }) { + neu_json_read_resp_t response = {}; + neu_json_read_paginate_resp_t page = {}; + neu_http_read_response_t context = {}; + context.response = paginate ? (void *) &page : (void *) &response; + context.paginate = paginate; + context.offset = std::numeric_limits::max(); + Json root(json_object(), json_decref); + EXPECT_EQ(0, neu_http_encode_read_response(root.get(), &context)); + auto items = json_object_get(root.get(), paginate ? "items" : "tags"); + EXPECT_TRUE(json_is_array(items)); + EXPECT_EQ(0u, json_array_size(items)); + response.n_tag = page.n_tag = 1; + EXPECT_EQ(-1, neu_http_encode_read_response(root.get(), &context)); + response.n_tag = page.n_tag = -1; + EXPECT_EQ(-1, neu_http_encode_read_response(root.get(), &context)); + } +}