diff --git a/README.md b/README.md index 08398cb..6033d7d 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Table of Contents * [encode_escape_forward_slash](#encode_escape_forward_slash) * [encode_skip_unsupported_value_types](#encode_skip_unsupported_value_types) * [encode_indent](#encode_indent) + * [encode_sort_keys](#encode_sort_keys) * [decode_array_with_array_mt](#decode_array_with_array_mt) * [decode_allow_comment](#decode_allow_comment) @@ -228,6 +229,17 @@ print(cjson.encode({ a = 1, b = { c = 2 } })) [Back to TOC](#table-of-contents) +encode_sort_keys +--------------------------- +**syntax:** `cjson.encode_sort_keys(enabled)` + +**default:** false + +If enabled, keys in encoded objects will be sorted lexicographically before +JSON string escaping is applied. + +[Back to TOC](#table-of-contents) + decode_array_with_array_mt -------------------------- **syntax:** `cjson.decode_array_with_array_mt(enabled)` diff --git a/lua_cjson.c b/lua_cjson.c index 19c22f2..f1aa4ad 100644 --- a/lua_cjson.c +++ b/lua_cjson.c @@ -94,6 +94,7 @@ #define DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH 1 #define DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES 0 #define DEFAULT_ENCODE_INDENT NULL +#define DEFAULT_ENCODE_SORT_KEYS 0 #ifdef DISABLE_INVALID_NUMBERS #undef DEFAULT_DECODE_INVALID_NUMBERS @@ -158,6 +159,48 @@ static const char *json_token_type_name[] = { NULL }; +typedef struct { + strbuf_t *buf; + size_t offset; + size_t length; + int raw_type; + const char *raw_string; + size_t raw_string_length; + lua_Number raw_number; +#if LUA_VERSION_NUM >= 503 + int raw_number_is_integer; + lua_Integer raw_integer; +#endif +} key_entry_t; + +/* Stores all keys for a table when key sorting is enabled. + * - buf: buffer holding serialized key strings + * - keys: array of key_entry_t pointing into buf + * - size: number of keys stored + * - capacity: allocated capacity of keys array + */ +typedef struct { + strbuf_t buf; + key_entry_t *keys; + size_t size; + size_t capacity; +} keybuf_t; + +#define KEYBUF_DEFAULT_CAPACITY 32 + +static void keybuf_init(keybuf_t *keybuf) +{ + memset(keybuf, 0, sizeof(*keybuf)); + strbuf_init(&keybuf->buf, 0); +} + +static void keybuf_free(keybuf_t *keybuf) +{ + strbuf_free(&keybuf->buf); + free(keybuf->keys); + memset(keybuf, 0, sizeof(*keybuf)); +} + typedef struct { json_token_type_t ch2token[256]; char escape2char[256]; /* Decoding */ @@ -172,6 +215,10 @@ typedef struct { * encode_keep_buffer is set */ strbuf_t encode_buf; + /* encode_keybuf is only allocated and used when + * encode_sort_keys is set */ + keybuf_t encode_keybuf; + int encode_sparse_convert; int encode_sparse_ratio; int encode_sparse_safe; @@ -182,6 +229,7 @@ typedef struct { int encode_empty_table_as_object; int encode_escape_forward_slash; const char *encode_indent; + int encode_sort_keys; int decode_invalid_numbers; int decode_max_depth; @@ -506,6 +554,27 @@ static int json_cfg_encode_escape_forward_slash(lua_State *l) return ret; } +static int json_cfg_encode_sort_keys(lua_State *l) +{ + json_config_t *cfg = json_arg_init(l, 1); + int old_value; + + old_value = cfg->encode_sort_keys; + + json_enum_option(l, 1, &cfg->encode_sort_keys, NULL, 1); + + /* Init / free the keybuf if the setting has changed */ + if (old_value ^ cfg->encode_sort_keys) { + if (cfg->encode_sort_keys) { + keybuf_init(&cfg->encode_keybuf); + } else { + keybuf_free(&cfg->encode_keybuf); + } + } + + return 1; +} + static int json_destroy_config(lua_State *l) { json_config_t *cfg; @@ -513,6 +582,8 @@ static int json_destroy_config(lua_State *l) cfg = (json_config_t *)lua_touserdata(l, 1); if (cfg) { strbuf_free(&cfg->encode_buf); + keybuf_free(&cfg->encode_keybuf); + if (cfg->encode_indent) { free((void *) cfg->encode_indent); cfg->encode_indent = NULL; @@ -555,6 +626,7 @@ static void json_create_config(lua_State *l) cfg->encode_escape_forward_slash = DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH; cfg->encode_skip_unsupported_value_types = DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES; cfg->encode_indent = DEFAULT_ENCODE_INDENT; + cfg->encode_sort_keys = DEFAULT_ENCODE_SORT_KEYS; /* Seed this instance's escape table from the shared template, then * apply the per-instance forward-slash setting. Mutating cfg->char2escape @@ -625,14 +697,8 @@ static void json_encode_exception(lua_State *l, json_config_t *cfg, strbuf_t *js lua_typename(l, lua_type(l, lindex)), reason); } -/* json_append_string args: - * - lua_State - * - JSON strbuf - * - String (Lua stack index) - * - * Returns nothing. Doesn't remove string from Lua stack */ -static void json_append_string(lua_State *l, json_config_t *cfg, - strbuf_t *json, int lindex) +static void json_append_string_contents(lua_State *l, json_config_t *cfg, + strbuf_t *json, int lindex) { const char *escstr; const char *str; @@ -645,11 +711,10 @@ static void json_append_string(lua_State *l, json_config_t *cfg, * This buffer is reused constantly for small strings * If there are any excess pages, they won't be hit anyway. * This gains ~5% speedup. */ - if (len > SIZE_MAX / 6 - 3) + if (len >= SIZE_MAX / 6) abort(); /* Overflow check */ - strbuf_ensure_empty_length(json, len * 6 + 2); + strbuf_ensure_empty_length(json, len * 6); - strbuf_append_char_unsafe(json, '\"'); for (i = 0; i < len; i++) { escstr = cfg->char2escape[(unsigned char)str[i]]; if (escstr) @@ -657,7 +722,20 @@ static void json_append_string(lua_State *l, json_config_t *cfg, else strbuf_append_char_unsafe(json, str[i]); } - strbuf_append_char_unsafe(json, '\"'); +} + +/* json_append_string args: + * - lua_State + * - JSON strbuf + * - String (Lua stack index) + * + * Returns nothing. Doesn't remove string from Lua stack */ +static void json_append_string(lua_State *l, json_config_t *cfg, + strbuf_t *json, int lindex) +{ + strbuf_append_char(json, '\"'); + json_append_string_contents(l, cfg, json, lindex); + strbuf_append_char(json, '\"'); } /* Find the size of the array on the top of the Lua stack @@ -792,9 +870,12 @@ static void json_append_array(lua_State *l, json_config_t *cfg, int current_dept } static void json_append_number(lua_State *l, json_config_t *cfg, - strbuf_t *json, int lindex) + strbuf_t *json, strbuf_t *error_json, + int lindex) { int len; + double num; + #if LUA_VERSION_NUM >= 503 if (lua_isinteger(l, lindex)) { lua_Integer num = lua_tointeger(l, lindex); @@ -804,12 +885,12 @@ static void json_append_number(lua_State *l, json_config_t *cfg, return; } #endif - double num = lua_tonumber(l, lindex); + num = lua_tonumber(l, lindex); if (cfg->encode_invalid_numbers == 0) { /* Prevent encoding invalid numbers */ if (isinf(num) || isnan(num)) - json_encode_exception(l, cfg, json, lindex, + json_encode_exception(l, cfg, error_json, lindex, "must not be NaN or Infinity"); } else if (cfg->encode_invalid_numbers == 1) { /* Encode NaN/Infinity separately to ensure Javascript compatible @@ -838,11 +919,120 @@ static void json_append_number(lua_State *l, json_config_t *cfg, strbuf_extend_length(json, len); } +static void keybuf_reserve_entry(lua_State *l, json_config_t *cfg, + keybuf_t *keybuf, strbuf_t *json, + int lindex) +{ + key_entry_t *keys; + size_t capacity; + + if (keybuf->size < keybuf->capacity) + return; + + if (keybuf->capacity == 0) { + capacity = KEYBUF_DEFAULT_CAPACITY; + } else { + if (keybuf->capacity > SIZE_MAX / 2) + json_encode_exception(l, cfg, json, lindex, + "too many object keys"); + + capacity = keybuf->capacity * 2; + } + + if (capacity > SIZE_MAX / sizeof(*keys)) + json_encode_exception(l, cfg, json, lindex, + "too many object keys"); + + keys = realloc(keybuf->keys, capacity * sizeof(*keys)); + if (!keys) + json_encode_exception(l, cfg, json, lindex, "out of memory"); + + keybuf->keys = keys; + keybuf->capacity = capacity; +} + +static const char *key_entry_sort_string(const key_entry_t *key, + size_t *length) +{ + if (key->raw_type == LUA_TSTRING) { + *length = key->raw_string_length; + return key->raw_string; + } + + *length = key->length; + return key->buf->buf + key->offset; +} + +/* Compare key_entry_t for qsort */ +static int cmp_key_entries(const void *a, const void *b) +{ + const key_entry_t *ka = a; + const key_entry_t *kb = b; + const char *ka_string; + const char *kb_string; + size_t ka_length; + size_t kb_length; + size_t min_length; + int res; + + ka_string = key_entry_sort_string(ka, &ka_length); + kb_string = key_entry_sort_string(kb, &kb_length); + min_length = ka_length < kb_length ? ka_length : kb_length; + res = memcmp(ka_string, kb_string, min_length); + if (res) + return res; + + if (ka_length < kb_length) + return -1; + + if (ka_length > kb_length) + return 1; + + if (ka->raw_type < kb->raw_type) + return -1; + + if (ka->raw_type > kb->raw_type) + return 1; + + /* Distinct numbers may round to the same serialized key. */ + if (ka->raw_type == LUA_TNUMBER) { +#if LUA_VERSION_NUM >= 503 + /* Order integer keys first without converting them to doubles. */ + if (ka->raw_number_is_integer != kb->raw_number_is_integer) + return ka->raw_number_is_integer ? -1 : 1; + + if (ka->raw_number_is_integer) { + if (ka->raw_integer < kb->raw_integer) + return -1; + + if (ka->raw_integer > kb->raw_integer) + return 1; + + return 0; + } +#endif + if (ka->raw_number < kb->raw_number) + return -1; + + if (ka->raw_number > kb->raw_number) + return 1; + } + + return 0; +} + static void json_append_object(lua_State *l, json_config_t *cfg, int current_depth, strbuf_t *json) { int comma, keytype, json_pos, err; int has_items = 0; + keybuf_t *keybuf; + key_entry_t key_entry; + key_entry_t *current_key; + size_t init_keybuf_size; + size_t init_keybuf_length; + size_t keys_count; + size_t i; /* Object */ strbuf_append_char(json, '{'); @@ -850,45 +1040,133 @@ static void json_append_object(lua_State *l, json_config_t *cfg, lua_pushnil(l); /* table, startkey */ comma = 0; - while (lua_next(l, -2) != 0) { - has_items = 1; - - json_pos = strbuf_length(json); - if (comma++ > 0) - strbuf_append_char(json, ','); + if (cfg->encode_sort_keys) { + keybuf = &cfg->encode_keybuf; + init_keybuf_size = keybuf->size; + init_keybuf_length = strbuf_length(&keybuf->buf); + + /* Collect keys into keybuf */ + while (lua_next(l, -2) != 0) { + has_items = 1; + keybuf_reserve_entry(l, cfg, keybuf, json, -1); + + keytype = lua_type(l, -2); + memset(&key_entry, 0, sizeof(key_entry)); + key_entry.buf = &keybuf->buf; + key_entry.offset = strbuf_length(&keybuf->buf); + key_entry.raw_type = keytype; + + if (keytype == LUA_TSTRING) { + key_entry.raw_string = lua_tolstring(l, -2, + &key_entry.raw_string_length); + json_append_string_contents(l, cfg, &keybuf->buf, -2); + } else if (keytype == LUA_TNUMBER) { + json_append_number(l, cfg, &keybuf->buf, json, -2); +#if LUA_VERSION_NUM >= 503 + key_entry.raw_number_is_integer = lua_isinteger(l, -2); + if (key_entry.raw_number_is_integer) + key_entry.raw_integer = lua_tointeger(l, -2); + else +#endif + key_entry.raw_number = lua_tonumber(l, -2); + } else { + json_encode_exception(l, cfg, json, -2, + "table key must be number or string"); + } - if (cfg->encode_indent) - json_append_newline_and_indent(json, cfg, current_depth); + key_entry.length = strbuf_length(&keybuf->buf) - key_entry.offset; + keybuf->keys[keybuf->size++] = key_entry; + lua_pop(l, 1); + } - /* table, key, value */ - keytype = lua_type(l, -2); - if (keytype == LUA_TNUMBER) { - strbuf_append_char(json, '"'); - json_append_number(l, cfg, json, -2); - strbuf_append_mem(json, "\":", 2); - } else if (keytype == LUA_TSTRING) { - json_append_string(l, cfg, json, -2); - strbuf_append_char(json, ':'); - } else { - json_encode_exception(l, cfg, json, -2, - "table key must be a number or string"); - /* never returns */ + keys_count = keybuf->size - init_keybuf_size; + if (keys_count > 1) { + qsort(keybuf->keys + init_keybuf_size, keys_count, + sizeof (key_entry_t), cmp_key_entries); } - if (cfg->encode_indent) - strbuf_append_char(json, ' '); + for (i = init_keybuf_size; i < init_keybuf_size + keys_count; i++) { + current_key = &keybuf->keys[i]; + json_pos = strbuf_length(json); + if (comma++ > 0) + strbuf_append_char(json, ','); - /* table, key, value */ - err = json_append_data(l, cfg, current_depth, json); - if (err) { - strbuf_set_length(json, json_pos); - if (comma == 1) { - comma = 0; + if (cfg->encode_indent) + json_append_newline_and_indent(json, cfg, current_depth); + + strbuf_ensure_empty_length(json, current_key->length + 3); + strbuf_append_char_unsafe(json, '"'); + strbuf_append_mem_unsafe(json, keybuf->buf.buf + current_key->offset, + current_key->length); + strbuf_append_mem_unsafe(json, "\":", 2); + + if (cfg->encode_indent) + strbuf_append_char(json, ' '); + + if (current_key->raw_type == LUA_TSTRING) { + lua_pushlstring(l, current_key->raw_string, + current_key->raw_string_length); + } else { +#if LUA_VERSION_NUM >= 503 + if (current_key->raw_number_is_integer) + lua_pushinteger(l, current_key->raw_integer); + else +#endif + lua_pushnumber(l, current_key->raw_number); + } + + lua_rawget(l, -2); + err = json_append_data(l, cfg, current_depth, json); + if (err) { + strbuf_set_length(json, json_pos); + if (comma == 1) + comma = 0; } + lua_pop(l, 1); } + /* Resize encode_keybuf to reuse allocated memory for forward keys */ + strbuf_set_length(&keybuf->buf, init_keybuf_length); + keybuf->size = init_keybuf_size; + } else { + while (lua_next(l, -2) != 0) { + has_items = 1; + + json_pos = strbuf_length(json); + if (comma++ > 0) + strbuf_append_char(json, ','); + + if (cfg->encode_indent) + json_append_newline_and_indent(json, cfg, current_depth); + + /* table, key, value */ + keytype = lua_type(l, -2); + if (keytype == LUA_TNUMBER) { + strbuf_append_char(json, '"'); + json_append_number(l, cfg, json, json, -2); + strbuf_append_mem(json, "\":", 2); + } else if (keytype == LUA_TSTRING) { + json_append_string(l, cfg, json, -2); + strbuf_append_char(json, ':'); + } else { + json_encode_exception(l, cfg, json, -2, + "table key must be a number or string"); + /* never returns */ + } + if (cfg->encode_indent) + strbuf_append_char(json, ' '); + + /* table, key, value */ + err = json_append_data(l, cfg, current_depth, json); + if (err) { + strbuf_set_length(json, json_pos); + if (comma == 1) { + comma = 0; + } + } - lua_pop(l, 1); - /* table, key */ + lua_pop(l, 1); + /* table, key */ + } } if (has_items && cfg->encode_indent) @@ -911,7 +1189,7 @@ static int json_append_data(lua_State *l, json_config_t *cfg, json_append_string(l, cfg, json, -1); break; case LUA_TNUMBER: - json_append_number(l, cfg, json, -1); + json_append_number(l, cfg, json, json, -1); break; case LUA_TBOOLEAN: if (lua_toboolean(l, -1)) @@ -1016,6 +1294,15 @@ static int json_encode(lua_State *l) strbuf_reset(encode_buf); } + if (cfg->encode_sort_keys) { + /* Reuse existing keybuf */ + if (!strbuf_allocated(&cfg->encode_keybuf.buf)) + keybuf_init(&cfg->encode_keybuf); + else + strbuf_reset(&cfg->encode_keybuf.buf); + cfg->encode_keybuf.size = 0; + } + json_append_data(l, cfg, 0, encode_buf); json = strbuf_string(encode_buf, &len); @@ -1024,6 +1311,12 @@ static int json_encode(lua_State *l) if (!cfg->encode_keep_buffer) strbuf_free(encode_buf); + /* Free keybuf if it is too large */ + if (cfg->encode_sort_keys && + cfg->encode_keybuf.capacity > KEYBUF_DEFAULT_CAPACITY*8) { + keybuf_free(&cfg->encode_keybuf); + } + return 1; } @@ -1716,6 +2009,7 @@ static int lua_cjson_new(lua_State *l) { "encode_escape_forward_slash", json_cfg_encode_escape_forward_slash }, { "encode_skip_unsupported_value_types", json_cfg_encode_skip_unsupported_value_types }, { "encode_indent", json_cfg_encode_indent }, + { "encode_sort_keys", json_cfg_encode_sort_keys }, { "new", lua_cjson_new }, { NULL, NULL } }; diff --git a/runtests.sh b/runtests.sh index 1c51236..4bd0fc1 100755 --- a/runtests.sh +++ b/runtests.sh @@ -59,21 +59,19 @@ rm -rf tests/cjson{,.so} if [ -z "$SKIP_CMAKE" ]; then echo "===== Testing Cmake build =====" - mkdir build - cd build - cmake .. - make - cd .. + make clean + rm -fr build + cmake -S . -B build + cmake --build build cp -r lua/cjson build/cjson.so tests do_tests rm -rf build tests/cjson{,.so} echo "===== Testing Cmake fpconv build =====" - mkdir build - cd build - cmake -DUSE_INTERNAL_FPCONV=1 .. - make - cd .. + make clean + rm -fr build + cmake -S . -B build -DUSE_INTERNAL_FPCONV=1 + cmake --build build cp -r lua/cjson build/cjson.so tests do_tests rm -rf build tests/cjson{,.so} diff --git a/tests/test.lua b/tests/test.lua index fc373f6..984c583 100755 --- a/tests/test.lua +++ b/tests/test.lua @@ -397,6 +397,101 @@ local cjson_tests = { json.encode, { { { a = "a" }, { b = "b" } } }, true, { '[{"a":"a"},{"b":"b"}]' } }, + -- Test keys sorting + { "Set encode_sort_keys(true)", + json.encode_sort_keys, { true }, true, { true } }, + { "Encode empty object with sorting", + json.encode, { {} }, + true, { '{}' } }, + { "Encode object with sorting", + json.encode, { { a = 0, b = 0, ab = 0, [1] = 0, ["$"] = 0, [4] = 0, ["%"] = 0 } }, + true, { '{"$":0,"%":0,"1":0,"4":0,"a":0,"ab":0,"b":0}' } }, + { "Encode object with string keys with sorting", + json.encode, { { aa = 1, ba = 3, ab = 2, bc = 4, cc = 5 } }, + true, { '{"aa":1,"ab":2,"ba":3,"bc":4,"cc":5}' } }, + { "Encode nested objects with sorting", + json.encode, { { a = { b = 2, a = 1, c = 3 }, c = 0, b = { b = { a = 0, b = 0 }, a = { a = 0, b = 0 } } } }, + true, { '{"a":{"a":1,"b":2,"c":3},"b":{"a":{"a":0,"b":0},"b":{"a":0,"b":0}},"c":0}' } }, + { "Encode array of objects with sorting", + json.encode, { { + { a = 0, [1] = 0, [4] = 0, b = 0 }, + { f = 0, [5] = 0, [10] = 0, x = 0 }, + { c = 0, [-2] = 0, [2] = 0, d = 0 }, + } }, + true, { '[{"1":0,"4":0,"a":0,"b":0},{"10":0,"5":0,"f":0,"x":0},{"-2":0,"2":0,"c":0,"d":0}]' } }, + { "Encode object with unicode keys", + json.encode, { { ["é"] = 1, ["a"] = 2, ["ß"] = 3, ["中"] = 4 } }, + true, { '{"a":2,"ß":3,"é":1,"中":4}' } }, + { "Encode escaped object keys with sorting", + json.encode, { { ["/"] = 1, ["0"] = 2 } }, + true, { [[{"\/":1,"0":2}]] } }, + { "Encode fractional numeric key with sorting", + json.encode, { { [1.5] = "fraction" } }, + true, { '{"1.5":"fraction"}' } }, + { "Sort colliding numeric keys independently of insertion order", function() + local encoder = json.new() + encoder.encode_sort_keys(true) + encoder.encode_number_precision(1) + local forward = { z = true } + forward[1.1] = "first" + forward[1.2] = "second" + local reverse = { z = true } + reverse[1.2] = "second" + reverse[1.1] = "first" + + return encoder.encode(forward), encoder.encode(reverse) + end, { }, true, { + '{"1":"first","1":"second","z":true}', + '{"1":"first","1":"second","z":true}' + } }, + { "Sort colliding integer, fractional and string keys", function() + local encoder = json.new() + encoder.encode_sort_keys(true) + encoder.encode_number_precision(1) + local data = { ["1"] = "string" } + data[1.1] = "fraction" + data[1] = "integer" + + return encoder.encode(data) + end, { }, true, { + '{"1":"integer","1":"fraction","1":"string"}' + } }, + { "Encode binary string key with sorting", + json.encode, { { ["a\0b"] = "nul" } }, + true, { '{"a\\u0000b":"nul"}' } }, + { "Encode large integer key with sorting", function() + if math.type and math.type(9007199254740993) == "integer" then + return json.encode({ [9007199254740993] = "large", z = true }) + end + + return "Lua integers not supported" + end, { }, true, { + math.type and math.type(9007199254740993) == "integer" + and '{"9007199254740993":"large","z":true}' + or "Lua integers not supported" + } }, + { "Reuse key buffer after encoding a large object", function() + local data = {} + for i = 1, 257 do + data["key" .. i] = i + end + + local decoded = json.decode(json.encode(data)) + return decoded.key1, decoded.key257, json.encode({ again = true }) + end, { }, true, { 1, 257, '{"again":true}' } }, + { "Set encode_keep_buffer(false) with sorting", + json.encode_keep_buffer, { false }, true, { false } }, + { "Encode invalid numeric key with sorting [throw error]", + json.encode, { { [Inf] = true } }, + false, { "Cannot serialise number: must not be NaN or Infinity" } }, + { "Encode after invalid numeric key with sorting", + json.encode, { { again = true } }, + true, { '{"again":true}' } }, + { "Set encode_keep_buffer(true) with sorting", + json.encode_keep_buffer, { true }, true, { true } }, + { "Set encode_sort_keys(false)", + json.encode_sort_keys, { false }, true, { false } }, + -- Test locale support -- -- The standard Lua interpreter is ANSI C online doesn't support locales