diff --git a/HISTORY.rst b/HISTORY.rst index 453853e..72c9db4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,16 @@ History ======= +1.2.3 (2026-08-12) +------------------ + +* Added file-discovery value counts: + + * ``value_count`` on ``FileDiscoveryLocatorResult``, the total over the files in the group. + * ``value_counts`` and ``value_count_status`` on ``FileDiscoveryFile``, typed by ``ValueCountStatus``. + +Requires server version 3.26.15 + 1.2.2 (2026-07-31) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index d05d458..5c150fb 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -81,6 +81,7 @@ SchemaDiscoveryRequest, SchemaDiscoveryResult, TableConstraints, + ValueCountStatus, ) from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary, DiscoveryConfigLibraryId @@ -319,4 +320,5 @@ "ValidationErrorDetails", "ValidationErrorType", "ValidationStatus", + "ValueCountStatus", ] diff --git a/datamasque/client/models/data_selection.py b/datamasque/client/models/data_selection.py index 54dfbb8..20cbc07 100644 --- a/datamasque/client/models/data_selection.py +++ b/datamasque/client/models/data_selection.py @@ -1,5 +1,6 @@ """Models related to data selection in endpoints such as /api/async-generate-ruleset.""" +import json from typing import Optional, Union from pydantic import BaseModel, ConfigDict @@ -19,6 +20,33 @@ """ +def serialize_locator(locator: Locator) -> str: + """ + Returns the string form of `locator` that the API uses as a key, such as in `FileDiscoveryFile.value_counts`. + + A string locator is its own key. + A :data:`JsonPath` becomes compact JSON, e.g. `'["employees","*","email"]'`. + """ + + if isinstance(locator, str): + return locator + return json.dumps(locator, separators=(",", ":"), ensure_ascii=False) + + +def deserialize_locator(key: str) -> Locator: + """ + Returns the `Locator` that `key`, a serialized locator from the API, identifies. + + A key that holds no JSON array names a column, and comes back as a string. + """ + + try: + parsed = json.loads(key) + except json.JSONDecodeError: + return key + return parsed if isinstance(parsed, list) else key + + class UserSelection(BaseModel): """Information about selected files and locators for file masking ruleset generation.""" diff --git a/datamasque/client/models/discovery.py b/datamasque/client/models/discovery.py index 223f6e5..3e5f1fb 100644 --- a/datamasque/client/models/discovery.py +++ b/datamasque/client/models/discovery.py @@ -6,7 +6,13 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from datamasque.client.models.connection import ConnectionConfig, ConnectionId, unwrap_connection_id -from datamasque.client.models.data_selection import HashColumnsTableConfig, Locator, UserSelection +from datamasque.client.models.data_selection import ( + HashColumnsTableConfig, + Locator, + UserSelection, + deserialize_locator, + serialize_locator, +) from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, unwrap_discovery_config_id from datamasque.client.models.pagination import Page from datamasque.client.models.rg_config import RGConfig, RGConfigId, unwrap_rg_config_id @@ -434,6 +440,14 @@ class FileDiscoveryMatch(BaseModel): hit_ratio: Optional[int] = None # None for metadata matches, percentage 0-100 for IDD matches. +class ValueCountStatus(Enum): + """Whether a file's values were counted, and when they were not, why.""" + + counted = "counted" + in_data_discovery_disabled = "in_data_discovery_disabled" + file_type_has_no_values = "file_type_has_no_values" + + class FileDiscoveryLocatorResult(BaseModel): """A locator (column/path) within a discovered file.""" @@ -443,6 +457,7 @@ class FileDiscoveryLocatorResult(BaseModel): matches: list[FileDiscoveryMatch] data_types: list[str] safe_data_preview: Optional[SafeDataPreview] = None + value_count: Optional[int] = None class FileDiscoveryFile(BaseModel): @@ -454,6 +469,23 @@ class FileDiscoveryFile(BaseModel): file_type: str delimiter: Optional[str] = None encoding: Optional[str] = None + value_counts: dict[str, int] = Field(default_factory=dict) + value_count_status: Optional[ValueCountStatus] = None + + @field_validator("value_count_status", mode="before") + @classmethod + def _blank_status_is_none(cls, value: Any) -> Any: + return value or None + + def get_value_count_of_locator(self, locator: Locator) -> Optional[int]: + """Returns how many values this file holds at `locator`, or `None` if this file holds no count for it.""" + + return self.value_counts.get(serialize_locator(locator)) + + def parse_value_counts(self) -> list[tuple[Locator, int]]: + """Returns this file's value counts, with each locator in the form that the discovery results use.""" + + return [(deserialize_locator(key), count) for key, count in self.value_counts.items()] class FileDiscoveryResult(BaseModel): diff --git a/pyproject.toml b/pyproject.toml index 2945092..def1e15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datamasque-python" -version = "1.2.2" +version = "1.2.3" description = "Official Python client for the DataMasque data-masking API." authors = [ { name = "DataMasque Ltd" }, diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 8b5585d..f73c902 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -31,6 +31,7 @@ SchemaDiscoveryRequest, SchemaDiscoveryResult, StringPreview, + ValueCountStatus, ) from datamasque.client.exceptions import ( AsyncRulesetGenerationInProgressError, @@ -1012,6 +1013,181 @@ def test_file_discovery_result_parses_server_response(): assert non_sensitive_match.categories is None +def test_file_discovery_value_counts_parse(): + """A JSON file's value counts parse, keyed by serialized locator, with the group total on the locator.""" + result = FileDiscoveryResult.model_validate( + { + "id": 8, + "connection": {"id": "conn-1", "name": "my files"}, + "file_type": "json", + "files": [ + { + "path": "data/staff.json", + "file_type": "json", + "value_counts": {'["employees","*","email"]': 12}, + "value_count_status": "counted", + } + ], + "results": [ + { + "locator": ["employees", "*", "email"], + "data_types": ["string"], + "matches": [{"flagged_by": "IDD", "description": "Email", "label": "email"}], + "value_count": 30, + } + ], + } + ) + + file = result.files[0] + assert file.value_count_status is ValueCountStatus.counted + assert file.value_counts == {'["employees","*","email"]': 12} + assert result.results[0].value_count == 30 + + +def test_file_discovery_uncounted_values_parse(): + """A file type that holds no values reports the reason, no counts, and a null count on the locator.""" + result = FileDiscoveryResult.model_validate( + { + "id": 9, + "connection": {"id": "conn-1", "name": "my files"}, + "file_type": "csv", + "files": [ + { + "path": "data/people.csv", + "file_type": "csv", + "value_counts": {}, + "value_count_status": "file_type_has_no_values", + } + ], + "results": [ + { + "locator": "email", + "data_types": ["string"], + "matches": [{"flagged_by": "MDD", "description": "Email", "label": "email"}], + "value_count": None, + } + ], + } + ) + + file = result.files[0] + assert file.value_count_status is ValueCountStatus.file_type_has_no_values + assert file.value_counts == {} + assert result.results[0].value_count is None + + +def test_file_discovery_result_without_value_counts_parses(): + """A result from a server that predates value counts parses, with a blank status read as `None`.""" + result = FileDiscoveryResult.model_validate( + { + "id": 10, + "connection": {"id": "conn-1", "name": "my files"}, + "file_type": "csv", + "files": [{"path": "data/people.csv", "file_type": "csv", "value_count_status": ""}], + "results": [ + { + "locator": "email", + "data_types": ["string"], + "matches": [{"flagged_by": "MDD", "description": "Email", "label": "email"}], + } + ], + } + ) + + file = result.files[0] + assert file.value_count_status is None + assert file.value_counts == {} + assert result.results[0].value_count is None + + +def test_file_discovery_value_counts_read_by_locator(): + """A file's counts are read with a `Locator`, and listed with each locator in the form the results use.""" + result = FileDiscoveryResult.model_validate( + { + "id": 11, + "connection": {"id": "conn-1", "name": "my files"}, + "file_type": "json", + "files": [ + { + "path": "data/staff.json", + "file_type": "json", + "value_counts": {'["employees","*","email"]': 12, '["employees","*","phone"]': 4}, + "value_count_status": "counted", + } + ], + "results": [ + { + "locator": ["employees", "*", "email"], + "data_types": ["string"], + "matches": [{"flagged_by": "IDD", "description": "Email", "label": "email"}], + "value_count": 30, + } + ], + } + ) + + file = result.files[0] + assert file.get_value_count_of_locator(result.results[0].locator) == 12 + assert file.get_value_count_of_locator(["employees", "*", "phone"]) == 4 + assert file.get_value_count_of_locator(["employees", "*", "address"]) is None + assert file.parse_value_counts() == [(["employees", "*", "email"], 12), (["employees", "*", "phone"], 4)] + + +def test_file_discovery_value_count_of_non_ascii_locator(): + """A locator with a non-ASCII element matches its key, which the API sends with no escape.""" + result = FileDiscoveryResult.model_validate( + { + "id": 13, + "connection": {"id": "conn-1", "name": "my files"}, + "file_type": "json", + "files": [ + { + "path": "data/personnel.json", + "file_type": "json", + "value_counts": {'["personnel","*","prénom"]': 7}, + "value_count_status": "counted", + } + ], + "results": [ + { + "locator": ["personnel", "*", "prénom"], + "data_types": ["string"], + "matches": [{"flagged_by": "IDD", "description": "First name", "label": "first_name"}], + "value_count": 7, + } + ], + } + ) + + file = result.files[0] + assert file.get_value_count_of_locator(result.results[0].locator) == 7 + assert file.parse_value_counts() == [(["personnel", "*", "prénom"], 7)] + + +def test_file_discovery_without_value_counts_reads_no_count(): + """A file that carries no counts reports `None` for every locator, and lists nothing.""" + result = FileDiscoveryResult.model_validate( + { + "id": 12, + "connection": {"id": "conn-1", "name": "my files"}, + "file_type": "csv", + "files": [{"path": "data/people.csv", "file_type": "csv", "value_count_status": ""}], + "results": [ + { + "locator": "email", + "data_types": ["string"], + "matches": [{"flagged_by": "MDD", "description": "Email", "label": "email"}], + } + ], + } + ) + + file = result.files[0] + assert file.get_value_count_of_locator("email") is None + assert file.parse_value_counts() == [] + + def test_file_data_discovery_ignore_rules_serialize(): """`in_data_discovery.ignore_rules` round-trips into the wire payload.""" req = FileDataDiscoveryRequest( diff --git a/uv.lock b/uv.lock index 1a32e7c..929f8cb 100644 --- a/uv.lock +++ b/uv.lock @@ -419,7 +419,7 @@ toml = [ [[package]] name = "datamasque-python" -version = "1.2.2" +version = "1.2.3" source = { editable = "." } dependencies = [ { name = "pydantic" },