Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions src/a4d/clean/validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"""

import re
import unicodedata
from typing import Any

import polars as pl
Expand All @@ -29,8 +30,8 @@ def sanitize_str(text: str) -> str:
Punctuation, casing and spacing must not decide whether a clinic's typed
value matches an allowed one:
1. Convert to lowercase
2. Remove spaces
3. Remove special characters (keep only alphanumeric)
2. Fold accents to the plain letter (é -> e, đ -> d), so they are not dropped
3. Remove spaces and special characters (keep only a-z and 0-9)

Args:
text: String to sanitize
Expand All @@ -43,10 +44,14 @@ def sanitize_str(text: str) -> str:
'activeremote'
>>> sanitize_str("Lost Follow Up")
'lostfollowup'
>>> sanitize_str("Hà Nội*")
'hanoi'
"""
if not isinstance(text, str):
return text
return re.sub(r"[^a-z0-9]", "", text.lower())
text = unicodedata.normalize("NFKD", text.lower().replace("đ", "d"))
text = "".join(c for c in text if not unicodedata.combining(c))
return re.sub(r"[^a-z0-9]", "", text)


def load_validation_rules() -> dict[str, Any]:
Expand Down Expand Up @@ -298,7 +303,7 @@ def validate_province(
sanitizing both sides for comparison and setting an unrecognised province
to "Undefined". Sanitizing folds accents, which recovers spellings a
stricter comparison loses (`Thái Nguyễn` -> `Thái Nguyên`); this is safe
because the 209-entry province list has no two entries that sanitize
because the province list has no two entries that sanitize
alike, and validate_allowed_values raises if any two ever do.

Args:
Expand Down
60 changes: 60 additions & 0 deletions tests/test_clean/test_validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
from a4d.clean.validators import (
fix_patient_id,
load_validation_rules,
sanitize_str,
validate_all_columns,
validate_allowed_values,
validate_column_from_rules,
validate_province,
)
from a4d.config import settings

Expand Down Expand Up @@ -750,3 +752,61 @@ def test_validate_allowed_values_rejects_aliases_for_a_non_allowed_canonical():
allowed_values=["Active"],
aliases={"Active Remotte": ["Active - Remote"]},
)


def test_sanitize_str_folds_accents_instead_of_dropping_them():
"""An accented letter becomes its plain letter; it is not deleted ('Takéo' was 'tako')."""
assert sanitize_str("Takéo") == sanitize_str("Takeo") == "takeo"
assert sanitize_str("Nam Định") == sanitize_str("Namdinh") == "namdinh"
assert sanitize_str("Hưng Yên") == "hungyen"


def test_validate_allowed_values_accepts_a_value_typed_without_accents(collector):
"""A value typed without accents matches and is published with the official spelling."""
df = pl.DataFrame(
{
"file_name": ["test.xlsx"] * 3,
"sheet_name": "Jan26",
"patient_id": ["XX_QA001", "XX_QA002", "XX_QA003"],
"province": ["Takeo", "Ha Noi", "Dong Nai"],
}
)

result = validate_allowed_values(
df=df,
column="province",
allowed_values=["Takéo", "Hà Nội*", "Đồng Nai"],
replace_invalid=True,
)

assert result["province"].to_list() == ["Takéo", "Hà Nội*", "Đồng Nai"]
assert len(collector) == 0


def test_validate_province_accepts_names_typed_without_accents(collector):
"""With the real province list: names without accents are accepted, unknown names are not."""
df = pl.DataFrame(
{
"file_name": ["test.xlsx"] * 4,
"sheet_name": "Jan26",
"patient_id": ["XX_QA001", "XX_QA002", "XX_QA003", "XX_QA004"],
"province": ["Takeo", "Kratie", "Hanoi", "Atlantis"],
}
)

result = validate_province(df)

assert result["province"].to_list() == [
"Takéo",
"Kratié",
"Hà Nội*",
settings.error_val_character,
]


def test_no_two_provinces_sanitize_alike():
"""Folding accents must not make two provinces of the list the same."""
from a4d.reference.provinces import load_canonical_provinces

keys = [sanitize_str(p) for p in load_canonical_provinces()]
assert len(keys) == len(set(keys))
Loading