diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..97a40a8 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# ruff format, line length 100 +b84a31eee8373113ba20d9e89f4ff5b3d98cc58b diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63b393a..be0617a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,12 +6,16 @@ on: jobs: label-gate: uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/label-gate.yml@main + ruff: + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/ruff.yml@ruff-ty-workflows + ty: + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/ty.yml@ruff-ty-workflows build-test-sqlite: - uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test.yml@main + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-sqlite.yml@ruff-ty-workflows build-test-duckdb: - uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-duckdb.yml@main + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-duckdb.yml@ruff-ty-workflows build-test-postgres: - uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-postgres-v2.yml@main + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-postgres-v3.yml@ruff-ty-workflows with: postgres-db: orm_loader_test setup-commands: | diff --git a/README.md b/README.md index 18739b3..f6f221a 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ It supports: ```python from orm_loader.tables import ORMTableBase + class MyTable(ORMTableBase, Base): __tablename__ = "my_table" - ``` You can inherit from it directly or pick it up through one of the higher-level mixins. @@ -56,7 +56,6 @@ Features include: ```python class MyTable(FileLoadableTableInterface, ORMTableBase, Base): __tablename__ = "my_table" - ``` The main extension points here are reader choice and the normal SQLAlchemy model definitions themselves. Most downstream projects do not need to override much beyond the model schema. diff --git a/docs/loaders/conversion.md b/docs/loaders/conversion.md index ad8b628..df4d651 100644 --- a/docs/loaders/conversion.md +++ b/docs/loaders/conversion.md @@ -102,8 +102,8 @@ class HexHandler(ConvertingHandler): def arrow_type(self, sa_type): return pa.int64() - def parse(self, cleaned, sa_type): - ... # return Interpretation(values, invalid) + def parse(self, cleaned, sa_type): ... # return Interpretation(values, invalid) + policy = ConversionPolicy(handlers={MyHexType: HexHandler()}) ``` diff --git a/docs/registry/index.md b/docs/registry/index.md index 6c41fa1..474fde0 100644 --- a/docs/registry/index.md +++ b/docs/registry/index.md @@ -66,14 +66,15 @@ from orm_loader.tables import ORMTableBase Base = declarative_base() + class Person(ORMTableBase, Base): __tablename__ = "person" # columns omitted for brevity + class VisitOccurrence(ORMTableBase, Base): __tablename__ = "visit_occurrence" # columns omitted for brevity - ``` ## 2. Create a ModelRegistry @@ -81,14 +82,9 @@ class VisitOccurrence(ORMTableBase, Base): The registry is the coordination object that holds models and specs. ```python - from orm_loader.registry import ModelRegistry -registry = ModelRegistry( - model_version="5.4", - model_name="OMOP" -) - +registry = ModelRegistry(model_version="5.4", model_name="OMOP") ``` ## 3. Load Specifications @@ -116,10 +112,12 @@ At this point, the registry knows: Models can be registered explicitly: ```python -registry.register_models([ - Person, - VisitOccurrence, -]) +registry.register_models( + [ + Person, + VisitOccurrence, + ] +) ``` Or discovered automatically from a package: diff --git a/docs/tables/mat_view.md b/docs/tables/mat_view.md index f03a7be..8ba8ae5 100644 --- a/docs/tables/mat_view.md +++ b/docs/tables/mat_view.md @@ -24,18 +24,12 @@ The class declaration is the integration point that downstream code can import a ```python class RecentObservationMV(Base, MaterializedViewMixin): __mv_name__ = "mv_recent_observation" - __mv_select__ = ( - sa.select( - Observation.observation_id, - Observation.person_id, - Observation.observation_date, - Observation.value_as_number, - ) - .where( - Observation.observation_date - >= sa.func.current_date() - sa.text("INTERVAL '30 days'") - ) - ) + __mv_select__ = sa.select( + Observation.observation_id, + Observation.person_id, + Observation.observation_date, + Observation.value_as_number, + ).where(Observation.observation_date >= sa.func.current_date() - sa.text("INTERVAL '30 days'")) __mv_dependencies__ = {"observation"} __mv_indexes__ = ( MaterializedViewIndex( @@ -120,12 +114,14 @@ class VocabSummaryMV(MaterializedViewMixin): __mv_select__ = ... __mv_schema_tag__ = Role.VOCAB.value # resolves via the connection's vocab schema + # Uses __mv_schema_tag__ ("primary" by default) via the connection's own schema_translate_map. RecentObservationMV.create_mv(engine) # Override for one call. RecentObservationMV.create_mv(engine, schema_tag=Role.VOCAB.value) + # Declaratively mapped: the mapped table's own schema is authoritative, no # __mv_schema_tag__ needed (or consulted) at all. class VocabPatientSummaryMV(Base, MaterializedViewMixin): diff --git a/pyproject.toml b/pyproject.toml index 001065b..3433784 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,8 +55,8 @@ dev = [ "oa-configurator[postgres,duckdb]>=2.0.0,<3", "pandas>=2.3.3", "pytest>=9.0.3", - "ty>=0.0.59", - "ruff>=0.14.11", + "ty==0.0.90", + "ruff==0.17.1", "mkdocs-material>=9.7.7", "mkdocstrings-python>=2.0.1", "requests>=2.33.0", @@ -74,11 +74,6 @@ cache-keys = [{ file = "pyproject.toml" }, { git = { commit = true, tags = true [tool.ruff] line-length = 100 -target-version = "py311" - -[tool.mypy] -python_version = "3.11" -strict = true [tool.pytest.ini_options] testpaths = ["tests"] @@ -87,17 +82,11 @@ python_classes = ["Test*"] python_functions = ["test_*"] addopts = "-ra -m 'not db_dialect'" -[tool.pyright] -reportMissingTypeStubs = false - -# Temporary ty override until #43 picks it up in a follow-up -# with pyarrow-stubs, which type the compute functions too narrowly -[[tool.ty.overrides]] -include = ["src/orm_loader/conversion/**", "src/orm_loader/loaders/processor.py", "src/orm_loader/loaders/readers/**"] +[tool.ty.rules] +# raises on stale suppressions +unused-ignore-comment = "error" -[tool.ty.overrides.rules] -unresolved-attribute = "ignore" -invalid-argument-type = "ignore" -invalid-return-type = "ignore" -invalid-assignment = "ignore" -no-matching-overload = "ignore" +[tool.ty.analysis] +# pyarrow ships no inline types and pyarrow-stubs is being retired. Upstream typing is +# not finalised yet (https://github.com/apache/arrow/discussions/45919), so pyarrow is Any. +replace-imports-with-any = ["pyarrow", "pyarrow.**"] diff --git a/src/orm_loader/backends/__init__.py b/src/orm_loader/backends/__init__.py index c8e9cca..891c239 100644 --- a/src/orm_loader/backends/__init__.py +++ b/src/orm_loader/backends/__init__.py @@ -1,23 +1,24 @@ -from .duckdb import DuckDBBackend -from .postgres import PostgresBackend -from .resolve import resolve_backend -from .sqlite import SQLiteBackend +from ..mappers.materialised_view_errors import ( + ConcurrentRefreshNotEligibleError, + MaterializationError, + MaterializationFailure, + MaterializationOperation, +) from .base import ( + STAGING_SCHEMA, DatabaseBackend, DedupeResult, Dialect, ForeignKeyViolationError, - STAGING_SCHEMA, staging_schema_claim, ) -from ..mappers.materialised_view_errors import ( - ConcurrentRefreshNotEligibleError, - MaterializationError, - MaterializationFailure, - MaterializationOperation, -) +from .duckdb import DuckDBBackend +from .postgres import PostgresBackend +from .resolve import resolve_backend +from .sqlite import SQLiteBackend __all__ = [ + "STAGING_SCHEMA", "ConcurrentRefreshNotEligibleError", "DatabaseBackend", "DedupeResult", @@ -28,7 +29,6 @@ "MaterializationFailure", "MaterializationOperation", "PostgresBackend", - "STAGING_SCHEMA", "SQLiteBackend", "resolve_backend", "staging_schema_claim", diff --git a/src/orm_loader/backends/base.py b/src/orm_loader/backends/base.py index 2e453c1..fc6d02e 100644 --- a/src/orm_loader/backends/base.py +++ b/src/orm_loader/backends/base.py @@ -5,7 +5,7 @@ from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Type +from typing import TYPE_CHECKING, Any, ClassVar import pyarrow as pa import sqlalchemy as sa @@ -68,7 +68,9 @@ class ForeignKeyViolationError(RuntimeError): qualifier: ClassVar[str] = "" - def __init__(self, constraint: sa.ForeignKeyConstraint, orphan_keys: list[tuple[Any, ...]]) -> None: + def __init__( + self, constraint: sa.ForeignKeyConstraint, orphan_keys: list[tuple[Any, ...]] + ) -> None: self.constraint = constraint self.orphan_keys = orphan_keys local_cols = ", ".join(c.name for c in constraint.columns) @@ -88,10 +90,12 @@ class DeferredForeignKeyViolationError(ForeignKeyViolationError): STAGING_SCHEMA: str = "staging" -def staging_schema_claim(physical_schema: str | None = None, *, reserved: bool = True) -> SchemaClaim: +def staging_schema_claim( + physical_schema: str | None = None, *, reserved: bool = True +) -> SchemaClaim: """SchemaClaim reserving orm-loader's staging schema. Since orm-loader doesn't have an engine creation step of its own, - this claim needs to be registered by the caller ingesting tables into + this claim needs to be registered by the caller ingesting tables into the database through orm-loader. Add to your own ``create_engines(schema_claims=[...])`` call before @@ -202,7 +206,9 @@ def qualified_staging_name(self, tablename: str) -> str: e.g. '"staging"."_staging_concept"' or '"_staging_concept"'. """ return qualified( - self.identifier_preparer, self.staging_name_for_table(tablename), physical_schema=self.staging_schema + self.identifier_preparer, + self.staging_name_for_table(tablename), + physical_schema=self.staging_schema, ) @property @@ -283,7 +289,7 @@ def _as_connection( def _insertable_column_names( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], staging: sa.Table | None = None, ) -> list[str]: """ @@ -301,7 +307,7 @@ def _insertable_column_names( @staticmethod def _staging_value( - table_cls: Type["FileTableProtocol"], staging: sa.Table, name: str + table_cls: type[FileTableProtocol], staging: sa.Table, name: str ) -> sa.ColumnElement[Any]: """Apply a SQL-expression default to null staging values during merge.""" column = staging.c[name] @@ -310,7 +316,7 @@ def _staging_value( def deferred_fk_constraints( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], ) -> tuple[sa.ForeignKeyConstraint, ...]: """FK constraints on *table_cls* that this backend never enforces at the DB level. @@ -321,7 +327,7 @@ def deferred_fk_constraints( def validate_deferred_fks( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, ) -> None: """Raise if any row in *table_cls* violates one of its deferred_fk_constraints(). @@ -341,7 +347,7 @@ def validate_deferred_fks( def validate_foreign_keys( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, ) -> None: """Raise if any row in *table_cls* references a row that does not exist. @@ -365,7 +371,9 @@ def validate_foreign_keys( cross_database = { entry.constraint for entry in unenforced_foreign_keys( - tuple(metadata.tables.values()), resolved=self.resolved, physical_schemas=self.physical_schemas + tuple(metadata.tables.values()), + resolved=self.resolved, + physical_schemas=self.physical_schemas, ) if entry.reason == "cross_database" } @@ -434,7 +442,10 @@ def _validate_cross_database_fk( condition = remote_cols[0].in_(value for (value,) in local_values) else: condition = sa.tuple_(*remote_cols).in_(tuple(value) for value in local_values) - existing = {tuple(row) for row in remote_conn.execute(sa.select(*remote_cols).where(condition))} + existing = { + tuple(row) + for row in remote_conn.execute(sa.select(*remote_cols).where(condition)) + } finally: dispose_engines(primary_engine, vocab_engine) @@ -477,9 +488,13 @@ def _validate_one_fk( remote_cols = [element.column for element in fk.elements] referred_alias = remote_cols[0].table.alias() aliased_remote_cols = [referred_alias.c[col.name] for col in remote_cols] - matches = sa.and_(*(local_cols[i] == aliased_remote_cols[i] for i in range(len(local_cols)))) + matches = sa.and_( + *(local_cols[i] == aliased_remote_cols[i] for i in range(len(local_cols))) + ) not_null = sa.and_(*(col.isnot(None) for col in local_cols)) - orphan_exists = sa.exists(sa.select(sa.literal(1)).select_from(referred_alias).where(matches)) + orphan_exists = sa.exists( + sa.select(sa.literal(1)).select_from(referred_alias).where(matches) + ) orphans = session.execute( sa.select(*local_cols).where(not_null, ~orphan_exists).limit(5) @@ -489,7 +504,7 @@ def _validate_one_fk( def create_staging_table( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, ) -> sa.Table: """ @@ -520,20 +535,24 @@ def create_staging_table( if c.computed is None ] staging = self._staging_table(table_cls, columns) - staging.create(bind=session.connection(bind_arguments={"mapper": table_cls}), checkfirst=True) + staging.create( + bind=session.connection(bind_arguments={"mapper": table_cls}), checkfirst=True + ) session.commit() return staging def drop_staging_table( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, ) -> None: """Drop a staging table and what belongs to its row key, if they exist.""" staging = self._staging_table(table_cls, []) staging.drop(bind=session.connection(bind_arguments={"mapper": table_cls}), checkfirst=True) - def _staging_table(self, table_cls: Type["FileTableProtocol"], columns: Sequence[sa.Column[Any]]) -> sa.Table: + def _staging_table( + self, table_cls: type[FileTableProtocol], columns: Sequence[sa.Column[Any]] + ) -> sa.Table: """ Return the definition of the staging table, which create and drop both use. @@ -560,7 +579,7 @@ def _staging_table(self, table_cls: Type["FileTableProtocol"], columns: Sequence @abstractmethod def insert_batches( self, - ctx: "LoaderContext", + ctx: LoaderContext, staging: sa.Table, batches: Iterable[pa.Table], columns: Sequence[str], @@ -611,17 +630,17 @@ def _staging_reference(staging: sa.Table, connection: Connection) -> str: schema = translate.get(staging.schema, staging.schema) return qualified(connection, staging.name, physical_schema=schema) - def count_staging(self, ctx: "LoaderContext", staging: sa.Table) -> int: + def count_staging(self, ctx: LoaderContext, staging: sa.Table) -> int: """Return the number of rows in the staging table.""" count = sa.select(sa.func.count()).select_from(staging) return ctx.session.execute(count, bind_arguments={"mapper": ctx.tableclass}).scalar_one() def deduplicate_staging( self, - ctx: "LoaderContext", + ctx: LoaderContext, staging: sa.Table, key_columns: Sequence[str], - ) -> "DedupeResult": + ) -> DedupeResult: """ Keep the first row for each primary key in the staging table. @@ -652,11 +671,15 @@ def deduplicate_staging( ).subquery() duplicates = sa.select(ranked.c.row_key).where(ranked.c.position > 1) before = self.count_staging(ctx, staging) - ctx.session.execute(sa.delete(staging).where(row_key.in_(duplicates)), bind_arguments=arguments) + ctx.session.execute( + sa.delete(staging).where(row_key.in_(duplicates)), bind_arguments=arguments + ) after = self.count_staging(ctx, staging) return DedupeResult(tuple(key_columns), before, before - after) - def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> str | int | None: + def disable_fk_check( + self, session: so.Session, *, mapper: Any | None = None + ) -> str | int | None: """Relax foreign key enforcement and return the state to restore. The default relaxes nothing and returns ``None``, which is right for a @@ -675,7 +698,7 @@ def restore_fk_check( def check_replace_feasible( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], pk_cols: list[str], session: so.Session, ) -> None: @@ -691,7 +714,7 @@ def check_replace_feasible( @abstractmethod def merge_replace( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, pk_cols: list[str], staging: sa.Table, @@ -710,7 +733,7 @@ def merge_replace( @abstractmethod def merge_upsert( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, pk_cols: list[str], staging: sa.Table, @@ -727,7 +750,7 @@ def merge_upsert( @abstractmethod def merge_insert( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, staging: sa.Table, *, @@ -742,7 +765,7 @@ def merge_insert( def _pagination_column( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], staging: sa.Table, ) -> sa.ColumnElement[int]: """Column this backend orders a staging-table scan by, for pagination. @@ -755,7 +778,7 @@ def _pagination_column( def _ensure_pagination_index( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], staging: sa.Table, session: so.Session, ) -> None: @@ -776,7 +799,7 @@ def _commits_between_pagination_batches(self) -> bool: def _paginate_batches( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], staging: sa.Table, session: so.Session, merge_batch_size: int | None, @@ -787,7 +810,7 @@ def _paginate_batches( ------ sa.ColumnElement[bool] or None A predicate to filter the staging table to a single batch of rows. - None if merge_batch_size is None or the staging table is smaller than + None if merge_batch_size is None or the staging table is smaller than merge_batch_size. """ if merge_batch_size is not None: @@ -809,21 +832,21 @@ def _paginate_batches( def _commit_batch_if_needed( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, batch_range: sa.ColumnElement[bool] | None, *, validate_deferred_fks: bool = True, ) -> None: - """Validate deferred FKs and commit after one pagination batch, + """Validate deferred FKs and commit after one pagination batch, if this backend commits between batches. - Called from every merge_*() method after each batch of rows is processed. + Called from every merge_*() method after each batch of rows is processed. Instead of committing directly, a batch that violates a deferred FK is caught before the next batch starts. Without this, an earlier batch's commit cannot be undone once a later batch's load_file()-level validation fails. - No-op when there is nothing to validate to not add cost. This is true for every + No-op when there is nothing to validate to not add cost. This is true for every backend but DuckDB, and for DuckDB whenever table_cls has no deferred/ unenforced FK at all. """ @@ -834,7 +857,7 @@ def _commit_batch_if_needed( session.commit() def _self_referential_deferred_fks( - self, table_cls: Type["FileTableProtocol"] + self, table_cls: type[FileTableProtocol] ) -> tuple[sa.ForeignKeyConstraint, ...]: """Deferred FKs on table_cls that point back at table_cls itself. @@ -845,7 +868,9 @@ def _self_referential_deferred_fks( dependency order instead. """ target = table_cls.__table__ - return tuple(fk for fk in self.deferred_fk_constraints(table_cls) if fk.referred_table is target) + return tuple( + fk for fk in self.deferred_fk_constraints(table_cls) if fk.referred_table is target + ) @contextmanager def relaxed_foreign_keys( @@ -890,4 +915,3 @@ def relaxed_foreign_keys( if previous_state is not None: self.restore_fk_check(session, previous_state, mapper=mapper) logger.info(f"{scope}Foreign key checks restored.") - diff --git a/src/orm_loader/backends/duckdb.py b/src/orm_loader/backends/duckdb.py index ff5aeda..0799191 100644 --- a/src/orm_loader/backends/duckdb.py +++ b/src/orm_loader/backends/duckdb.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Iterable, Iterator, Sequence -from typing import TYPE_CHECKING, Any, ClassVar, Type, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast import pyarrow as pa import sqlalchemy as sa @@ -48,8 +48,8 @@ def identifier_preparer(self) -> IdentifierPreparer: def _unenforced_constraints( self, tables: tuple[sa.Table, ...] ) -> tuple[UnenforcedForeignKey, ...]: - """Every FK in *tables* DuckDB will not enforce, including reason: - - omitted (cross-database/cross-schema) or + """Every FK in *tables* DuckDB will not enforce, including reason: + - omitted (cross-database/cross-schema) or - created but never added (cyclic/``use_alter=True``). Delegates to oa-configurator's :func:`unenforced_foreign_keys`, the @@ -71,7 +71,7 @@ def _unenforced_constraints( def deferred_fk_constraints( self, - table_cls: Type["FileTableProtocol"], + table_cls: type[FileTableProtocol], ) -> tuple[sa.ForeignKeyConstraint, ...]: """FK constraints on *table_cls* this backend's database will not enforce: - omitted (cross-database/cross-schema), or @@ -82,10 +82,15 @@ def deferred_fk_constraints( if not own_fks: return () metadata = table_cls.__table__.metadata - unenforced = {entry.constraint for entry in self._unenforced_constraints(tuple(metadata.tables.values()))} + unenforced = { + entry.constraint + for entry in self._unenforced_constraints(tuple(metadata.tables.values())) + } return tuple(fk for fk in own_fks if fk in unenforced) - def validate_deferred_fks(self, table_cls: Type["FileTableProtocol"], session: so.Session) -> None: + def validate_deferred_fks( + self, table_cls: type[FileTableProtocol], session: so.Session + ) -> None: """Validate every deferred FK on table_cls, including a cross-database one. A cross-schema or cyclic deferred FK points to a table in the same @@ -103,11 +108,15 @@ def validate_deferred_fks(self, table_cls: Type["FileTableProtocol"], session: s if entry.constraint not in own_fks: continue if entry.reason == "cross_database": - self._validate_cross_database_fk(entry.constraint, session, DeferredForeignKeyViolationError) + self._validate_cross_database_fk( + entry.constraint, session, DeferredForeignKeyViolationError + ) else: self._validate_one_fk(entry.constraint, session, DeferredForeignKeyViolationError) - def validate_foreign_keys(self, table_cls: Type["FileTableProtocol"], session: so.Session) -> None: + def validate_foreign_keys( + self, table_cls: type[FileTableProtocol], session: so.Session + ) -> None: """Raise if any row in *table_cls* references a row that does not exist. DuckDB enforces the foreign keys it creates, so only the ones it @@ -125,7 +134,10 @@ def _tables_referencing(self, target: sa.Table) -> set[sa.Table]: """ tables = tuple(target.metadata.tables.values()) incoming = tuple( - fk for other in tables if other is not target for fk in other.foreign_key_constraints + fk + for other in tables + if other is not target + for fk in other.foreign_key_constraints if fk.referred_table is target ) if not incoming: @@ -142,21 +154,28 @@ def _is_referenced_by_existing_rows(self, target: sa.Table, session: so.Session) bind = session.get_bind() inspector = sa.inspect(bind) return any( - inspector.has_table(other.name, schema=physical_schema_of(bind, schema_tag=other.schema)) - and session.execute(sa.select(sa.literal(1)).select_from(other).limit(1)).first() is not None + inspector.has_table( + other.name, schema=physical_schema_of(bind, schema_tag=other.schema) + ) + and session.execute(sa.select(sa.literal(1)).select_from(other).limit(1)).first() + is not None for other in self._tables_referencing(target) ) - def _staging_table(self, table_cls: type["FileTableProtocol"], columns: Sequence[sa.Column[Any]]) -> sa.Table: + def _staging_table( + self, table_cls: type[FileTableProtocol], columns: Sequence[sa.Column[Any]] + ) -> sa.Table: """Number the rows with a sequence, which DuckDB creates and drops with the table.""" name = self.staging_name_for_table(table_cls.__tablename__) sequence = sa.Sequence(f"{name}_rownum_seq", schema=self.staging_schema) - row_key = sa.Column("_rownum", sa.BigInteger, sequence, server_default=sequence.next_value()) + row_key = sa.Column( + "_rownum", sa.BigInteger, sequence, server_default=sequence.next_value() + ) return sa.Table(name, sa.MetaData(), *columns, row_key, schema=self.staging_schema) def _paginate_batches( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], staging: sa.Table, session: so.Session, merge_batch_size: int | None, @@ -176,7 +195,7 @@ def _paginate_batches( def _self_referential_batches( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], staging: sa.Table, session: so.Session, merge_batch_size: int | None, @@ -213,20 +232,24 @@ def _self_referential_batches( remote_cols = [cast(sa.ColumnElement[Any], element.column) for element in fk.elements] not_null = sa.and_(*(col.isnot(None) for col in local_cols)) has_match = sa.exists( - sa.select(sa.literal(1)).select_from(target).where( - sa.and_(*(local_cols[i] == remote_cols[i] for i in range(len(remote_cols)))) - ) + sa.select(sa.literal(1)) + .select_from(target) + .where(sa.and_(*(local_cols[i] == remote_cols[i] for i in range(len(remote_cols))))) ) per_fk.append((not_null, has_match)) ready = sa.and_(*(sa.or_(sa.not_(not_null), has_match) for not_null, has_match in per_fk)) while True: - query = sa.select(order_column).where(ready).order_by(order_column).limit(merge_batch_size) + query = ( + sa.select(order_column).where(ready).order_by(order_column).limit(merge_batch_size) + ) keys = session.execute(query).scalars().all() if keys: chunk: sa.ColumnElement[bool] = order_column.in_(keys) else: - remaining = session.execute(sa.select(sa.func.count()).select_from(staging)).scalar_one() + remaining = session.execute( + sa.select(sa.func.count()).select_from(staging) + ).scalar_one() if not remaining: return chunk = sa.true() @@ -243,7 +266,7 @@ def _commits_between_pagination_batches(self) -> bool: def insert_batches( self, - ctx: "LoaderContext", + ctx: LoaderContext, staging: sa.Table, batches: Iterable[pa.Table], columns: Sequence[str], @@ -274,7 +297,7 @@ def insert_batches( def check_replace_feasible( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], pk_cols: list[str], session: so.Session, ) -> None: @@ -293,9 +316,12 @@ def check_replace_feasible( target = table_cls.__table__ target_columns = {column.name: column for column in target.columns} update_cols = [c for c in self._insertable_column_names(table_cls) if c not in pk_cols] - deferred_cols = {col.name for fk in self.deferred_fk_constraints(table_cls) for col in fk.columns} + deferred_cols = { + col.name for fk in self.deferred_fk_constraints(table_cls) for col in fk.columns + } enforced_fk_cols = [ - c for c in update_cols + c + for c in update_cols if c in target_columns and target_columns[c].foreign_keys and c not in deferred_cols ] if not enforced_fk_cols: @@ -313,7 +339,7 @@ def check_replace_feasible( def merge_replace( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, pk_cols: list[str], staging: sa.Table, @@ -344,7 +370,9 @@ def merge_replace( self.check_replace_feasible(table_cls, pk_cols, session) target = table_cls.__table__ target_columns = {column.name: column for column in target.columns} - update_cols = [c for c in self._insertable_column_names(table_cls, staging) if c not in pk_cols] + update_cols = [ + c for c in self._insertable_column_names(table_cls, staging) if c not in pk_cols + ] pk_join = sa.and_(*(target_columns[c] == staging.c[c] for c in pk_cols)) def _batch_keys(extra: sa.ColumnElement[bool] | None = None) -> sa.Select[Any]: @@ -359,9 +387,16 @@ def _pk_in(keys: sa.Select[Any]) -> sa.ColumnElement[bool]: return sa.tuple_(*(target_columns[c] for c in pk_cols)).in_(keys) def _update(extra: sa.ColumnElement[bool] | None = None) -> None: - stmt = sa.update(target).values( - {target_columns[c]: self._staging_value(table_cls, staging, c) for c in update_cols} - ).where(pk_join) + stmt = ( + sa.update(target) + .values( + { + target_columns[c]: self._staging_value(table_cls, staging, c) + for c in update_cols + } + ) + .where(pk_join) + ) if extra is not None: stmt = stmt.where(_pk_in(_batch_keys(extra))) session.execute(stmt) @@ -380,17 +415,25 @@ def _drop_matched_from_staging(extra: sa.ColumnElement[bool] | None = None) -> N self_referential_fks = self._self_referential_deferred_fks(table_cls) if self_referential_fks: insertable_cols = self._insertable_column_names(table_cls, staging) - non_paginated_select = sa.select(*(self._staging_value(table_cls, staging, c) for c in insertable_cols)) + non_paginated_select = sa.select( + *(self._staging_value(table_cls, staging, c) for c in insertable_cols) + ) def _insert(select_: Any) -> sa.Insert: - return sa.insert(target).from_select([target_columns[c] for c in insertable_cols], select_) + return sa.insert(target).from_select( + [target_columns[c] for c in insertable_cols], select_ + ) def _process_batch(batch_range: sa.ColumnElement[bool] | None) -> None: # batch_range's rows already in target are updated; what's # left in staging afterward is new and gets inserted here. _update(batch_range) _drop_matched_from_staging(batch_range) - select_ = non_paginated_select if batch_range is None else non_paginated_select.where(batch_range) + select_ = ( + non_paginated_select + if batch_range is None + else non_paginated_select.where(batch_range) + ) session.execute(_insert(select_)) # Runs even when batch_range is None: merge_insert() runs # next and needs staging fully drained. @@ -412,7 +455,7 @@ def _process_batch(batch_range: sa.ColumnElement[bool] | None) -> None: def merge_upsert( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, pk_cols: list[str], staging: sa.Table, @@ -434,11 +477,17 @@ def _upsert(select_: Any) -> sa.Insert: .on_conflict_do_nothing(index_elements=[target_columns[c] for c in pk_cols]) ) - non_paginated_select = sa.select(*(self._staging_value(table_cls, staging, c) for c in insertable_cols)) + non_paginated_select = sa.select( + *(self._staging_value(table_cls, staging, c) for c in insertable_cols) + ) self_referential = bool(self._self_referential_deferred_fks(table_cls)) for batch_range in self._paginate_batches(table_cls, staging, session, merge_batch_size): - select_ = non_paginated_select if batch_range is None else non_paginated_select.where(batch_range) + select_ = ( + non_paginated_select + if batch_range is None + else non_paginated_select.where(batch_range) + ) session.execute(_upsert(select_)) if self_referential and batch_range is not None: # Must be removed from staging in this same transaction, @@ -450,7 +499,7 @@ def _upsert(select_: Any) -> sa.Insert: def merge_insert( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, staging: sa.Table, *, @@ -461,15 +510,23 @@ def merge_insert( target = table_cls.__table__ target_columns = {column.name: column for column in target.columns} insertable_cols = self._insertable_column_names(table_cls, staging) - non_paginated_select = sa.select(*(self._staging_value(table_cls, staging, c) for c in insertable_cols)) + non_paginated_select = sa.select( + *(self._staging_value(table_cls, staging, c) for c in insertable_cols) + ) def _insert(select_: Any) -> sa.Insert: - return sa.insert(target).from_select([target_columns[c] for c in insertable_cols], select_) + return sa.insert(target).from_select( + [target_columns[c] for c in insertable_cols], select_ + ) self_referential = bool(self._self_referential_deferred_fks(table_cls)) for batch_range in self._paginate_batches(table_cls, staging, session, merge_batch_size): - select_ = non_paginated_select if batch_range is None else non_paginated_select.where(batch_range) + select_ = ( + non_paginated_select + if batch_range is None + else non_paginated_select.where(batch_range) + ) session.execute(_insert(select_)) if self_referential and batch_range is not None: # Must be removed from staging in this same transaction, diff --git a/src/orm_loader/backends/postgres.py b/src/orm_loader/backends/postgres.py index e555009..5376f83 100644 --- a/src/orm_loader/backends/postgres.py +++ b/src/orm_loader/backends/postgres.py @@ -2,8 +2,8 @@ import io import logging -from contextlib import contextmanager, nullcontext from collections.abc import Iterable, Mapping, Sequence +from contextlib import contextmanager, nullcontext from typing import TYPE_CHECKING, Any, cast import pyarrow as pa @@ -19,13 +19,13 @@ from sqlalchemy.exc import OperationalError from sqlalchemy.sql.compiler import IdentifierPreparer -from ..mappers.materialised_view_mixin import MaterializedViewBackend from ..mappers.materialised_view_errors import ( ConcurrentRefreshNotEligibleError, MaterializationError, MaterializationFailure, MaterializationOperation, ) +from ..mappers.materialised_view_mixin import MaterializedViewBackend from .base import DatabaseBackend if TYPE_CHECKING: @@ -73,7 +73,9 @@ def dialect(self) -> Dialect: def identifier_preparer(self) -> IdentifierPreparer: return postgresql.dialect().identifier_preparer - def _staging_table(self, table_cls: type["FileTableProtocol"], columns: Sequence[sa.Column[Any]]) -> sa.Table: + def _staging_table( + self, table_cls: type[FileTableProtocol], columns: Sequence[sa.Column[Any]] + ) -> sa.Table: """Unlogged, with an identity column, which paginates the merge in O(N log N) without an index.""" return sa.Table( self.staging_name_for_table(table_cls.__tablename__), @@ -86,7 +88,7 @@ def _staging_table(self, table_cls: type["FileTableProtocol"], columns: Sequence def insert_batches( self, - ctx: "LoaderContext", + ctx: LoaderContext, staging: sa.Table, batches: Iterable[pa.Table], columns: Sequence[str], @@ -101,12 +103,16 @@ def insert_batches( cursor = connection.connection.cursor() if not hasattr(cursor, "copy"): cursor.close() - raise RuntimeError("The PostgreSQL sink needs a psycopg 3 connection, whose cursors provide copy()") + raise RuntimeError( + "The PostgreSQL sink needs a psycopg 3 connection, whose cursors provide copy()" + ) table_ref = self._staging_reference(staging, connection) column_list = ", ".join(self.identifier_preparer.quote(c) for c in columns) options = pacsv.WriteOptions(include_header=False, quoting_style="needed") try: - with cursor.copy(f"COPY {table_ref} ({column_list}) FROM STDIN WITH (FORMAT csv)") as copy: + with cursor.copy( + f"COPY {table_ref} ({column_list}) FROM STDIN WITH (FORMAT csv)" + ) as copy: for table in batches: buffer = io.BytesIO() pacsv.write_csv(table, buffer, options) @@ -198,7 +204,7 @@ def _commits_between_pagination_batches(self) -> bool: def merge_replace( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, pk_cols: list[str], staging: sa.Table, @@ -221,7 +227,7 @@ def merge_replace( def merge_upsert( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, pk_cols: list[str], staging: sa.Table, @@ -242,10 +248,16 @@ def _upsert(select_: Any) -> sa.Insert: .on_conflict_do_nothing(index_elements=[target_columns[c] for c in pk_cols]) ) - non_paginated_select = sa.select(*(self._staging_value(table_cls, staging, c) for c in insertable_cols)) + non_paginated_select = sa.select( + *(self._staging_value(table_cls, staging, c) for c in insertable_cols) + ) for batch_range in self._paginate_batches(table_cls, staging, session, merge_batch_size): - select_ = non_paginated_select if batch_range is None else non_paginated_select.where(batch_range) + select_ = ( + non_paginated_select + if batch_range is None + else non_paginated_select.where(batch_range) + ) session.execute(_upsert(select_)) self._commit_batch_if_needed( table_cls, session, batch_range, validate_deferred_fks=validate_deferred_fks @@ -253,7 +265,7 @@ def _upsert(select_: Any) -> sa.Insert: def merge_insert( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, staging: sa.Table, *, @@ -269,13 +281,21 @@ def merge_insert( target = table_cls.__table__ target_columns = {column.name: column for column in target.columns} insertable_cols = self._insertable_column_names(table_cls, staging) - non_paginated_select = sa.select(*(self._staging_value(table_cls, staging, c) for c in insertable_cols)) + non_paginated_select = sa.select( + *(self._staging_value(table_cls, staging, c) for c in insertable_cols) + ) def _insert(select_: Any) -> sa.Insert: - return sa.insert(target).from_select([target_columns[c] for c in insertable_cols], select_) + return sa.insert(target).from_select( + [target_columns[c] for c in insertable_cols], select_ + ) for batch_range in self._paginate_batches(table_cls, staging, session, merge_batch_size): - select_ = non_paginated_select if batch_range is None else non_paginated_select.where(batch_range) + select_ = ( + non_paginated_select + if batch_range is None + else non_paginated_select.where(batch_range) + ) session.execute(_insert(select_)) self._commit_batch_if_needed( table_cls, session, batch_range, validate_deferred_fks=validate_deferred_fks @@ -321,19 +341,18 @@ def refresh_materialized_view( *, schema: str | None = None, concurrently: bool = False, - declared_indexes: tuple["MaterializedViewIndex", ...] = (), + declared_indexes: tuple[MaterializedViewIndex, ...] = (), ) -> None: with self._as_connection(bind) as conn: - if concurrently: - if not any(index.unique for index in declared_indexes): - raise ConcurrentRefreshNotEligibleError( - MaterializationFailure( - operation=MaterializationOperation.REFRESH, - schema=schema, - name=name, - reason="no simple unique index is declared", - ) + if concurrently and not any(index.unique for index in declared_indexes): + raise ConcurrentRefreshNotEligibleError( + MaterializationFailure( + operation=MaterializationOperation.REFRESH, + schema=schema, + name=name, + reason="no simple unique index is declared", ) + ) safe_name = qualified(conn, name, physical_schema=schema) concurrency = "CONCURRENTLY " if concurrently else "" @@ -369,7 +388,9 @@ def drop_materialized_view( try: conn.execute( DropMaterializedView( - qualified(conn, name, physical_schema=schema), if_exists=if_exists, cascade=cascade + qualified(conn, name, physical_schema=schema), + if_exists=if_exists, + cascade=cascade, ) ) except Exception as error: @@ -387,7 +408,7 @@ def create_materialized_view_index( self, bind: Engine | Connection, name: str, - index: "MaterializedViewIndex", + index: MaterializedViewIndex, *, schema: str | None = None, if_not_exists: bool = True, @@ -398,7 +419,9 @@ def create_materialized_view_index( try: conn.execute( CreateMaterializedViewIndex( - qualified(conn, name, physical_schema=schema), index, if_not_exists=if_not_exists + qualified(conn, name, physical_schema=schema), + index, + if_not_exists=if_not_exists, ) ) except Exception as error: diff --git a/src/orm_loader/backends/resolve.py b/src/orm_loader/backends/resolve.py index 0b69066..89c6e98 100644 --- a/src/orm_loader/backends/resolve.py +++ b/src/orm_loader/backends/resolve.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, cast +from typing import Any import sqlalchemy.orm as so from oa_configurator import Bindable, UnregisteredSchemaTagError, physical_schema_of @@ -18,17 +18,15 @@ def _dialect(bindable: Bindable) -> Dialect: - if hasattr(bindable, "dialect"): - dialect_name = cast(Any, bindable).dialect.name - else: + dialect = getattr(bindable, "dialect", None) + if dialect is None: raise TypeError(f"Unsupported bindable type: {type(bindable)!r}") + dialect_name = dialect.name try: return Dialect(dialect_name) except ValueError as exc: - raise NotImplementedError( - f"Unsupported SQLAlchemy dialect '{dialect_name}'" - ) from exc + raise NotImplementedError(f"Unsupported SQLAlchemy dialect '{dialect_name}'") from exc def resolve_backend( diff --git a/src/orm_loader/backends/sqlite.py b/src/orm_loader/backends/sqlite.py index 76f39dc..396999c 100644 --- a/src/orm_loader/backends/sqlite.py +++ b/src/orm_loader/backends/sqlite.py @@ -1,7 +1,7 @@ from __future__ import annotations -from collections.abc import Iterable, Mapping, Sequence import logging +from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any import pyarrow as pa @@ -63,8 +63,7 @@ def _normalize_fk_check_state(previous_state: str | int) -> str: if normalised in {"0", "OFF"}: return "OFF" raise ValueError( - f"Invalid SQLite foreign_keys state {previous_state!r}. " - "Expected 0, 1, 'OFF', or 'ON'." + f"Invalid SQLite foreign_keys state {previous_state!r}. Expected 0, 1, 'OFF', or 'ON'." ) @property @@ -85,7 +84,7 @@ def default_index_strategy(self) -> str: def insert_batches( self, - ctx: "LoaderContext", + ctx: LoaderContext, staging: sa.Table, batches: Iterable[pa.Table], columns: Sequence[str], @@ -116,7 +115,9 @@ def _storable(column: pa.ChunkedArray) -> list[Any]: column = pc.cast(column, pa.string()) return column.to_pylist() - def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> str | int | None: + def disable_fk_check( + self, session: so.Session, *, mapper: Any | None = None + ) -> str | int | None: """Turn ``PRAGMA foreign_keys`` off and return its previous value. SQLite ignores the pragma inside an open transaction, so the state is @@ -130,7 +131,7 @@ def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> execute_options = {"bind_arguments": {"mapper": mapper}} if mapper is not None else {} previous_state = session.execute(text("PRAGMA foreign_keys"), **execute_options).scalar() if not isinstance(previous_state, int): - raise RuntimeError("Expected SQLite FK state to be an int") + raise RuntimeError("Expected SQLite FK state to be an int") # noqa: TRY004 - bad database result session.execute(text("PRAGMA foreign_keys = OFF"), **execute_options) if session.execute(text("PRAGMA foreign_keys"), **execute_options).scalar(): raise RuntimeError( @@ -151,13 +152,13 @@ def restore_fk_check( session.execute(text(f"PRAGMA foreign_keys = {safe_state}"), **execute_options) def _pagination_column( - self, table_cls: type["FileTableProtocol"], staging: sa.Table + self, table_cls: type[FileTableProtocol], staging: sa.Table ) -> sa.ColumnElement[int]: """SQLite's implicit rowid is already gapless and indexed.""" return sa.literal_column("rowid") def _ensure_pagination_index( - self, table_cls: type["FileTableProtocol"], staging: sa.Table, session: so.Session + self, table_cls: type[FileTableProtocol], staging: sa.Table, session: so.Session ) -> None: """No-op: rowid needs no index, and the staging table has no _rownum column.""" return @@ -172,7 +173,7 @@ def _commits_between_pagination_batches(self) -> bool: def merge_replace( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, pk_cols: list[str], staging: sa.Table, @@ -210,7 +211,7 @@ def _delete(extra: sa.ColumnElement[bool] | None = None) -> sa.Delete: def merge_upsert( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, pk_cols: list[str], staging: sa.Table, @@ -230,16 +231,19 @@ def _upsert(select_: Any) -> sa.Insert: .on_conflict_do_nothing(index_elements=[target_columns[c] for c in pk_cols]) ) - non_paginated_select = sa.select(*(self._staging_value(table_cls, staging, c) for c in insertable_cols)) + non_paginated_select = sa.select( + *(self._staging_value(table_cls, staging, c) for c in insertable_cols) + ) for batch_range in self._paginate_batches(table_cls, staging, session, merge_batch_size): - - select_ = non_paginated_select.where(batch_range if batch_range is not None else sa.true()) + select_ = non_paginated_select.where( + batch_range if batch_range is not None else sa.true() + ) session.execute(_upsert(select_)) def merge_insert( self, - table_cls: type["FileTableProtocol"], + table_cls: type[FileTableProtocol], session: so.Session, staging: sa.Table, *, @@ -250,11 +254,19 @@ def merge_insert( target = table_cls.__table__ target_columns = {column.name: column for column in target.columns} insertable_cols = self._insertable_column_names(table_cls, staging) - non_paginated_select = sa.select(*(self._staging_value(table_cls, staging, c) for c in insertable_cols)) + non_paginated_select = sa.select( + *(self._staging_value(table_cls, staging, c) for c in insertable_cols) + ) def _insert(select_: Any) -> sa.Insert: - return sa.insert(target).from_select([target_columns[c] for c in insertable_cols], select_) + return sa.insert(target).from_select( + [target_columns[c] for c in insertable_cols], select_ + ) for batch_range in self._paginate_batches(table_cls, staging, session, merge_batch_size): - select_ = non_paginated_select if batch_range is None else non_paginated_select.where(batch_range) + select_ = ( + non_paginated_select + if batch_range is None + else non_paginated_select.where(batch_range) + ) session.execute(_insert(select_)) diff --git a/src/orm_loader/conversion/handlers/base.py b/src/orm_loader/conversion/handlers/base.py index e785135..23a7287 100644 --- a/src/orm_loader/conversion/handlers/base.py +++ b/src/orm_loader/conversion/handlers/base.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from ..policy import NullPolicy + class ColumnHandler(ABC): """Interprets the raw values of the columns of one kind of column type. @@ -128,7 +129,9 @@ def interpret( arr = arr.dictionary_decode() return self._interpret_text(arr, sa_type, nulls) - def _interpret_text(self, raw: pa.Array, sa_type: TypeEngine[Any], nulls: NullPolicy) -> Interpretation: + def _interpret_text( + self, raw: pa.Array, sa_type: TypeEngine[Any], nulls: NullPolicy + ) -> Interpretation: trimmed = pc.utf8_trim_whitespace(raw) missing = self.missing(trimmed, nulls) cleaned = pc.if_else(missing, NULL_STRING, self.clean(raw, trimmed)) diff --git a/src/orm_loader/conversion/handlers/booleans.py b/src/orm_loader/conversion/handlers/booleans.py index 4807150..125d362 100644 --- a/src/orm_loader/conversion/handlers/booleans.py +++ b/src/orm_loader/conversion/handlers/booleans.py @@ -33,8 +33,12 @@ def arrow_type(self, sa_type: TypeEngine[Any]) -> pa.DataType: def parse(self, cleaned: pa.Array, sa_type: TypeEngine[Any]) -> Interpretation: lowered = pc.utf8_lower(cleaned) - is_true = pc.fill_null(pc.is_in(lowered, value_set=pa.array(self.true_words, pa.string())), False) - is_false = pc.fill_null(pc.is_in(lowered, value_set=pa.array(self.false_words, pa.string())), False) + is_true = pc.fill_null( + pc.is_in(lowered, value_set=pa.array(self.true_words, pa.string())), False + ) + is_false = pc.fill_null( + pc.is_in(lowered, value_set=pa.array(self.false_words, pa.string())), False + ) known = pc.or_(is_true, is_false) values = pc.if_else(known, is_true, pa.scalar(None, pa.bool_())) return Interpretation(values, pc.and_(pc.is_valid(cleaned), pc.invert(known))) diff --git a/src/orm_loader/conversion/handlers/dates.py b/src/orm_loader/conversion/handlers/dates.py index 8188152..d48e5ae 100644 --- a/src/orm_loader/conversion/handlers/dates.py +++ b/src/orm_loader/conversion/handlers/dates.py @@ -36,14 +36,27 @@ class DateFormats: DEFAULT: ClassVar[tuple[str, ...]] = ("yyyymmdd", "dd-mon-yyyy", "yyyy-mm-dd") MONTHS: ClassVar[tuple[str, ...]] = ( - "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec", + "jan", + "feb", + "mar", + "apr", + "may", + "jun", + "jul", + "aug", + "sep", + "oct", + "nov", + "dec", ) @staticmethod def yyyymmdd(values: pa.Array) -> tuple[pa.Array, pa.Array]: ok = pc.and_(pc.equal(pc.utf8_length(values), 8), pc.utf8_is_digit(values)) cut = pc.utf8_slice_codeunits - return ok, pc.binary_join_element_wise(cut(values, 0, 4), cut(values, 4, 6), cut(values, 6, 8), "-") + return ok, pc.binary_join_element_wise( + cut(values, 0, 4), cut(values, 4, 6), cut(values, 6, 8), "-" + ) @staticmethod def yyyy_mm_dd(values: pa.Array) -> tuple[pa.Array, pa.Array]: @@ -53,7 +66,9 @@ def yyyy_mm_dd(values: pa.Array) -> tuple[pa.Array, pa.Array]: def dd_mon_yyyy(values: pa.Array) -> tuple[pa.Array, pa.Array]: ok = pc.match_substring_regex(values, r"^\d{2}-[A-Za-z]{3}-\d{4}$") cut = pc.utf8_slice_codeunits - month = pc.index_in(pc.utf8_lower(cut(values, 3, 6)), value_set=pa.array(DateFormats.MONTHS)) + month = pc.index_in( + pc.utf8_lower(cut(values, 3, 6)), value_set=pa.array(DateFormats.MONTHS) + ) month = pc.utf8_lpad(pc.cast(pc.add(month, 1), pa.string()), 2, "0") return ok, pc.binary_join_element_wise(cut(values, 7, 11), month, cut(values, 0, 2), "-") @@ -61,13 +76,17 @@ def dd_mon_yyyy(values: pa.Array) -> tuple[pa.Array, pa.Array]: def dd_mm_yyyy(values: pa.Array) -> tuple[pa.Array, pa.Array]: ok = pc.match_substring_regex(values, r"^\d{2}/\d{2}/\d{4}$") cut = pc.utf8_slice_codeunits - return ok, pc.binary_join_element_wise(cut(values, 6, 10), cut(values, 3, 5), cut(values, 0, 2), "-") + return ok, pc.binary_join_element_wise( + cut(values, 6, 10), cut(values, 3, 5), cut(values, 0, 2), "-" + ) @staticmethod def mm_dd_yyyy(values: pa.Array) -> tuple[pa.Array, pa.Array]: ok = pc.match_substring_regex(values, r"^\d{2}/\d{2}/\d{4}$") cut = pc.utf8_slice_codeunits - return ok, pc.binary_join_element_wise(cut(values, 6, 10), cut(values, 0, 2), cut(values, 3, 5), "-") + return ok, pc.binary_join_element_wise( + cut(values, 6, 10), cut(values, 0, 2), cut(values, 3, 5), "-" + ) SHAPES: ClassVar[Mapping[str, DateShape]] = MappingProxyType( { @@ -134,7 +153,9 @@ class DateHandler(ConvertingHandler): def __post_init__(self) -> None: unknown = [name for name in self.formats if name not in DateFormats.SHAPES] if unknown: - raise ValueError(f"Unknown date formats {unknown}; choose from {sorted(DateFormats.SHAPES)}") + raise ValueError( + f"Unknown date formats {unknown}; choose from {sorted(DateFormats.SHAPES)}" + ) def arrow_type(self, sa_type: TypeEngine[Any]) -> pa.DataType: return pa.date32() @@ -152,12 +173,12 @@ class DateTimeHandler(DateHandler): - A timestamp has date and time separated by ``T`` or a space, with optional seconds and up to six fractional digits. - - A date-only value becomes midnight. - - The column type decides about time zones: + - A date-only value becomes midnight. + - The column type decides about time zones: - ``DateTime(timezone=True)`` column requires an offset (``Z`` or - ``+hh:mm``) and stores UTC, and - - a column without a time zone rejects an offset. - - A date-only value is not read for a time zone column. + ``+hh:mm``) and stores UTC, and + - a column without a time zone rejects an offset. + - A date-only value is not read for a time zone column. - Typed input follows the same rule: a time zone aware timestamp does not fill a naive column and a naive one does not fill an aware column. """ @@ -172,7 +193,11 @@ def arrow_type(self, sa_type: TypeEngine[Any]) -> pa.DataType: return pa.timestamp("us", "UTC" if getattr(sa_type, "timezone", False) else None) def expected(self, sa_type: TypeEngine[Any]) -> str: - return "DateTime with an offset (Z or +hh:mm)" if getattr(sa_type, "timezone", False) else "DateTime without an offset" + return ( + "DateTime with an offset (Z or +hh:mm)" + if getattr(sa_type, "timezone", False) + else "DateTime without an offset" + ) def _interpret_typed(self, arr: pa.Array, sa_type: TypeEngine[Any]) -> Interpretation: target = self.arrow_type(sa_type) @@ -184,10 +209,14 @@ def _interpret_typed(self, arr: pa.Array, sa_type: TypeEngine[Any]) -> Interpret def parse(self, cleaned: pa.Array, sa_type: TypeEngine[Any]) -> Interpretation: target = self.arrow_type(sa_type) aware = target.tz is not None - is_iso = pc.fill_null(pc.match_substring_regex(cleaned, self.AWARE if aware else self.NAIVE), False) + is_iso = pc.fill_null( + pc.match_substring_regex(cleaned, self.AWARE if aware else self.NAIVE), False + ) stamps, stamp_failed = safe_cast(pc.if_else(is_iso, cleaned, NULL_STRING), target) if aware: - return Interpretation(stamps, pc.or_(stamp_failed, pc.and_(pc.is_valid(cleaned), pc.invert(is_iso)))) + return Interpretation( + stamps, pc.or_(stamp_failed, pc.and_(pc.is_valid(cleaned), pc.invert(is_iso))) + ) rest = pc.dictionary_encode(pc.if_else(is_iso, NULL_STRING, cleaned)) dates, invalid = parse_dates(rest.dictionary, self.formats) dates = pc.take(dates, rest.indices) diff --git a/src/orm_loader/conversion/handlers/numbers.py b/src/orm_loader/conversion/handlers/numbers.py index 39f5006..5f0b4c2 100644 --- a/src/orm_loader/conversion/handlers/numbers.py +++ b/src/orm_loader/conversion/handlers/numbers.py @@ -36,7 +36,9 @@ def pattern(self, sa_type: TypeEngine[Any]) -> str: def parse(self, cleaned: pa.Array, sa_type: TypeEngine[Any]) -> Interpretation: fits = pc.fill_null(pc.match_substring_regex(cleaned, self.pattern(sa_type)), False) - candidate = pc.replace_substring_regex(pc.if_else(fits, cleaned, NULL_STRING), self.STRIPPED, "") + candidate = pc.replace_substring_regex( + pc.if_else(fits, cleaned, NULL_STRING), self.STRIPPED, "" + ) values, cast_failed = safe_cast(candidate, self.arrow_type(sa_type)) invalid = pc.and_(pc.is_valid(cleaned), pc.invert(fits)) return Interpretation(values, pc.or_(invalid, cast_failed)) @@ -63,7 +65,13 @@ def check(self, values: pa.Array, sa_type: TypeEngine[Any]) -> pa.Array | None: @staticmethod def bounds(sa_type: TypeEngine[Any]) -> tuple[int, int]: """Return the smallest and largest value the column type holds.""" - bits = 64 if isinstance(sa_type, BigInteger) else 16 if isinstance(sa_type, SmallInteger) else 32 + bits = ( + 64 + if isinstance(sa_type, BigInteger) + else 16 + if isinstance(sa_type, SmallInteger) + else 32 + ) return -(1 << (bits - 1)), (1 << (bits - 1)) - 1 @@ -102,7 +110,9 @@ class NumericHandler(NumberHandler): """ MAX_PRECISION: ClassVar[int] = 38 - SHAPE: ClassVar[str] = r"^(?P[+-]?)(?P\d*)(?:\.(?P\d*))?(?:[eE](?P[+-]?\d{1,3}))?$" + SHAPE: ClassVar[str] = ( + r"^(?P[+-]?)(?P\d*)(?:\.(?P\d*))?(?:[eE](?P[+-]?\d{1,3}))?$" + ) @staticmethod def precision_scale(sa_type: TypeEngine[Any]) -> tuple[int | None, int | None]: @@ -111,7 +121,11 @@ def precision_scale(sa_type: TypeEngine[Any]) -> tuple[int | None, int | None]: def validate(self, sa_type: TypeEngine[Any]) -> None: precision, scale = self.precision_scale(sa_type) - if precision is None or not 1 <= precision <= self.MAX_PRECISION or not 0 <= (scale or 0) <= precision: + if ( + precision is None + or not 1 <= precision <= self.MAX_PRECISION + or not 0 <= (scale or 0) <= precision + ): raise ValueError( f"Numeric needs a precision from 1 to {self.MAX_PRECISION} and a scale from 0 to the precision, " f"declared ({precision}, {scale}). A PassthroughHandler for the column leaves the value to the database" @@ -151,18 +165,24 @@ def normalised(self, cleaned: pa.Array, precision: int, scale: int) -> pa.Array: implied = pc.subtract(fraction, exponent) floor = pa.scalar(0, pa.int32()) needed = pc.max_element_wise(pc.subtract(implied, trailing), floor) - growth = pc.max_element_wise(pc.subtract(pc.add(pa.scalar(scale, pa.int32()), exponent), fraction), floor) + growth = pc.max_element_wise( + pc.subtract(pc.add(pa.scalar(scale, pa.int32()), exponent), fraction), floor + ) holds = pc.and_( - pc.less_equal(needed, scale), pc.less_equal(pc.subtract(significant, implied), precision - scale) + pc.less_equal(needed, scale), + pc.less_equal(pc.subtract(significant, implied), precision - scale), ) parsable = pc.and_( - pc.less_equal(pc.add(count, growth), self.MAX_PRECISION), pc.less_equal(implied, self.MAX_PRECISION) + pc.less_equal(pc.add(count, growth), self.MAX_PRECISION), + pc.less_equal(implied, self.MAX_PRECISION), ) readable = pc.and_( pc.and_(shaped, pc.greater_equal(count, 1)), pc.or_(pc.equal(significant, 0), pc.and_(holds, parsable)), ) - text = pc.if_else(pc.equal(significant, 0), "0", pc.replace_substring_regex(cleaned, r"^\+", "")) + text = pc.if_else( + pc.equal(significant, 0), "0", pc.replace_substring_regex(cleaned, r"^\+", "") + ) return pc.if_else(pc.fill_null(readable, False), text, NULL_STRING) def _split(self, cleaned: pa.Array) -> tuple[pa.Array, pa.Array, pa.Array, pa.Array]: diff --git a/src/orm_loader/conversion/handlers/text.py b/src/orm_loader/conversion/handlers/text.py index 58300ff..0ff3651 100644 --- a/src/orm_loader/conversion/handlers/text.py +++ b/src/orm_loader/conversion/handlers/text.py @@ -7,6 +7,7 @@ import pyarrow as pa import pyarrow.compute as pc +import sqlalchemy as sa from sqlalchemy.types import TypeEngine from ..values import Interpretation, no_failures @@ -52,5 +53,6 @@ class EnumHandler(TextHandler): """Keeps text verbatim and rejects values that are not a label the column stores.""" def check(self, values: pa.Array, sa_type: TypeEngine[Any]) -> pa.Array | None: + assert isinstance(sa_type, sa.Enum) known = pc.is_in(values, value_set=pa.array(sa_type.enums, pa.string())) return pc.and_(pc.is_valid(values), pc.invert(known)) diff --git a/src/orm_loader/conversion/policy.py b/src/orm_loader/conversion/policy.py index 3f5560b..8f75469 100644 --- a/src/orm_loader/conversion/policy.py +++ b/src/orm_loader/conversion/policy.py @@ -171,7 +171,10 @@ def __post_init__(self) -> None: object.__setattr__(self, "columns", MappingProxyType(dict(self.columns))) def for_column( - self, sa_type: TypeEngine[Any], table_name: str | None = None, column_name: str | None = None + self, + sa_type: TypeEngine[Any], + table_name: str | None = None, + column_name: str | None = None, ) -> ColumnHandler: """Return the handler that interprets a column. diff --git a/src/orm_loader/conversion/values.py b/src/orm_loader/conversion/values.py index b5e198b..41e391e 100644 --- a/src/orm_loader/conversion/values.py +++ b/src/orm_loader/conversion/values.py @@ -30,7 +30,7 @@ class Interpretation(NamedTuple): def underlying_type(sa_type: TypeEngine) -> TypeEngine: """Return the type a ``TypeDecorator`` stores its values as, or the type itself.""" while isinstance(sa_type, TypeDecorator): - sa_type = sa_type.impl + sa_type = sa_type.impl_instance return sa_type diff --git a/src/orm_loader/helpers/__init__.py b/src/orm_loader/helpers/__init__.py index 255d98e..3a405f9 100644 --- a/src/orm_loader/helpers/__init__.py +++ b/src/orm_loader/helpers/__init__.py @@ -1,20 +1,27 @@ -from .errors import ConversionError, IngestError, PolicyError, RejectLimitError, RequiredValueError, ValidationError from .bootstrap import bootstrap, create_db, create_tables +from .discovery import get_model_by_tablename from .engines import dispose_engines +from .errors import ( + ConversionError, + IngestError, + PolicyError, + RejectLimitError, + RequiredValueError, + ValidationError, +) from .metadata import Base -from .discovery import get_model_by_tablename __all__ = [ + "Base", "ConversionError", + "IngestError", "PolicyError", "RejectLimitError", "RequiredValueError", - "IngestError", "ValidationError", "bootstrap", "create_db", "create_tables", "dispose_engines", - "Base", "get_model_by_tablename", ] diff --git a/src/orm_loader/helpers/bootstrap.py b/src/orm_loader/helpers/bootstrap.py index b1b0566..feb27e2 100644 --- a/src/orm_loader/helpers/bootstrap.py +++ b/src/orm_loader/helpers/bootstrap.py @@ -1,6 +1,6 @@ import logging from collections.abc import Iterable -from typing import cast +from typing import overload import sqlalchemy as sa from oa_configurator import ( @@ -23,7 +23,13 @@ logger = logging.getLogger(__name__) -def _resolve_binds(resolved: ResolvedDatabase, bindable: Bindable | None) -> tuple[Bindable, Bindable]: +@overload +def _resolve_binds(resolved: ResolvedDatabase, bindable: None) -> tuple[sa.Engine, sa.Engine]: ... +@overload +def _resolve_binds(resolved: ResolvedDatabase, bindable: Bindable) -> tuple[Bindable, Bindable]: ... +def _resolve_binds( + resolved: ResolvedDatabase, bindable: Bindable | None +) -> tuple[Bindable, Bindable]: """Return (primary_bind, vocab_bind) to run DDL against. Built from resolved.create_engines()/create_engine() when bindable is @@ -81,27 +87,28 @@ def create_tables( If a table has no schema tag. """ tables = list(tables) - tagged = [(table, table.schema) for table in tables] - if not tagged: + if not tables: return - untagged = sorted(table.name for table, tag in tagged if tag is None) + untagged = sorted(table.name for table in tables if table.schema is None) if untagged: raise ValueError(f"create_tables() needs schema-tagged tables, got untagged {untagged}.") - schema_tags = {tag for _, tag in tagged if tag is not None} + tag_of = {table: table.schema for table in tables if table.schema is not None} + schema_tags = set(tag_of.values()) if not resolved.tags_share_a_transaction(*schema_tags): raise ValueError( "create_tables() requires all tables to share one database transaction; " f"got schema tags {sorted(schema_tags)}." ) - # Includes tags claimed only via schema_claims=[...] so + # Includes tags claimed only via schema_claims=[...] so # foreign_key_can_span() can resolve them too. physical_schemas = { - tag: physical_schema_of(connection, schema_tag=tag) for tag in claimed_schema_tags(connection) + tag: physical_schema_of(connection, schema_tag=tag) + for tag in claimed_schema_tags(connection) } crossing = any( not resolved.foreign_key_can_span( - cast(str, table.schema), - referred_schema_tag(fk, cast(str, table.schema)), + tag_of[table], + referred_schema_tag(fk, tag_of[table]), physical_schemas=physical_schemas, ) for table in tables @@ -124,9 +131,7 @@ def create_tables( candidate for candidate in metadata.tables.values() if candidate.schema is not None - and resolved.tags_share_a_transaction( - candidate.schema, cast(str, table.schema) - ) + and resolved.tags_share_a_transaction(candidate.schema, tag_of[table]) ) for metadata, related in related_by_metadata.items(): @@ -173,8 +178,12 @@ def create_db(resolved: ResolvedDatabase, *, bindable: Bindable | None = None) - "the module(s) defining your ORM models likely weren't imported yet. " "SQLAlchemy only registers a model on Base when its module runs." ) - owned = bindable is None - primary_bind, vocab_bind = _resolve_binds(resolved, bindable) + owned_engines: tuple[sa.Engine, sa.Engine] | None = None + if bindable is None: + owned_engines = _resolve_binds(resolved, None) + primary_bind, vocab_bind = owned_engines + else: + primary_bind, vocab_bind = _resolve_binds(resolved, bindable) tables_by_bind: dict[int, tuple[Bindable, dict[str, list[sa.Table]]]] = {} untagged: list[str] = [] @@ -207,10 +216,8 @@ def create_db(resolved: ResolvedDatabase, *, bindable: Bindable | None = None) - ) create_tables(connection, tables, resolved=resolved) finally: - if owned: - # _resolve_binds() only returns Engine instances on this path - # (bindable is None): create_engines()/create_engine(). - dispose_engines(cast(sa.engine.Engine, primary_bind), cast(sa.engine.Engine, vocab_bind)) + if owned_engines is not None: + dispose_engines(*owned_engines) def bootstrap( diff --git a/src/orm_loader/helpers/discovery.py b/src/orm_loader/helpers/discovery.py index 6ee411f..377b92d 100644 --- a/src/orm_loader/helpers/discovery.py +++ b/src/orm_loader/helpers/discovery.py @@ -1,12 +1,19 @@ -from typing import TypeVar +from typing import overload + from .metadata import Base -ModelT = TypeVar("ModelT", bound=Base) -def get_model_by_tablename( - tablename: str, - base: type[ModelT] = Base, # ty: ignore[invalid-parameter-default] -) -> type[ModelT] | None: +@overload +def get_model_by_tablename(tablename: str) -> type[Base] | None: ... + + +@overload +def get_model_by_tablename[ModelT: Base]( + tablename: str, base: type[ModelT] +) -> type[ModelT] | None: ... + + +def get_model_by_tablename(tablename: str, base: type[Base] = Base) -> type[Base] | None: tablename = tablename.lower().strip() for mapper in base.registry.mappers: cls = mapper.class_ diff --git a/src/orm_loader/helpers/errors.py b/src/orm_loader/helpers/errors.py index cb5cb3e..78141ab 100644 --- a/src/orm_loader/helpers/errors.py +++ b/src/orm_loader/helpers/errors.py @@ -73,7 +73,11 @@ def __init__( self.count = count self.samples = samples target = ".".join(part for part in (table_name, column_name) if part) or "column" - hint = " Declare the source format with DateHandler(formats=...)." if "Date" in expected else "" + hint = ( + " Declare the source format with DateHandler(formats=...)." + if "Date" in expected + else "" + ) super().__init__( f"{target}: {count} value(s) cannot be read as {expected}. Examples: {samples}.{hint}" ) diff --git a/src/orm_loader/helpers/metadata.py b/src/orm_loader/helpers/metadata.py index 3ad0a84..cebf426 100644 --- a/src/orm_loader/helpers/metadata.py +++ b/src/orm_loader/helpers/metadata.py @@ -11,5 +11,6 @@ metadata = MetaData(naming_convention=NAMING_CONVENTIONS) + class Base(DeclarativeBase): - metadata = metadata \ No newline at end of file + metadata = metadata diff --git a/src/orm_loader/loaders/loader.py b/src/orm_loader/loaders/loader.py index 7bfaaf5..ab38e2d 100644 --- a/src/orm_loader/loaders/loader.py +++ b/src/orm_loader/loaders/loader.py @@ -186,7 +186,12 @@ def stage_file(ctx: LoaderContext, reader: FileReader, staging: sa.Table) -> int return 0 ignored = [name for name in reader.file_columns if name not in reader.columns] if ignored: - logger.warning("File %s has columns that %s does not have, which are ignored: %s", ctx.path.name, table_name, ignored) + logger.warning( + "File %s has columns that %s does not have, which are ignored: %s", + ctx.path.name, + table_name, + ignored, + ) _require_columns_present(ctx.tableclass, reader.columns, normalise=ctx.normalise) backend = ctx.backend rejects_dir = ctx.rejects_dir or ctx.path.parent / "rejects" diff --git a/src/orm_loader/loaders/processor.py b/src/orm_loader/loaders/processor.py index 62fa9d2..676a6ab 100644 --- a/src/orm_loader/loaders/processor.py +++ b/src/orm_loader/loaders/processor.py @@ -60,7 +60,9 @@ def check_writable(self) -> None: while not directory.exists() and directory != directory.parent: directory = directory.parent if not directory.is_dir() or not os.access(directory, os.W_OK): - raise IngestError(f"Cannot write rejected rows to {self.path.parent}: directory is not writable") + raise IngestError( + f"Cannot write rejected rows to {self.path.parent}: directory is not writable" + ) def write(self, rows: pa.Table) -> None: """Append rejected rows, creating the directory and file on first use.""" @@ -147,7 +149,10 @@ def __init__( self.out_columns = [*columns, *(name for name in defaults if name not in columns)] self.types = {name: underlying_type(model_columns[name].type) for name in self.out_columns} self.handlers = ( - {name: policy.for_column(self.types[name], self.table_name, name) for name in self.out_columns} + { + name: policy.for_column(self.types[name], self.table_name, name) + for name in self.out_columns + } if normalise else {} ) @@ -166,7 +171,8 @@ def __init__( self.required = [ name for name in self.out_columns - if (not model_columns[name].nullable or name in primary_keys) and name not in expressions + if (not model_columns[name].nullable or name in primary_keys) + and name not in expressions ] self._literals: dict[str, pa.Scalar] = {} self.rejects = RejectsWriter(rejects_path) @@ -217,7 +223,9 @@ def process(self, raw: pa.RecordBatch) -> pa.Table: if not self.normalise: return pa.Table.from_batches([raw]) results = { - name: self.handlers[name].interpret(raw.column(name), self.types[name], self.policy.nulls) + name: self.handlers[name].interpret( + raw.column(name), self.types[name], self.policy.nulls + ) for name in self.columns } self._fill_defaults(results, raw.num_rows) @@ -232,7 +240,9 @@ def _fill_defaults(self, results: dict[str, Interpretation], count: int) -> None """Give a missing value, and a column the file lacks, the column's default.""" for name, default in self.defaults.items(): if name not in results: - results[name] = Interpretation(pa.repeat(self._fill_value(name, default), count), no_failures(count)) + results[name] = Interpretation( + pa.repeat(self._fill_value(name, default), count), no_failures(count) + ) continue values, invalid = results[name] missing = pc.and_(pc.is_null(values), pc.invert(invalid)) @@ -248,7 +258,9 @@ def _fill_value(self, name: str, default: Any) -> pa.Scalar: def _read_default(self, name: str, value: Any) -> pa.Scalar: """Read a default value the way the column's handler reads a value from the file.""" - values, invalid = self.handlers[name].interpret(pa.array([value]), self.types[name], self.policy.nulls) + values, invalid = self.handlers[name].interpret( + pa.array([value]), self.types[name], self.policy.nulls + ) if invalid[0].as_py(): raise ConversionError( table_name=self.table_name, @@ -264,14 +276,18 @@ def _failures(self, results: dict[str, Any]) -> dict[str, list[tuple[pa.Array, s for name in self.columns: invalid = results[name].invalid if pc.any(invalid).as_py(): - found[name] = [(invalid, f"cannot be read as {self.handlers[name].expected(self.types[name])}")] + found[name] = [ + (invalid, f"cannot be read as {self.handlers[name].expected(self.types[name])}") + ] for name in self.required: missing = pc.and_(pc.is_null(results[name].values), pc.invert(results[name].invalid)) if pc.any(missing).as_py(): found.setdefault(name, []).append((missing, "required value is missing")) return found - def _conversion_error(self, name: str, raw: pa.Array | pa.ChunkedArray, invalid: pa.Array) -> ConversionError | None: + def _conversion_error( + self, name: str, raw: pa.Array | pa.ChunkedArray, invalid: pa.Array + ) -> ConversionError | None: """Describe the unreadable values of a column, or return None when there are none. Parameters @@ -312,7 +328,11 @@ def _raise(self, raw: pa.RecordBatch, results: dict[str, Any], first: int) -> No missing = pc.is_null(results[name].values) count = pc.sum(pc.cast(missing, pa.int64())).as_py() or 0 if count: - rows = self._row_numbers(first, raw.num_rows).take(pc.indices_nonzero(missing).slice(0, 3)).to_pylist() + rows = ( + self._row_numbers(first, raw.num_rows) + .take(pc.indices_nonzero(missing).slice(0, 3)) + .to_pylist() + ) raise RequiredValueError( table_name=self.table_name, column_name=name, count=count, rows=rows ) @@ -334,9 +354,12 @@ def _reject( reason = pc.if_else(first_hit, pa.scalar(why), reason) rejected = pc.or_(rejected, mask) row_numbers = self._row_numbers(first, raw.num_rows) - plain = [c.dictionary_decode() if pa.types.is_dictionary(c.type) else c for c in raw.columns] + plain = [ + c.dictionary_decode() if pa.types.is_dictionary(c.type) else c for c in raw.columns + ] listing = pa.table( - [row_numbers, column, reason, *plain], names=["row", "column", "reason", *raw.schema.names] + [row_numbers, column, reason, *plain], + names=["row", "column", "reason", *raw.schema.names], ) dropped = listing.filter(rejected) self.rejects.write(dropped) @@ -367,7 +390,9 @@ def _reject_skipped(self) -> None: text, empty = pa.string(), pa.nulls(count, pa.string()) rows = pa.table( { - "row": pa.array([None if n is None else n - 1 for n, _ in self.skipped], pa.int64()), + "row": pa.array( + [None if n is None else n - 1 for n, _ in self.skipped], pa.int64() + ), "column": empty, "reason": pa.array([why for _, why in self.skipped], text), } diff --git a/src/orm_loader/loaders/readers/base.py b/src/orm_loader/loaders/readers/base.py index 95e991a..98a5d5d 100644 --- a/src/orm_loader/loaders/readers/base.py +++ b/src/orm_loader/loaders/readers/base.py @@ -62,7 +62,9 @@ def normalise_headers(self, names: Iterable[str]) -> list[str]: IngestError Two columns have the same normalised name. """ - normalised = [name.strip().strip('"').strip().lower().removesuffix("_hash") for name in names] + normalised = [ + name.strip().strip('"').strip().lower().removesuffix("_hash") for name in names + ] if len(set(normalised)) != len(normalised): raise IngestError(f"{self.path.name}: duplicate column names in header {normalised}") return normalised diff --git a/src/orm_loader/loaders/readers/delimited.py b/src/orm_loader/loaders/readers/delimited.py index ac3b8bf..40a523c 100644 --- a/src/orm_loader/loaders/readers/delimited.py +++ b/src/orm_loader/loaders/readers/delimited.py @@ -79,7 +79,12 @@ def _header_names(self) -> list[str]: return self.normalise_headers(header.lstrip("\ufeff").rstrip("\r\n").split(self.delimiter)) def _skip_row(self, row: pv.InvalidRow) -> Literal["skip"]: - self.skipped.append((row.number, f"expected {row.expected_columns} columns, got {row.actual_columns}: {row.text.rstrip()}")) + self.skipped.append( + ( + row.number, + f"expected {row.expected_columns} columns, got {row.actual_columns}: {row.text.rstrip()}", + ) + ) return "skip" def batches(self) -> Iterator[pa.RecordBatch]: @@ -118,7 +123,9 @@ def batches(self) -> Iterator[pa.RecordBatch]: if batch.num_rows: yield batch except UnicodeDecodeError as exc: - raise IngestError(f"{self.path.name}: cannot decode the file as {self.encoding}: {exc.reason}.") from exc + raise IngestError( + f"{self.path.name}: cannot decode the file as {self.encoding}: {exc.reason}." + ) from exc except pa.ArrowInvalid as exc: if "Empty CSV file" in str(exc): return diff --git a/src/orm_loader/loaders/readers/detection.py b/src/orm_loader/loaders/readers/detection.py index 95eb0e4..6cc5942 100644 --- a/src/orm_loader/loaders/readers/detection.py +++ b/src/orm_loader/loaders/readers/detection.py @@ -8,17 +8,21 @@ logger = logging.getLogger(__name__) + def infer_encoding(file): - with open(file, 'rb') as infile: + with open(file, "rb") as infile: encoding = chardet.detect(infile.read(10000)) - if encoding['encoding'] == 'ascii': - encoding['encoding'] = 'utf-8' # utf-8 valid superset of ascii, so being more conservative here just because it flakes occasionally + if encoding["encoding"] == "ascii": + encoding["encoding"] = ( + "utf-8" # utf-8 valid superset of ascii, so being more conservative here just because it flakes occasionally + ) return encoding + def infer_delim(file): - with open(file, 'rb') as infile: + with open(file, "rb") as infile: line = infile.readline() - return '\t' if line.count(b'\t') > line.count(b',') else ',' + return "\t" if line.count(b"\t") > line.count(b",") else "," def resolve_quote_mode(quote_mode: str, delimiter: str) -> str: diff --git a/src/orm_loader/mappers/__init__.py b/src/orm_loader/mappers/__init__.py index fe637e3..0bd59c4 100644 --- a/src/orm_loader/mappers/__init__.py +++ b/src/orm_loader/mappers/__init__.py @@ -8,12 +8,17 @@ # will not affect package-level consumers. A later breaking release can rename # the modules and remove the legacy paths. - from .materialised_view_contracts import ( CreateMaterializedViewIndex, DropMaterializedView, MaterializedViewIndex, ) +from .materialised_view_errors import ( + ConcurrentRefreshNotEligibleError, + MaterializationError, + MaterializationFailure, + MaterializationOperation, +) from .materialised_view_mixin import ( CreateMaterializedView, MaterializedViewBackend, @@ -21,12 +26,6 @@ refresh_all_mvs, resolve_mv_refresh_order, ) -from .materialised_view_errors import ( - ConcurrentRefreshNotEligibleError, - MaterializationError, - MaterializationFailure, - MaterializationOperation, -) __all__ = [ "ConcurrentRefreshNotEligibleError", diff --git a/src/orm_loader/mappers/materialised_view_mixin.py b/src/orm_loader/mappers/materialised_view_mixin.py index 19d5938..fc56ab5 100644 --- a/src/orm_loader/mappers/materialised_view_mixin.py +++ b/src/orm_loader/mappers/materialised_view_mixin.py @@ -1,18 +1,20 @@ from abc import ABC, abstractmethod from collections import defaultdict, deque -from typing import Any +from collections.abc import Set as AbstractSet +from typing import Any, ClassVar import sqlalchemy as sa -from sqlalchemy.ext import compiler -from sqlalchemy.schema import DDLElement from oa_configurator import ( Role, open_connection, physical_schema_of, ) +from sqlalchemy.ext import compiler +from sqlalchemy.schema import DDLElement from .materialised_view_contracts import MaterializedViewIndex + class CreateMaterializedView(DDLElement): """ `CreateMaterializedView` @@ -52,13 +54,11 @@ def __init__( self.with_data = with_data self.if_not_exists = if_not_exists + @compiler.compiles(CreateMaterializedView) def _create_view( - element: CreateMaterializedView, - compiler: sa.sql.compiler.SQLCompiler, - **kwargs: Any + element: CreateMaterializedView, compiler: sa.sql.compiler.SQLCompiler, **kwargs: Any ) -> str: - """ `_create_view` @@ -142,7 +142,6 @@ def create_materialized_view_index( class MaterializedViewMixin: - """ `MaterializedViewMixin` @@ -226,13 +225,13 @@ class DailyObservationCountsMV(Base, MaterializedViewMixin): observation_date = sa.Column(sa.Date, nullable=False) concept_id = sa.Column(sa.Integer, nullable=False) n_observations = sa.Column(sa.Integer, nullable=False) - - + + ``` Query like a normal mapped class: ```python - + rows = ( session.query(DailyObservationCount) .filter(DailyObservationCount.observation_date >= date(2025, 1, 1)) @@ -248,16 +247,17 @@ class DailyObservationCountsMV(Base, MaterializedViewMixin): * Treat as immutable cache """ + __mv_name__: str __mv_select__: sa.sql.Select[Any] - __mv_dependencies__: set[str] = set() + __mv_dependencies__: ClassVar[AbstractSet[str]] = frozenset() __mv_schema_tag__: str = Role.PRIMARY.value __mv_indexes__: tuple[MaterializedViewIndex, ...] = () @classmethod def _resolve_schema_tag(cls, schema_tag: str | None) -> str | None: """Effective schema_tag for one lifecycle call. - A declratively mapped class's own mapped table's schema is authoritative, + A declratively mapped class's own mapped table's schema is authoritative, overriding any ``__mv_schema_tag__``. """ if schema_tag is not None: @@ -268,13 +268,17 @@ def _resolve_schema_tag(cls, schema_tag: str | None) -> str | None: return cls.__mv_schema_tag__ @staticmethod - def _materialized_view_backend(bind: "sa.engine.Connection | sa.engine.Engine") -> MaterializedViewBackend: + def _materialized_view_backend( + bind: "sa.engine.Connection | sa.engine.Engine", + ) -> MaterializedViewBackend: """Return the bind's backend, or raise when it has no materialized views.""" from ..backends.resolve import resolve_backend backend = resolve_backend(bind) if not isinstance(backend, MaterializedViewBackend): - raise NotImplementedError(f"Backend '{backend.name}' does not support materialized views") + raise NotImplementedError( + f"Backend '{backend.name}' does not support materialized views" + ) return backend @classmethod @@ -316,7 +320,7 @@ def create_mv( with engine.begin() as conn: RecentObservationMV.create_mv(conn) - + ``` This emits SQL equivalent to: @@ -427,12 +431,17 @@ def drop_mv( backend = cls._materialized_view_backend(bind) tag = cls._resolve_schema_tag(schema_tag) backend.drop_materialized_view( - bind, cls.__mv_name__, schema=physical_schema_of(bind, schema_tag=tag), - if_exists=if_exists, cascade=cascade, + bind, + cls.__mv_name__, + schema=physical_schema_of(bind, schema_tag=tag), + if_exists=if_exists, + cascade=cascade, ) -def resolve_mv_refresh_order(mv_classes: list[type[MaterializedViewMixin]]) -> list[type]: +def resolve_mv_refresh_order( + mv_classes: list[type[MaterializedViewMixin]], +) -> list[type[MaterializedViewMixin]]: """ `resolve_mv_refresh_order` @@ -458,9 +467,7 @@ def resolve_mv_refresh_order(mv_classes: list[type[MaterializedViewMixin]]) -> l graph[dep].add(cls.__mv_name__) indegree[cls.__mv_name__] += 1 - queue = deque( - name for name, deg in indegree.items() if deg == 0 - ) + queue = deque(name for name, deg in indegree.items() if deg == 0) ordered: list[str] = [] @@ -474,18 +481,17 @@ def resolve_mv_refresh_order(mv_classes: list[type[MaterializedViewMixin]]) -> l queue.append(downstream) if len(ordered) != len(indegree): - raise RuntimeError( - "Cycle detected in materialized view dependencies" - ) + raise RuntimeError("Cycle detected in materialized view dependencies") return [name_to_mv[name] for name in ordered] -def refresh_all_mvs(bind: "sa.engine.Connection | sa.engine.Engine", mv_classes: list[type[MaterializedViewMixin]]) -> None: - +def refresh_all_mvs( + bind: "sa.engine.Connection | sa.engine.Engine", mv_classes: list[type[MaterializedViewMixin]] +) -> None: """ `refresh_all_mvs` - + Handle refreshing multiple materialized views in dependency order. Examples @@ -499,7 +505,7 @@ def refresh_all_mvs(bind: "sa.engine.Connection | sa.engine.Engine", mv_classes: refresh_all_mvs(engine, ALL_MVS) ``` """ - ordered: list[type[MaterializedViewMixin]] = resolve_mv_refresh_order(mv_classes) # ty: ignore[invalid-assignment] + ordered = resolve_mv_refresh_order(mv_classes) for mv in ordered: mv.refresh_mv(bind) diff --git a/src/orm_loader/registry/__init__.py b/src/orm_loader/registry/__init__.py index 1cc6701..f7f64ce 100644 --- a/src/orm_loader/registry/__init__.py +++ b/src/orm_loader/registry/__init__.py @@ -1,18 +1,18 @@ -from .registry import ModelRegistry, ModelDescriptor, TableSpec, FieldSpec -from .validation_report import ValidationIssue, SeverityLevel +from .registry import FieldSpec, ModelDescriptor, ModelRegistry, TableSpec from .validation import Validator -from .validation_runner import ValidationRunner from .validation_presets import always_on_validators +from .validation_report import SeverityLevel, ValidationIssue +from .validation_runner import ValidationRunner __all__ = [ - "ModelRegistry", + "FieldSpec", "ModelDescriptor", + "ModelRegistry", + "SeverityLevel", "TableSpec", - "FieldSpec", - "Validator", "ValidationIssue", - "SeverityLevel", "ValidationRunner", + "Validator", "always_on_validators", ] @@ -64,4 +64,4 @@ def validate(self, *, model, spec=None, fields=None) -> list[ValidationIssue print(report.to_json()) sys.exit(report.exit_code()) -""" \ No newline at end of file +""" diff --git a/src/orm_loader/registry/registry.py b/src/orm_loader/registry/registry.py index b05f430..d302987 100644 --- a/src/orm_loader/registry/registry.py +++ b/src/orm_loader/registry/registry.py @@ -1,11 +1,11 @@ - -from dataclasses import dataclass -from typing import Optional, Type -import sqlalchemy as sa import csv import importlib -import pkgutil import logging +import pkgutil +from dataclasses import dataclass + +import sqlalchemy as sa + from ..tables.typing import ORMTableProtocol logger = logging.getLogger(__name__) @@ -27,9 +27,9 @@ No domain rules or business logic are enforced here. """ + @dataclass(frozen=True) class TableSpec: - """ Table-level specification descriptor. @@ -49,11 +49,13 @@ class TableSpec: user_guidance Optional additional guidance for implementers. """ + table_name: str schema: str is_required: bool description: str - user_guidance: Optional[str] = None + user_guidance: str | None = None + @dataclass(frozen=True) class FieldSpec: @@ -82,6 +84,7 @@ class FieldSpec: fk_field Referenced field name, if the field is a foreign key. """ + table_name: str field_name: str data_type: str @@ -91,6 +94,7 @@ class FieldSpec: fk_table: str | None fk_field: str | None + @dataclass(frozen=True) class ModelDescriptor: """ @@ -114,15 +118,15 @@ class ModelDescriptor: foreign_keys Mapping of column name to referenced (table, field). """ - model_class: Type[ORMTableProtocol] + + model_class: type[ORMTableProtocol] table_name: str columns: dict[str, sa.Column] primary_keys: set[str] foreign_keys: dict[str, tuple[str, str]] # col -> (table, field) - @classmethod - def from_model(cls, model: Type[ORMTableProtocol]) -> "ModelDescriptor": + def from_model(cls, model: type[ORMTableProtocol]) -> "ModelDescriptor": """ Construct a ModelDescriptor from an ORM model class. @@ -161,15 +165,15 @@ def from_model(cls, model: Type[ORMTableProtocol]) -> "ModelDescriptor": primary_keys={c.name for c in table.primary_key.columns}, foreign_keys=fks, ) - + @property - def cls(self) -> Type[ORMTableProtocol]: + def cls(self) -> type[ORMTableProtocol]: """ Alias for the underlying ORM model class. Returns ------- - Type[ORMTableProtocol] + type[ORMTableProtocol] The ORM model class. """ return self.model_class @@ -204,8 +208,8 @@ def load_table_specs(csv_resource) -> dict[str, TableSpec]: ) return out -def load_field_specs(csv_resource) -> dict[str, dict[str, FieldSpec]]: +def load_field_specs(csv_resource) -> dict[str, dict[str, FieldSpec]]: """ Load field specifications from a CSV resource. @@ -253,17 +257,17 @@ class ModelRegistry: Load table and field specifications from CSV files (currently only OMOP format supported). TODO: support generalised specification formats via LinkML or similar. - Register ORM model classes and compare against specifications to confirm accurate and + Register ORM model classes and compare against specifications to confirm accurate and complete implementation. - Model-specific constraints can be created to extend context-specific validation such + Model-specific constraints can be created to extend context-specific validation such as OMOP domain constraints, value set adherence, etc. Validation logic itself is implemented elsewhere and consumes the registry as contextual input. """ - def __init__(self, *, model_version: str, model_name: Optional[str] = None): + def __init__(self, *, model_version: str, model_name: str | None = None): """ Initialise a new model registry. @@ -361,10 +365,9 @@ def missing_required_tables(self) -> set[str]: Required table names that are not implemented. """ return { - t for t, spec in self._table_specs.items() - if spec.is_required and t not in self._models + t for t, spec in self._table_specs.items() if spec.is_required and t not in self._models } - + def discover_models(self, package: str) -> None: """ Discover and register ORM models from a Python package. @@ -379,9 +382,7 @@ def discover_models(self, package: str) -> None: """ module = importlib.import_module(package) - for _, modname, _ in pkgutil.walk_packages( - module.__path__, module.__name__ + "." - ): + for _, modname, _ in pkgutil.walk_packages(module.__path__, module.__name__ + "."): mod = importlib.import_module(modname) for obj in vars(mod).values(): diff --git a/src/orm_loader/registry/validation.py b/src/orm_loader/registry/validation.py index ece6633..87bb1bc 100644 --- a/src/orm_loader/registry/validation.py +++ b/src/orm_loader/registry/validation.py @@ -1,7 +1,10 @@ from __future__ import annotations + from typing import Protocol -from .validation_report import ValidationIssue, SeverityLevel -from .registry import ModelDescriptor, TableSpec, FieldSpec + +from .registry import FieldSpec, ModelDescriptor, TableSpec +from .validation_report import SeverityLevel, ValidationIssue + class Validator(Protocol): def validate( @@ -10,12 +13,10 @@ def validate( model: ModelDescriptor, spec: TableSpec | None = None, fields: dict[str, FieldSpec] | None = None, - ) -> list[ValidationIssue]: - ... + ) -> list[ValidationIssue]: ... class ColumnNullabilityValidator: - """ Ensures required columns are NOT NULL in the ORM model. """ @@ -55,9 +56,9 @@ def validate( ) return issues - -class ColumnPresenceValidator: + +class ColumnPresenceValidator: """ Ensures all specified columns are present on the ORM model. """ @@ -75,7 +76,7 @@ def validate( issues: list[ValidationIssue] = [] model_cols = set(model.columns.keys()) - for field_name, field_spec in fields.items(): + for field_name in fields: if field_name not in model_cols: issues.append( ValidationIssue( @@ -89,6 +90,7 @@ def validate( return issues + class PrimaryKeyValidator: """ Validates primary key presence and correctness. @@ -142,11 +144,7 @@ def validate( ) if fields: - spec_pks = { - name - for name, fs in fields.items() - if fs.is_primary_key - } + spec_pks = {name for name, fs in fields.items() if fs.is_primary_key} for missing in sorted(spec_pks - pk_cols): issues.append( @@ -183,9 +181,8 @@ def validate( return issues - + class ForeignKeyShapeValidator: - """ Validates structural correctness of foreign key definitions. """ @@ -226,4 +223,4 @@ def validate( ) ) - return issues \ No newline at end of file + return issues diff --git a/src/orm_loader/registry/validation_presets.py b/src/orm_loader/registry/validation_presets.py index 2abec0f..9719e8d 100644 --- a/src/orm_loader/registry/validation_presets.py +++ b/src/orm_loader/registry/validation_presets.py @@ -1,10 +1,11 @@ from .validation import ( - ColumnNullabilityValidator, - ColumnPresenceValidator, - PrimaryKeyValidator, + ColumnNullabilityValidator, + ColumnPresenceValidator, ForeignKeyShapeValidator, + PrimaryKeyValidator, ) + def always_on_validators(): return [ ColumnPresenceValidator(), diff --git a/src/orm_loader/registry/validation_report.py b/src/orm_loader/registry/validation_report.py index 422fbf6..4e8ad04 100644 --- a/src/orm_loader/registry/validation_report.py +++ b/src/orm_loader/registry/validation_report.py @@ -1,9 +1,7 @@ +import json from collections import defaultdict from dataclasses import dataclass from enum import Enum -from typing import Optional -import json - """ Validation Reporting @@ -30,6 +28,7 @@ class SeverityLevel(Enum): WARN = "WARN" INFO = "INFO" + @dataclass class ValidationIssue: """ @@ -55,13 +54,15 @@ class ValidationIssue: hint Optional human-readable guidance. """ + table: str - level: SeverityLevel + level: SeverityLevel message: str - field: Optional[str] = None - expected: Optional[str] = None - actual: Optional[str] = None - hint: Optional[str] = None + field: str | None = None + expected: str | None = None + actual: str | None = None + hint: str | None = None + @dataclass class ValidationReport: @@ -75,7 +76,8 @@ class ValidationReport: - JSON serialisation - CI/CD pipeline integration """ - def __init__(self, *, model_version: str, model_name: Optional[str] = None): + + def __init__(self, *, model_version: str, model_name: str | None = None): """ Initialise a new validation report. @@ -89,7 +91,7 @@ def __init__(self, *, model_version: str, model_name: Optional[str] = None): self.model_version = model_version self.model_name = model_name self.issues: list[ValidationIssue] = [] - + def add(self, issue: ValidationIssue) -> None: """ Add a validation issue to the report. @@ -100,7 +102,7 @@ def add(self, issue: ValidationIssue) -> None: Validation issue to record. """ self.issues.append(issue) - + def is_valid(self) -> bool: """ Return whether the report contains no issues. @@ -113,7 +115,6 @@ def is_valid(self) -> bool: return not self.issues def summary(self) -> str: - """ Return a one-line human-readable summary. @@ -127,7 +128,7 @@ def summary(self) -> str: by[i.level] += 1 model = self.model_name.upper() if self.model_name else "MODEL" return f"{model} v{self.model_version}: {by[SeverityLevel.ERROR]} error(s), {by[SeverityLevel.WARN]} warning(s), {by[SeverityLevel.INFO]} info" - + def render_text_report(self) -> str: """ Render a human-readable multi-line text report. @@ -155,7 +156,7 @@ def render_text_report(self) -> str: lines.append(f" {icon} {i.message}{field}{hint}") return "\n".join(lines) - + def to_dict(self) -> dict: """ Return a structured dictionary representation of the report. @@ -187,7 +188,7 @@ def to_dict(self) -> dict: for i in self.issues ], } - + def to_json(self) -> str: """ Serialise the report to a JSON string. @@ -198,7 +199,7 @@ def to_json(self) -> str: JSON-formatted validation report. """ return json.dumps(self.to_dict(), indent=2) - + def exit_code(self) -> int: """ Return a process exit code suitable for CI/CD pipelines. @@ -208,4 +209,4 @@ def exit_code(self) -> int: int 1 if any ERROR-level issues are present, otherwise 0. """ - return 1 if any(i.level == SeverityLevel.ERROR for i in self.issues) else 0 \ No newline at end of file + return 1 if any(i.level == SeverityLevel.ERROR for i in self.issues) else 0 diff --git a/src/orm_loader/registry/validation_runner.py b/src/orm_loader/registry/validation_runner.py index a29486d..cf10b16 100644 --- a/src/orm_loader/registry/validation_runner.py +++ b/src/orm_loader/registry/validation_runner.py @@ -1,7 +1,6 @@ -from .validation_report import ValidationReport, SeverityLevel from .registry import ModelRegistry from .validation import Validator - +from .validation_report import SeverityLevel, ValidationReport """ Validation Runner @@ -17,6 +16,7 @@ - supporting optional fail-fast behaviour """ + class ValidationRunner: """ Coordinates execution of validators against a model registry. @@ -24,6 +24,7 @@ class ValidationRunner: The runner itself is deliberately simple and does not embed any validation logic. All rules are provided via Validator instances. """ + def __init__(self, validators: list[Validator], fail_fast: bool = False): """ Initialise a validation runner. @@ -38,7 +39,6 @@ def __init__(self, validators: list[Validator], fail_fast: bool = False): self.validators = validators self.fail_fast = fail_fast - def run(self, registry: ModelRegistry) -> ValidationReport: """ Run validation across all registered models. @@ -53,9 +53,7 @@ def run(self, registry: ModelRegistry) -> ValidationReport: ValidationReport Aggregated validation report. """ - report = ValidationReport( - model_version=registry.model_version - ) + report = ValidationReport(model_version=registry.model_version) for table_name, desc in registry.models().items(): table_spec = registry._table_specs.get(table_name) @@ -71,10 +69,7 @@ def run(self, registry: ModelRegistry) -> ValidationReport: for issue in issues: report.add(issue) - if ( - self.fail_fast - and issue.level == SeverityLevel.ERROR - ): + if self.fail_fast and issue.level == SeverityLevel.ERROR: return report return report diff --git a/src/orm_loader/tables/__init__.py b/src/orm_loader/tables/__init__.py index c9edaea..880939a 100644 --- a/src/orm_loader/tables/__init__.py +++ b/src/orm_loader/tables/__init__.py @@ -2,13 +2,13 @@ from .loadable_table import FileLoadableTableInterface from .orm_table import ORMTableBase from .serialisable_table import SerialisableTableInterface -from .typing import ORMTableProtocol, FileTableProtocol +from .typing import FileTableProtocol, ORMTableProtocol __all__ = [ - "ORMTableBase", "FileLoadableTableInterface", - "SerialisableTableInterface", + "FileTableProtocol", "IdAllocator", + "ORMTableBase", "ORMTableProtocol", - "FileTableProtocol", -] \ No newline at end of file + "SerialisableTableInterface", +] diff --git a/src/orm_loader/tables/loadable_table.py b/src/orm_loader/tables/loadable_table.py index 3f18d25..af42f16 100644 --- a/src/orm_loader/tables/loadable_table.py +++ b/src/orm_loader/tables/loadable_table.py @@ -6,12 +6,13 @@ target table. Readers choose the file formats; backends choose the merge and index handling. """ + import logging -from collections.abc import Mapping +from collections.abc import Generator, Mapping from contextlib import contextmanager from pathlib import Path from time import perf_counter -from typing import Any, Generator, Type +from typing import Any import sqlalchemy as sa import sqlalchemy.orm as so @@ -22,7 +23,7 @@ from ..backends.resolve import resolve_backend from ..conversion import ConversionPolicy from ..helpers import IngestError -from ..loaders import FileReader, LoadReport, LoaderContext, open_reader, stage_file +from ..loaders import FileReader, LoaderContext, LoadReport, open_reader, stage_file from .orm_table import ORMTableBase from .typing import FileTableProtocol @@ -63,7 +64,7 @@ class FileLoadableTableInterface(ORMTableBase): @classmethod @contextmanager def manage_indices( - cls: Type['FileTableProtocol'], + cls: type["FileTableProtocol"], session: so.Session, index_strategy: str = "auto", *, @@ -91,9 +92,11 @@ def manage_indices( connection = session.connection(bind_arguments={"mapper": cls}) inspector = sa.inspect(connection) schema = physical_schema_of(connection, schema_tag=cls.__table__.schema) - existing_in_db = {idx['name'] for idx in inspector.get_indexes(cls.__tablename__, schema=schema)} + existing_in_db = { + idx["name"] for idx in inspector.get_indexes(cls.__tablename__, schema=schema) + } to_drop = [i for i in indices if i.name in existing_in_db] - + if to_drop: logger.info(f"Table `{table_name}`: Dropping {len(to_drop)} active indices.") drop_started = perf_counter() @@ -133,17 +136,18 @@ def manage_indices( except Exception as e: if not merge_failed: raise - logger.error(f"Table `{table_name}`: Index rebuild after the failed merge also failed - {e}") + logger.error( + f"Table `{table_name}`: Index rebuild after the failed merge also failed - {e}" + ) else: if failed_indices and not merge_failed: raise IngestError( f"Table `{table_name}`: the merge was committed but index(es) {failed_indices} could not be rebuilt." ) - @classmethod def _restore_indices( - cls: Type['FileTableProtocol'], + cls: type["FileTableProtocol"], session: so.Session, indices: list[sa.Index], ) -> list[str]: @@ -169,7 +173,9 @@ def _restore_indices( connection = session.connection(bind_arguments={"mapper": cls}) inspector = sa.inspect(connection) schema = physical_schema_of(connection, schema_tag=cls.__table__.schema) - existing_idx_names = {idx['name'] for idx in inspector.get_indexes(table_name, schema=schema)} + existing_idx_names = { + idx["name"] for idx in inspector.get_indexes(table_name, schema=schema) + } for idx in indices: if idx.name not in existing_idx_names: @@ -187,23 +193,24 @@ def _restore_indices( logger.info( f"Table `{table_name}`: Commit after restoring index `{idx.name}` " f"completed in {_format_elapsed(perf_counter() - commit_started)}." - ) - except Exception as e: + ) + except Exception: session.rollback() failed_indices.append(str(idx.name)) - logger.error(f"Table `{table_name}`: Failed to restore {idx.name}: {e}") + logger.exception(f"Table `{table_name}`: Failed to restore {idx.name}") else: - logger.debug(f"Table `{table_name}`: Index {idx.name} already exists on disk. Skipping.") + logger.debug( + f"Table `{table_name}`: Index {idx.name} already exists on disk. Skipping." + ) logger.info( f"Table `{table_name}`: Index verification/rebuild completed in " f"{_format_elapsed(perf_counter() - rebuild_started)}." ) return failed_indices - @classmethod def load_file( - cls: Type[FileTableProtocol], + cls: type[FileTableProtocol], session: so.Session, path: Path, *, @@ -223,7 +230,6 @@ def load_file( rejects_dir: Path | None = None, report: LoadReport | None = None, ) -> int: - """ Load a file into the target table. @@ -314,13 +320,13 @@ def load_file( logger.debug(f"Table `{cls.__tablename__}`: Loading file {path}") if path.stem.lower() != cls.__tablename__: - raise ValueError( - f"File name '{path.name}' does not match table '{cls.__tablename__}'" - ) + raise ValueError(f"File name '{path.name}' does not match table '{cls.__tablename__}'") _require_bind(session, cls) if reader is not None and (chunksize is not None or quote_mode is not None): - raise ValueError("chunksize and quote_mode configure the default reader; set them on the reader passed as `reader`") + raise ValueError( + "chunksize and quote_mode configure the default reader; set them on the reader passed as `reader`" + ) policy = policy or ConversionPolicy() if reader is None: reader = open_reader( @@ -377,18 +383,21 @@ def load_file( validate_deferred_fks=validate_deferred_fks, ) if validate_deferred_fks: - logger.info(f"Table `{cls.__tablename__}`: Validating deferred foreign keys.") + logger.info( + f"Table `{cls.__tablename__}`: Validating deferred foreign keys." + ) backend.validate_deferred_fks(cls, session) finally: backend.drop_staging_table(cls, session) - logger.info(f"Table `{cls.__tablename__}`: Successfully finished ingestion. Total rows: {total}") + logger.info( + f"Table `{cls.__tablename__}`: Successfully finished ingestion. Total rows: {total}" + ) return total - @classmethod def _target_has_rows( - cls: Type[FileTableProtocol], + cls: type[FileTableProtocol], session: so.Session, ) -> bool: """ @@ -400,10 +409,9 @@ def _target_has_rows( ).first() return row is not None - @classmethod def validate_foreign_keys( - cls: Type[FileTableProtocol], + cls: type[FileTableProtocol], session: so.Session, *, staging_schema_tag: str | None = None, @@ -450,7 +458,7 @@ def validate_foreign_keys( @classmethod def merge_from_staging( - cls: Type[FileTableProtocol], + cls: type[FileTableProtocol], session: so.Session, merge_strategy: str = "replace", *, @@ -490,7 +498,12 @@ def merge_from_staging( logger.info(f"Table `{target}`: Merge replace delete phase starting.") delete_started = perf_counter() backend.merge_replace( - cls, session, pk_cols, staging, merge_batch_size=merge_batch_size, validate_deferred_fks=validate_deferred_fks + cls, + session, + pk_cols, + staging, + merge_batch_size=merge_batch_size, + validate_deferred_fks=validate_deferred_fks, ) logger.info( f"Table `{target}`: Merge replace delete phase completed in " @@ -499,7 +512,11 @@ def merge_from_staging( logger.info(f"Table `{target}`: Merge insert phase starting.") insert_started = perf_counter() backend.merge_insert( - cls, session, staging, merge_batch_size=merge_batch_size, validate_deferred_fks=validate_deferred_fks + cls, + session, + staging, + merge_batch_size=merge_batch_size, + validate_deferred_fks=validate_deferred_fks, ) logger.info( f"Table `{target}`: Merge insert phase completed in " @@ -514,7 +531,12 @@ def merge_from_staging( logger.info(f"Table `{target}`: Merge upsert phase starting.") upsert_started = perf_counter() backend.merge_upsert( - cls, session, pk_cols, staging, merge_batch_size=merge_batch_size, validate_deferred_fks=validate_deferred_fks + cls, + session, + pk_cols, + staging, + merge_batch_size=merge_batch_size, + validate_deferred_fks=validate_deferred_fks, ) logger.info( f"Table `{target}`: Merge upsert phase completed in " @@ -524,7 +546,11 @@ def merge_from_staging( logger.info(f"Table `{target}`: Merge insert phase starting.") insert_started = perf_counter() backend.merge_insert( - cls, session, staging, merge_batch_size=merge_batch_size, validate_deferred_fks=validate_deferred_fks + cls, + session, + staging, + merge_batch_size=merge_batch_size, + validate_deferred_fks=validate_deferred_fks, ) logger.info( f"Table `{target}`: Merge insert phase completed in " diff --git a/src/orm_loader/tables/orm_table.py b/src/orm_loader/tables/orm_table.py index 6f72954..a47fd1d 100644 --- a/src/orm_loader/tables/orm_table.py +++ b/src/orm_loader/tables/orm_table.py @@ -211,9 +211,7 @@ def max_id(cls, session: so.Session) -> int: """ pks = cls.pk_columns() if len(pks) != 1: - raise ValueError( - f"{cls.__name__} has composite PK; max_id() not supported" - ) + raise ValueError(f"{cls.__name__} has composite PK; max_id() not supported") pk = pks[0] return session.query(sa.func.max(pk)).scalar() or 0 @@ -240,5 +238,3 @@ def allocator(cls, session: so.Session) -> IdAllocator: An ID allocator initialised to the current maximum ID. """ return IdAllocator(cls.max_id(session)) - - diff --git a/src/orm_loader/tables/serialisable_table.py b/src/orm_loader/tables/serialisable_table.py index 0ec0b05..9bdaff9 100644 --- a/src/orm_loader/tables/serialisable_table.py +++ b/src/orm_loader/tables/serialisable_table.py @@ -1,8 +1,8 @@ -from typing import Any, Unpack -from collections.abc import Iterator -import json -import hashlib import datetime +import hashlib +import json +from collections.abc import Iterator +from typing import Any, Unpack from .orm_table import ORMTableBase from .typing import ToDictKwargs @@ -34,7 +34,7 @@ def json_default(obj: Any) -> str: return obj.isoformat() raise TypeError(f"Object of type {type(obj)} is not JSON serialisable") - + class SerialisableTableInterface(ORMTableBase): """ Mixin for SQLAlchemy ORM tables providing explicit serialisation helpers. @@ -60,7 +60,6 @@ def to_dict( only: set[str] | None = None, exclude: set[str] | None = None, ) -> dict[str, Any]: - """ Convert the ORM instance into a dictionary. @@ -118,7 +117,7 @@ def to_json(self, **kwargs: Unpack[ToDictKwargs]) -> str: default=json_default, sort_keys=True, ) - + def fingerprint(self) -> str: """ Compute a stable fingerprint for the ORM instance. @@ -137,7 +136,7 @@ def fingerprint(self) -> str: """ payload = self.to_json(include_nulls=True) return hashlib.sha256(payload.encode("utf-8")).hexdigest() - + def __iter__(self) -> Iterator[tuple[str, Any]]: """ Iterate over the ORM instance as ``(key, value)`` pairs. diff --git a/src/orm_loader/tables/typing.py b/src/orm_loader/tables/typing.py index 23e3db4..ae9bdbc 100644 --- a/src/orm_loader/tables/typing.py +++ b/src/orm_loader/tables/typing.py @@ -1,9 +1,20 @@ -from typing import Protocol, ClassVar, runtime_checkable, TYPE_CHECKING, Optional, Dict, Any, Unpack, TypedDict -import sqlalchemy.orm as so -import sqlalchemy as sa from collections.abc import Mapping -from pathlib import Path from contextlib import AbstractContextManager +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Optional, + Protocol, + TypedDict, + Unpack, + runtime_checkable, +) + +import sqlalchemy as sa +import sqlalchemy.orm as so + if TYPE_CHECKING: from oa_configurator import ResolvedDatabase @@ -11,11 +22,13 @@ from ..conversion import ConversionPolicy from ..loaders import FileReader, LoadReport + class ToDictKwargs(TypedDict, total=False): include_nulls: bool only: set[str] | None exclude: set[str] | None + @runtime_checkable class ORMTableProtocol(Protocol): """ @@ -58,6 +71,7 @@ def column_defaults(cls) -> dict[str, Any]: ... @classmethod def sql_expression_defaults(cls) -> dict[str, sa.ClauseElement]: ... + @runtime_checkable class FileTableProtocol(ORMTableProtocol, Protocol): """ @@ -91,9 +105,9 @@ def load_file( staging_schema_tag: str | None = None, validate_deferred_fks: bool = True, resolved: Optional["ResolvedDatabase"] = None, - physical_schemas: Optional[Mapping[str, str | None]] = None, + physical_schemas: Mapping[str, str | None] | None = None, policy: Optional["ConversionPolicy"] = None, - rejects_dir: Optional[Path] = None, + rejects_dir: Path | None = None, report: Optional["LoadReport"] = None, ) -> int: ... @@ -116,7 +130,7 @@ def validate_foreign_keys( *, staging_schema_tag: str | None = None, resolved: Optional["ResolvedDatabase"] = None, - physical_schemas: Optional[Mapping[str, str | None]] = None, + physical_schemas: Mapping[str, str | None] | None = None, ) -> None: ... @classmethod @@ -128,9 +142,8 @@ def _restore_indices(cls, session: so.Session, indices: list[sa.Index]) -> list[ @classmethod def manage_indices( cls, session: so.Session, index_strategy: str = "auto", *, backend: "DatabaseBackend" - ) -> AbstractContextManager[None]: - ... - + ) -> AbstractContextManager[None]: ... + @runtime_checkable class SerializedTableProtocol(Protocol): @@ -151,7 +164,7 @@ def to_dict( include_nulls: bool = False, only: set[str] | None = None, exclude: set[str] | None = None, - ) -> Dict[str, Any]: ... + ) -> dict[str, Any]: ... def to_json(self, **kwargs: Unpack[ToDictKwargs]) -> str: ... diff --git a/tests/backends/test_base_backend.py b/tests/backends/test_base_backend.py index ed73130..1e7472e 100644 --- a/tests/backends/test_base_backend.py +++ b/tests/backends/test_base_backend.py @@ -3,14 +3,15 @@ import importlib import importlib.abc import sys +from collections.abc import Iterable, Sequence from importlib.machinery import ModuleSpec from types import ModuleType -from typing import TYPE_CHECKING, Sequence, Type, cast, Any +from typing import TYPE_CHECKING, Any, cast +import pyarrow as pa import pytest import sqlalchemy as sa import sqlalchemy.orm as so -from sqlalchemy.engine import Engine from orm_loader.backends import ( DatabaseBackend, @@ -19,6 +20,7 @@ ) if TYPE_CHECKING: + from orm_loader.loaders.loader import LoaderContext from orm_loader.tables.typing import FileTableProtocol @@ -38,12 +40,15 @@ class FakeBackend(DatabaseBackend): def __init__(self) -> None: self.calls: list[tuple[str, object]] = [] - def insert_batches(self, ctx, staging, batches, columns) -> int: - total = 0 + def insert_batches( + self, + ctx: LoaderContext, + staging: sa.Table, + batches: Iterable[pa.Table], + columns: Sequence[str], + ) -> None: for table in batches: ctx.session.execute(staging.insert(), table.to_pylist()) - total += table.num_rows - return total @property def name(self) -> str: @@ -64,49 +69,50 @@ def staging_name_for_table(tablename: str) -> str: return f"_staging_{tablename}" def create_staging_table( - self, _table_cls: Type[FileTableProtocol], _session: so.Session - ) -> None: - return None + self, table_cls: type[FileTableProtocol], session: so.Session + ) -> sa.Table: + return sa.Table("staging", sa.MetaData(), sa.Column("id", sa.Integer)) - def drop_staging_table( - self, _table_cls: Type[FileTableProtocol], _session: so.Session - ) -> None: + def drop_staging_table(self, table_cls: type[FileTableProtocol], session: so.Session) -> None: return None def merge_replace( self, - _table_cls: Type[FileTableProtocol], - _session: so.Session, - _target_name: str, - _pk_cols: list[str], + table_cls: type[FileTableProtocol], + session: so.Session, + pk_cols: list[str], + staging: sa.Table, *, - _merge_batch_size: int | None = None, + merge_batch_size: int | None = None, + validate_deferred_fks: bool = True, ) -> None: return None def merge_upsert( self, - _table_cls: Type[FileTableProtocol], - _session: so.Session, - _target_name: str, - _pk_cols: list[str], + table_cls: type[FileTableProtocol], + session: so.Session, + pk_cols: list[str], + staging: sa.Table, *, - _merge_batch_size: int | None = None, + merge_batch_size: int | None = None, + validate_deferred_fks: bool = True, ) -> None: return None def merge_insert( self, - _table_cls: Type[FileTableProtocol], - _session: so.Session, - _target_name: str, + table_cls: type[FileTableProtocol], + session: so.Session, + staging: sa.Table, *, - _merge_batch_size: int | None = None, + merge_batch_size: int | None = None, + validate_deferred_fks: bool = True, ) -> None: return None def _pagination_column( - self, _table_cls: Type[FileTableProtocol], _staging: sa.Table + self, table_cls: type[FileTableProtocol], staging: sa.Table ) -> sa.ColumnElement[int]: return sa.literal_column("rowid") @@ -114,7 +120,9 @@ def _pagination_column( def _commits_between_pagination_batches(self) -> bool: return True - def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> str | int | None: + def disable_fk_check( + self, session: so.Session, *, mapper: Any | None = None + ) -> str | int | None: self.calls.append(("disable_fk_check", session)) return "enabled" @@ -134,12 +142,13 @@ class _ComputedTable: ) -_ComputedTableCls = cast("Type[FileTableProtocol]", _ComputedTable) +_ComputedTableCls = cast("type[FileTableProtocol]", _ComputedTable) def test_database_backend_is_abstract(): + abstract_backend: Any = DatabaseBackend with pytest.raises(TypeError): - DatabaseBackend() # type: ignore + abstract_backend() def test_fake_backend_can_implement_contract(): @@ -179,7 +188,9 @@ def test_relaxed_foreign_keys_toggles_fk_and_restores(sqlite_session): def test_relaxed_foreign_keys_leaves_keys_enforced_when_backend_cannot_toggle(sqlite_session): class NoFKBackend(FakeBackend): - def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> str | int | None: + def disable_fk_check( + self, session: so.Session, *, mapper: Any | None = None + ) -> str | int | None: return super(FakeBackend, self).disable_fk_check(session, mapper=mapper) backend = NoFKBackend() @@ -193,9 +204,11 @@ def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> def test_relaxed_foreign_keys_rolls_back_and_restores(sqlite_session): backend = FakeBackend() - with pytest.raises(RuntimeError, match="boom"): - with backend.relaxed_foreign_keys(sqlite_session): - raise RuntimeError("boom") + with ( + pytest.raises(RuntimeError, match="boom"), + backend.relaxed_foreign_keys(sqlite_session), + ): + raise RuntimeError("boom") assert backend.calls == [ ("disable_fk_check", sqlite_session), @@ -204,7 +217,7 @@ def test_relaxed_foreign_keys_rolls_back_and_restores(sqlite_session): def test_backends_package_exports(): - import orm_loader.backends as backends + from orm_loader import backends assert backends.DatabaseBackend is DatabaseBackend assert backends.Dialect is Dialect @@ -230,8 +243,10 @@ class _Unknown: class dialect: name = "unknown" + unknown_bindable: Any = _Unknown() with pytest.raises(NotImplementedError, match="Unsupported SQLAlchemy dialect"): - resolve_backend(cast(Engine, _Unknown())) + resolve_backend(unknown_bindable) + def test_backends_import_does_not_require_psycopg(): blocker = _BlockPsycopg() diff --git a/tests/backends/test_duckdb_backend.py b/tests/backends/test_duckdb_backend.py index 670a315..f8eb47e 100644 --- a/tests/backends/test_duckdb_backend.py +++ b/tests/backends/test_duckdb_backend.py @@ -2,17 +2,23 @@ import tempfile from pathlib import Path -from typing import TYPE_CHECKING, Type, cast +from typing import TYPE_CHECKING, cast import pytest import sqlalchemy as sa import sqlalchemy.orm as so -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Resolver, StackConfig +from oa_configurator import ( + CDMDatabaseConfig, + ConnectionConfig, + ResolvedCDMDatabase, + Resolver, + StackConfig, +) -from orm_loader.mappers import MaterializedViewBackend from orm_loader.backends import Dialect, DuckDBBackend, staging_schema_claim from orm_loader.backends.base import DeferredForeignKeyViolationError from orm_loader.helpers.bootstrap import create_tables +from orm_loader.mappers import MaterializedViewBackend from tests.models import ( Base, ComputedColumnTable, @@ -29,13 +35,13 @@ if TYPE_CHECKING: from orm_loader.tables.typing import FileTableProtocol -_ComputedTableCls = cast("Type[FileTableProtocol]", ComputedColumnTable) -_CycleATableCls = cast("Type[FileTableProtocol]", CycleATable) -_CycleBTableCls = cast("Type[FileTableProtocol]", CycleBTable) -_ReferencedParentTableCls = cast("Type[FileTableProtocol]", ReferencedParentTable) -_SelfReferentialTableCls = cast("Type[FileTableProtocol]", SelfReferentialTable) -_VocabReferencingTableCls = cast("Type[FileTableProtocol]", VocabReferencingTable) -_VocabSchemaTableCls = cast("Type[FileTableProtocol]", VocabSchemaTable) +_ComputedTableCls = cast("type[FileTableProtocol]", ComputedColumnTable) +_CycleATableCls = cast("type[FileTableProtocol]", CycleATable) +_CycleBTableCls = cast("type[FileTableProtocol]", CycleBTable) +_ReferencedParentTableCls = cast("type[FileTableProtocol]", ReferencedParentTable) +_SelfReferentialTableCls = cast("type[FileTableProtocol]", SelfReferentialTable) +_VocabReferencingTableCls = cast("type[FileTableProtocol]", VocabReferencingTable) +_VocabSchemaTableCls = cast("type[FileTableProtocol]", VocabSchemaTable) _TARGET_TABLE = ComputedColumnTable.__tablename__ _STAGING_TABLE = f"_staging_{_TARGET_TABLE}" @@ -70,21 +76,29 @@ def duckdb_split_schema_session(): stack = StackConfig.for_session( connections={ "db": ConnectionConfig( - dialect="duckdb", database_name=str(Path(tmp_dir) / "split.duckdb"), test_only=True + dialect="duckdb", + database_name=str(Path(tmp_dir) / "split.duckdb"), + test_only=True, ) }, databases={ "cdm": CDMDatabaseConfig( - connection="db", cdm_schema="clinical", vocab_schema="vocabulary", results_schema="results" + connection="db", + cdm_schema="clinical", + vocab_schema="vocabulary", + results_schema="results", ) }, ) resolved = Resolver(stack).resolve_database("cdm") + assert isinstance(resolved, ResolvedCDMDatabase) primary_engine, _ = resolved.create_engines(schema_claims=[staging_schema_claim()]) try: with primary_engine.connect() as conn: create_tables( - conn, [VocabReferencingTable.__table__, VocabSchemaTable.__table__], resolved=resolved + conn, + [VocabReferencingTable.__table__, VocabSchemaTable.__table__], + resolved=resolved, ) conn.commit() with so.Session(bind=conn) as session: @@ -103,16 +117,23 @@ def duckdb_split_database_session(): stack = StackConfig.for_session( connections={ "primary": ConnectionConfig( - dialect="duckdb", database_name=str(Path(tmp_dir) / "primary.duckdb"), test_only=True + dialect="duckdb", + database_name=str(Path(tmp_dir) / "primary.duckdb"), + test_only=True, ), "vocab": ConnectionConfig( - dialect="duckdb", database_name=str(Path(tmp_dir) / "vocab.duckdb"), test_only=True + dialect="duckdb", + database_name=str(Path(tmp_dir) / "vocab.duckdb"), + test_only=True, ), }, databases={"cdm": CDMDatabaseConfig(connection="primary", vocab_connection="vocab")}, ) resolved = Resolver(stack).resolve_database("cdm") - primary_engine, vocab_engine = resolved.create_engines(schema_claims=[staging_schema_claim()]) + assert isinstance(resolved, ResolvedCDMDatabase) + primary_engine, vocab_engine = resolved.create_engines( + schema_claims=[staging_schema_claim()] + ) try: with vocab_engine.connect() as vocab_conn: create_tables(vocab_conn, [VocabSchemaTable.__table__], resolved=resolved) @@ -137,13 +158,18 @@ def duckdb_split_database_session_with_ephemeral_vocab(): stack = StackConfig.for_session( connections={ "primary": ConnectionConfig( - dialect="duckdb", database_name=str(Path(tmp_dir) / "primary.duckdb"), test_only=True + dialect="duckdb", + database_name=str(Path(tmp_dir) / "primary.duckdb"), + test_only=True, + ), + "vocab": ConnectionConfig( + dialect="duckdb", database_name=":memory:", test_only=True ), - "vocab": ConnectionConfig(dialect="duckdb", database_name=":memory:", test_only=True), }, databases={"cdm": CDMDatabaseConfig(connection="primary", vocab_connection="vocab")}, ) resolved = Resolver(stack).resolve_database("cdm") + assert isinstance(resolved, ResolvedCDMDatabase) primary_engine, _ = resolved.create_engines(schema_claims=[staging_schema_claim()]) try: with primary_engine.connect() as conn: @@ -168,7 +194,9 @@ def test_duckdb_backend_identity_and_capabilities(): @pytest.mark.db_dialect @pytest.mark.parametrize("merge_batch_size", [0, -1]) @pytest.mark.parametrize("merge_strategy", ["replace", "upsert", "insert"]) -def test_duckdb_backend_rejects_non_positive_merge_batch_size(merge_batch_size, merge_strategy, duckdb_session): +def test_duckdb_backend_rejects_non_positive_merge_batch_size( + merge_batch_size, merge_strategy, duckdb_session +): backend = DuckDBBackend() args = [_ComputedTableCls, duckdb_session] if merge_strategy in {"replace", "upsert"}: @@ -191,7 +219,9 @@ def test_duckdb_backend_create_staging_table(duckdb_session): cols = inspector.get_columns(_STAGING_TABLE) assert [c["name"] for c in cols] == ["id", "name", "_rownum"] target = {c.name: c.nullable for c in ComputedColumnTable.__table__.columns} - assert {c["name"]: c["nullable"] for c in cols if c["name"] != "_rownum"} == {name: target[name] for name in ("id", "name")} + assert {c["name"]: c["nullable"] for c in cols if c["name"] != "_rownum"} == { + name: target[name] for name in ("id", "name") + } @pytest.mark.duckdb @@ -215,22 +245,28 @@ def test_duckdb_backend_drop_staging_table_also_drops_rownum_sequence(duckdb_ses backend = DuckDBBackend() backend.create_staging_table(_ComputedTableCls, duckdb_session) - sequences_while_present = duckdb_session.execute( - sa.text("SELECT sequence_name FROM duckdb_sequences()") - ).scalars().all() + sequences_while_present = ( + duckdb_session.execute(sa.text("SELECT sequence_name FROM duckdb_sequences()")) + .scalars() + .all() + ) assert any(f"{_STAGING_TABLE}_rownum_seq" in name for name in sequences_while_present) backend.drop_staging_table(_ComputedTableCls, duckdb_session) - sequences_after_drop = duckdb_session.execute( - sa.text("SELECT sequence_name FROM duckdb_sequences()") - ).scalars().all() + sequences_after_drop = ( + duckdb_session.execute(sa.text("SELECT sequence_name FROM duckdb_sequences()")) + .scalars() + .all() + ) assert not any(f"{_STAGING_TABLE}_rownum_seq" in name for name in sequences_after_drop) @pytest.mark.duckdb @pytest.mark.db_dialect -def test_duckdb_backend_creates_staging_again_when_only_its_sequence_was_left_behind(duckdb_session): +def test_duckdb_backend_creates_staging_again_when_only_its_sequence_was_left_behind( + duckdb_session, +): backend = DuckDBBackend() backend.create_staging_table(_ComputedTableCls, duckdb_session) duckdb_session.execute(sa.text(f"DROP TABLE {_STAGING_TABLE}")) @@ -249,7 +285,9 @@ def test_duckdb_backend_rownum_populates_monotonically_on_insert(duckdb_session) duckdb_session.execute(sa.insert(staging), [{"id": 1, "name": "a"}, {"id": 2, "name": "b"}]) - rownums = duckdb_session.execute(sa.select(staging.c._rownum).order_by(staging.c.id)).scalars().all() + rownums = ( + duckdb_session.execute(sa.select(staging.c._rownum).order_by(staging.c.id)).scalars().all() + ) assert rownums == [1, 2] @@ -273,9 +311,11 @@ def test_duckdb_backend_relaxed_foreign_keys_runs_without_toggling_fk(duckdb_ses @pytest.mark.duckdb @pytest.mark.db_dialect def test_relaxed_foreign_keys_on_duckdb_does_not_log_toggling(duckdb_session, caplog): - with caplog.at_level("INFO", logger="orm_loader.backends.base"): - with DuckDBBackend().relaxed_foreign_keys(duckdb_session): - pass + with ( + caplog.at_level("INFO", logger="orm_loader.backends.base"), + DuckDBBackend().relaxed_foreign_keys(duckdb_session), + ): + pass assert "foreign key" not in caplog.text.lower() @@ -296,7 +336,9 @@ def test_duckdb_backend_deferred_fk_constraints_finds_the_cycle(duckdb_session, @pytest.mark.duckdb @pytest.mark.db_dialect -def test_duckdb_backend_validate_deferred_fks_passes_for_consistent_data(duckdb_session, duckdb_backend): +def test_duckdb_backend_validate_deferred_fks_passes_for_consistent_data( + duckdb_session, duckdb_backend +): duckdb_session.execute(sa.insert(CycleBTable), [{"id": 1, "a_id": None}]) duckdb_session.execute(sa.insert(CycleATable), [{"id": 1, "b_id": 1}]) @@ -336,7 +378,9 @@ def test_duckdb_backend_merge_replace_updates_in_place(duckdb_session): backend.merge_replace(_ComputedTableCls, duckdb_session, ["id"], staging) backend.merge_insert(_ComputedTableCls, duckdb_session, staging) - rows = {r.id: r.name for r in duckdb_session.execute(sa.select(ComputedColumnTable)).scalars().all()} + rows = { + r.id: r.name for r in duckdb_session.execute(sa.select(ComputedColumnTable)).scalars().all() + } assert rows == {1: "alpha-staged", 2: "beta", 3: "gamma"} @@ -354,13 +398,17 @@ def test_duckdb_backend_merge_replace_paginated_updates_in_place(duckdb_session) backend.merge_replace(_ComputedTableCls, duckdb_session, ["id"], staging, merge_batch_size=3) backend.merge_insert(_ComputedTableCls, duckdb_session, staging, merge_batch_size=3) - rows = {r.id: r.name for r in duckdb_session.execute(sa.select(ComputedColumnTable)).scalars().all()} + rows = { + r.id: r.name for r in duckdb_session.execute(sa.select(ComputedColumnTable)).scalars().all() + } assert rows == {i: f"staged{i}" for i in range(10)} @pytest.mark.duckdb @pytest.mark.db_dialect -def test_duckdb_backend_merge_replace_rejects_referenced_table_with_own_fk(duckdb_session, duckdb_backend): +def test_duckdb_backend_merge_replace_rejects_referenced_table_with_own_fk( + duckdb_session, duckdb_backend +): """merge_replace must reject a still-referenced row up front, before touching any data, rather than partway through a batch. """ @@ -371,7 +419,9 @@ def test_duckdb_backend_merge_replace_rejects_referenced_table_with_own_fk(duckd sa.insert(ReferencedParentTable), [{"id": 1, "grandparent_id": 1, "name": "p1"}] ) duckdb_session.execute(sa.insert(ReferencingChildTable), [{"id": 1, "parent_id": 1}]) - duckdb_session.execute(sa.insert(staging), [{"id": 1, "grandparent_id": 1, "name": "p1-updated"}]) + duckdb_session.execute( + sa.insert(staging), [{"id": 1, "grandparent_id": 1, "name": "p1-updated"}] + ) with pytest.raises(ValueError, match="referenced_parent_table"): duckdb_backend.merge_replace(_ReferencedParentTableCls, duckdb_session, ["id"], staging) @@ -380,7 +430,8 @@ def test_duckdb_backend_merge_replace_rejects_referenced_table_with_own_fk(duckd @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_insert_batch_validation_prevents_partial_commit( - duckdb_session_with_committed_schema, duckdb_backend, + duckdb_session_with_committed_schema, + duckdb_backend, ): """merge_batch_size=1 commits each row separately. Row 3's own batch (the orphan) must be caught and never committed, while rows 1 and 2, @@ -388,7 +439,10 @@ def test_duckdb_backend_merge_insert_batch_validation_prevents_partial_commit( """ session = duckdb_session_with_committed_schema staging = duckdb_backend.create_staging_table(_CycleATableCls, session) - session.execute(sa.insert(staging), [{"id": 1, "b_id": None}, {"id": 2, "b_id": None}, {"id": 3, "b_id": 999}]) + session.execute( + sa.insert(staging), + [{"id": 1, "b_id": None}, {"id": 2, "b_id": None}, {"id": 3, "b_id": 999}], + ) with pytest.raises(DeferredForeignKeyViolationError, match="cycle_a_table"): duckdb_backend.merge_insert(_CycleATableCls, session, staging, merge_batch_size=1) @@ -401,7 +455,8 @@ def test_duckdb_backend_merge_insert_batch_validation_prevents_partial_commit( @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_insert_loads_self_referential_rows_in_dependency_order( - duckdb_session, duckdb_backend, + duckdb_session, + duckdb_backend, ): """Staging rows arrive in reverse dependency order (each pointing at a row not yet committed); merge_insert must still load every row, in @@ -413,30 +468,44 @@ def test_duckdb_backend_merge_insert_loads_self_referential_rows_in_dependency_o [{"id": 3, "parent_id": 2}, {"id": 2, "parent_id": 1}, {"id": 1, "parent_id": None}], ) - duckdb_backend.merge_insert(_SelfReferentialTableCls, duckdb_session, staging, merge_batch_size=1) + duckdb_backend.merge_insert( + _SelfReferentialTableCls, duckdb_session, staging, merge_batch_size=1 + ) - rows = {r.id: r.parent_id for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all()} + rows = { + r.id: r.parent_id + for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all() + } assert rows == {1: None, 2: 1, 3: 2} @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_upsert_loads_self_referential_rows_in_dependency_order( - duckdb_session, duckdb_backend, + duckdb_session, + duckdb_backend, ): staging = duckdb_backend.create_staging_table(_SelfReferentialTableCls, duckdb_session) - duckdb_session.execute(sa.insert(staging), [{"id": 2, "parent_id": 1}, {"id": 1, "parent_id": None}]) + duckdb_session.execute( + sa.insert(staging), [{"id": 2, "parent_id": 1}, {"id": 1, "parent_id": None}] + ) - duckdb_backend.merge_upsert(_SelfReferentialTableCls, duckdb_session, ["id"], staging, merge_batch_size=1) + duckdb_backend.merge_upsert( + _SelfReferentialTableCls, duckdb_session, ["id"], staging, merge_batch_size=1 + ) - rows = {r.id: r.parent_id for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all()} + rows = { + r.id: r.parent_id + for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all() + } assert rows == {1: None, 2: 1} @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_replace_updates_and_inserts_self_referential_rows_together( - duckdb_session, duckdb_backend, + duckdb_session, + duckdb_backend, ): """An existing row is re-pointed at one only this load adds. Processed as two sealed phases, the update would be validated before its target @@ -448,15 +517,21 @@ def test_duckdb_backend_merge_replace_updates_and_inserts_self_referential_rows_ sa.insert(staging), [{"id": 1, "parent_id": 2}, {"id": 2, "parent_id": None}] ) - duckdb_backend.merge_replace(_SelfReferentialTableCls, duckdb_session, ["id"], staging, merge_batch_size=1) + duckdb_backend.merge_replace( + _SelfReferentialTableCls, duckdb_session, ["id"], staging, merge_batch_size=1 + ) - rows = {r.id: r.parent_id for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all()} + rows = { + r.id: r.parent_id + for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all() + } assert rows == {1: 2, 2: None} assert duckdb_session.execute(sa.select(staging.c.id)).scalars().all() == [] duckdb_backend.merge_insert(_SelfReferentialTableCls, duckdb_session, staging) rows_after_insert = { - r.id: r.parent_id for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all() + r.id: r.parent_id + for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all() } assert rows_after_insert == rows @@ -464,7 +539,8 @@ def test_duckdb_backend_merge_replace_updates_and_inserts_self_referential_rows_ @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_replace_catches_a_genuine_orphan( - duckdb_session_with_committed_schema, duckdb_backend, + duckdb_session_with_committed_schema, + duckdb_backend, ): """A replace batch re-pointing an existing row at a genuine orphan (not merely a not-yet-inserted new row) must still be caught, not mistaken @@ -477,7 +553,9 @@ def test_duckdb_backend_merge_replace_catches_a_genuine_orphan( session.execute(sa.insert(staging), [{"id": 1, "parent_id": 999}]) with pytest.raises(DeferredForeignKeyViolationError, match="self_referential_table"): - duckdb_backend.merge_replace(_SelfReferentialTableCls, session, ["id"], staging, merge_batch_size=1) + duckdb_backend.merge_replace( + _SelfReferentialTableCls, session, ["id"], staging, merge_batch_size=1 + ) session.rollback() assert session.execute(sa.select(SelfReferentialTable.parent_id)).scalars().all() == [None] @@ -487,7 +565,8 @@ def test_duckdb_backend_merge_replace_catches_a_genuine_orphan( @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_insert_catches_a_genuine_orphan_in_its_own_batch( - duckdb_session_with_committed_schema, duckdb_backend, + duckdb_session_with_committed_schema, + duckdb_backend, ): """A row whose parent never arrives is a genuine violation, caught in its own batch's commit. A row that became ready in an earlier batch @@ -508,7 +587,8 @@ def test_duckdb_backend_merge_insert_catches_a_genuine_orphan_in_its_own_batch( @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_insert_unbatched_has_no_intermediate_commit( - duckdb_session_with_committed_schema, duckdb_backend, + duckdb_session_with_committed_schema, + duckdb_backend, ): """Without merge_batch_size, nothing may commit until the caller's own validation runs, even when a later row is a genuine orphan. Rolling @@ -537,10 +617,15 @@ def test_duckdb_backend_merge_insert_accepts_a_self_link(duckdb_session, duckdb_ staging = duckdb_backend.create_staging_table(_SelfReferentialTableCls, duckdb_session) duckdb_session.execute(sa.insert(staging), [{"id": 1, "parent_id": 1}]) - duckdb_backend.merge_insert(_SelfReferentialTableCls, duckdb_session, staging, merge_batch_size=1) + duckdb_backend.merge_insert( + _SelfReferentialTableCls, duckdb_session, staging, merge_batch_size=1 + ) duckdb_backend.validate_deferred_fks(_SelfReferentialTableCls, duckdb_session) - rows = {r.id: r.parent_id for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all()} + rows = { + r.id: r.parent_id + for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all() + } assert rows == {1: 1} @@ -551,19 +636,27 @@ def test_duckdb_backend_merge_insert_accepts_a_two_row_cycle(duckdb_session, duc can ever become ready on its own, which is not a reason to reject them. """ staging = duckdb_backend.create_staging_table(_SelfReferentialTableCls, duckdb_session) - duckdb_session.execute(sa.insert(staging), [{"id": 1, "parent_id": 2}, {"id": 2, "parent_id": 1}]) + duckdb_session.execute( + sa.insert(staging), [{"id": 1, "parent_id": 2}, {"id": 2, "parent_id": 1}] + ) - duckdb_backend.merge_insert(_SelfReferentialTableCls, duckdb_session, staging, merge_batch_size=1) + duckdb_backend.merge_insert( + _SelfReferentialTableCls, duckdb_session, staging, merge_batch_size=1 + ) duckdb_backend.validate_deferred_fks(_SelfReferentialTableCls, duckdb_session) - rows = {r.id: r.parent_id for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all()} + rows = { + r.id: r.parent_id + for r in duckdb_session.execute(sa.select(SelfReferentialTable)).scalars().all() + } assert rows == {1: 2, 2: 1} @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_upsert_ignores_a_duplicate_with_a_bad_reference( - duckdb_session_with_committed_schema, duckdb_backend, + duckdb_session_with_committed_schema, + duckdb_backend, ): """A staging row ON CONFLICT DO NOTHING will discard must not be judged as if it were going to be stored. @@ -573,17 +666,22 @@ def test_duckdb_backend_merge_upsert_ignores_a_duplicate_with_a_bad_reference( staging = duckdb_backend.create_staging_table(_SelfReferentialTableCls, session) session.execute(sa.insert(staging), [{"id": 1, "parent_id": 999}]) - duckdb_backend.merge_upsert(_SelfReferentialTableCls, session, ["id"], staging, merge_batch_size=1) + duckdb_backend.merge_upsert( + _SelfReferentialTableCls, session, ["id"], staging, merge_batch_size=1 + ) duckdb_backend.validate_deferred_fks(_SelfReferentialTableCls, session) - rows = {r.id: r.parent_id for r in session.execute(sa.select(SelfReferentialTable)).scalars().all()} + rows = { + r.id: r.parent_id for r in session.execute(sa.select(SelfReferentialTable)).scalars().all() + } assert rows == {1: None} @pytest.mark.duckdb @pytest.mark.db_dialect def test_duckdb_backend_merge_insert_respects_validate_deferred_fks_false( - duckdb_session_with_committed_schema, duckdb_backend, + duckdb_session_with_committed_schema, + duckdb_backend, ): """Opting out of application validation must not be overridden by the self-referential batching path. @@ -596,13 +694,17 @@ def test_duckdb_backend_merge_insert_respects_validate_deferred_fks_false( _SelfReferentialTableCls, session, staging, merge_batch_size=1, validate_deferred_fks=False ) - rows = {r.id: r.parent_id for r in session.execute(sa.select(SelfReferentialTable)).scalars().all()} + rows = { + r.id: r.parent_id for r in session.execute(sa.select(SelfReferentialTable)).scalars().all() + } assert rows == {1: 999} @pytest.mark.duckdb @pytest.mark.db_dialect -def test_duckdb_backend_validate_deferred_fks_catches_a_cross_database_orphan(duckdb_split_database_session): +def test_duckdb_backend_validate_deferred_fks_catches_a_cross_database_orphan( + duckdb_split_database_session, +): """A cross-database FK is genuinely checked, via a second connection, not skipped. An orphan on the other database must still be caught. """ @@ -660,7 +762,9 @@ def test_duckdb_backend_validate_deferred_fks_refuses_an_ephemeral_remote_connec @pytest.mark.duckdb @pytest.mark.db_dialect -def test_duckdb_backend_merge_insert_batch_validation_is_a_no_op_without_deferred_fks(duckdb_session): +def test_duckdb_backend_merge_insert_batch_validation_is_a_no_op_without_deferred_fks( + duckdb_session, +): """A table with no deferred/unenforced FK must not pay for a validation query on every batch commit. """ @@ -713,7 +817,9 @@ def test_duckdb_backend_deferred_fk_constraints_classifies_cross_schema_fk_with_ @pytest.mark.duckdb @pytest.mark.db_dialect -def test_duckdb_backend_validate_deferred_fks_catches_a_cross_schema_orphan(duckdb_split_schema_session): +def test_duckdb_backend_validate_deferred_fks_catches_a_cross_schema_orphan( + duckdb_split_schema_session, +): """The cross-schema key was never created, so an orphaned reference inserts without error; only the anti-join check catches it. """ @@ -727,7 +833,9 @@ def test_duckdb_backend_validate_deferred_fks_catches_a_cross_schema_orphan(duck @pytest.mark.duckdb @pytest.mark.db_dialect -def test_duckdb_backend_tables_referencing_excludes_an_omitted_cross_schema_fk(duckdb_split_schema_session): +def test_duckdb_backend_tables_referencing_excludes_an_omitted_cross_schema_fk( + duckdb_split_schema_session, +): """VocabReferencingTable's foreign key into VocabSchemaTable was never created, so check_replace_feasible() must not treat it as a real referencing table. diff --git a/tests/backends/test_foreign_keys.py b/tests/backends/test_foreign_keys.py index 21c60b3..debace1 100644 --- a/tests/backends/test_foreign_keys.py +++ b/tests/backends/test_foreign_keys.py @@ -1,11 +1,11 @@ """Referential integrity after a load: the same check on every dialect.""" from pathlib import Path -from typing import Type, cast +from typing import cast import pytest - from oa_configurator import Dialect + from orm_loader.backends import STAGING_SCHEMA, ForeignKeyViolationError from orm_loader.tables.typing import FileTableProtocol from tests.models import ( @@ -16,11 +16,11 @@ UnreferencedGrandparentTable, ) -_Child = cast(Type[FileTableProtocol], ReferencingChildTable) -_Parent = cast(Type[FileTableProtocol], ReferencedParentTable) -_Grandparent = cast(Type[FileTableProtocol], UnreferencedGrandparentTable) -_Self = cast(Type[FileTableProtocol], SelfReferentialTable) -_Simple = cast(Type[FileTableProtocol], SimpleTable) +_Child = cast(type[FileTableProtocol], ReferencingChildTable) +_Parent = cast(type[FileTableProtocol], ReferencedParentTable) +_Grandparent = cast(type[FileTableProtocol], UnreferencedGrandparentTable) +_Self = cast(type[FileTableProtocol], SelfReferentialTable) +_Simple = cast(type[FileTableProtocol], SimpleTable) @pytest.fixture @@ -64,12 +64,21 @@ def test_null_references_are_not_violations(dialect_session, tmp_path, options): def test_an_orphan_is_reported_with_its_key(dialect_session, tmp_path, options): - _load(dialect_session, _Self, tmp_path, "id,parent_id\n1,\n2,99\n", options, validate_deferred_fks=False) + _load( + dialect_session, + _Self, + tmp_path, + "id,parent_id\n1,\n2,99\n", + options, + validate_deferred_fks=False, + ) with pytest.raises(ForeignKeyViolationError, match=r"self_referential_table.*\[\(99,\)\]"): _validate(dialect_session, _Self, options) -def test_tables_loaded_child_first_validate_once_the_parents_are_loaded(dialect_session, tmp_path, options): +def test_tables_loaded_child_first_validate_once_the_parents_are_loaded( + dialect_session, tmp_path, options +): if dialect_session.get_bind().dialect.name == Dialect.DUCKDB: pytest.skip("DuckDB enforces this key itself, so a child cannot be loaded first") _load(dialect_session, _Child, tmp_path, "id,parent_id\n1,1\n", options) diff --git a/tests/backends/test_postgres_backend.py b/tests/backends/test_postgres_backend.py index 3d745f0..ec56a3d 100644 --- a/tests/backends/test_postgres_backend.py +++ b/tests/backends/test_postgres_backend.py @@ -1,24 +1,29 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Type, cast +from typing import TYPE_CHECKING, cast import pytest import sqlalchemy as sa import sqlalchemy.orm as so -from sqlalchemy.dialects import postgresql - from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role, qualified from oa_configurator.testing import isolated_test_schema +from sqlalchemy import exc as sa_exc +from sqlalchemy.dialects import postgresql + from orm_loader.backends import STAGING_SCHEMA, Dialect, PostgresBackend from orm_loader.backends.resolve import resolve_backend -from tests.models import ComputedColumnTable, EnumTable, Role as AuthorRole, SimpleTable +from tests.models import ComputedColumnTable, EnumTable, SimpleTable +from tests.models import Role as AuthorRole _TARGET_TABLE = ComputedColumnTable.__tablename__ _STAGING_TABLE = f"_staging_{_TARGET_TABLE}" _PREPARER = postgresql.dialect().identifier_preparer -_STAGING_TABLE_WITH_SCHEMA: str = qualified(_PREPARER, _STAGING_TABLE, physical_schema=STAGING_SCHEMA) +_STAGING_TABLE_WITH_SCHEMA: str = qualified( + _PREPARER, _STAGING_TABLE, physical_schema=STAGING_SCHEMA +) -_ComputedTableCls = cast("Type[FileTableProtocol]", ComputedColumnTable) +_ComputedTableCls = cast("type[FileTableProtocol]", ComputedColumnTable) +_SimpleTableCls = cast("type[FileTableProtocol]", SimpleTable) if TYPE_CHECKING: @@ -26,7 +31,7 @@ class _FakeSession: - dialect = postgresql.dialect() + dialect: sa.engine.Dialect = postgresql.dialect() def __init__( self, @@ -77,6 +82,10 @@ def _sess(s: _FakeSession) -> so.Session: return cast(so.Session, s) +def _bind(s: _FakeSession) -> sa.engine.Connection: + return cast(sa.engine.Connection, s) + + def test_postgres_backend_identity_and_capabilities(): backend = PostgresBackend() @@ -101,13 +110,13 @@ def test_postgres_backend_rejects_non_positive_merge_batch_size(merge_batch_size merge(*args, merge_batch_size=merge_batch_size) - - def test_postgres_backend_default_staging_schema_is_none(): backend = PostgresBackend() assert backend.staging_schema is None - assert backend.qualified_staging_name(_TARGET_TABLE) == qualified(_PREPARER, _STAGING_TABLE, physical_schema=None) + assert backend.qualified_staging_name(_TARGET_TABLE) == qualified( + _PREPARER, _STAGING_TABLE, physical_schema=None + ) def test_postgres_backend_create_staging_table_drops_computed_columns(pg_session): @@ -143,18 +152,23 @@ def test_postgres_backend_materialized_view_respects_schema(pg_db) -> None: selectable = sa.select(sa.literal(1).label("n")) engine = pg_db.connection.engine - with isolated_test_schema(engine, prefix="mv_primary") as primary_schema, \ - isolated_test_schema(engine, prefix="mv_vocab") as vocab_schema: + with ( + isolated_test_schema(engine, prefix="mv_primary") as primary_schema, + isolated_test_schema(engine, prefix="mv_vocab") as vocab_schema, + ): with engine.begin() as conn: - backend.create_materialized_view(conn, "mv_schema_test", selectable, schema=vocab_schema) + backend.create_materialized_view( + conn, "mv_schema_test", selectable, schema=vocab_schema + ) backend.refresh_materialized_view(conn, "mv_schema_test", schema=vocab_schema) with engine.connect() as conn: assert sa.inspect(conn).has_table("mv_schema_test", schema=vocab_schema) assert not sa.inspect(conn).has_table("mv_schema_test", schema=primary_schema) - assert conn.execute( - sa.text(f'SELECT n FROM "{vocab_schema}".mv_schema_test') - ).scalar() == 1 + assert ( + conn.execute(sa.text(f'SELECT n FROM "{vocab_schema}".mv_schema_test')).scalar() + == 1 + ) def test_postgres_backend_materialized_view_methods_emit_expected_sql(): @@ -162,11 +176,14 @@ def test_postgres_backend_materialized_view_methods_emit_expected_sql(): session = _FakeSession() selectable = sa.select(sa.literal(1).label("n")) - backend.create_materialized_view(_sess(session), "mv_test", selectable) - backend.refresh_materialized_view(_sess(session), "mv_test") + backend.create_materialized_view(_bind(session), "mv_test", selectable) + backend.refresh_materialized_view(_bind(session), "mv_test") - assert any('CREATE MATERIALIZED VIEW IF NOT EXISTS mv_test as SELECT' in sql for sql in session.statements) - assert any('REFRESH MATERIALIZED VIEW mv_test;' == sql for sql in session.statements) + assert any( + "CREATE MATERIALIZED VIEW IF NOT EXISTS mv_test as SELECT" in sql + for sql in session.statements + ) + assert any("REFRESH MATERIALIZED VIEW mv_test;" == sql for sql in session.statements) def test_postgres_backend_rejects_mismatched_dialect_bind(): @@ -178,7 +195,7 @@ def test_postgres_backend_rejects_mismatched_dialect_bind(): selectable = sa.select(sa.literal(1).label("n")) with pytest.raises(TypeError, match="received a 'sqlite' connection; expected 'postgresql'"): - backend.create_materialized_view(_sess(session), "mv_test", selectable) + backend.create_materialized_view(_bind(session), "mv_test", selectable) assert session.statements == [] @@ -186,9 +203,10 @@ def test_postgres_backend_rejects_mismatched_dialect_bind(): def test_postgres_backend_rejects_dialect_that_drifted_after_resolve_backend(): """A bind resolved to PostgresBackend, then a differently-dialected bind handed to one of its methods, must not run Postgres-only DDL against it.""" - from orm_loader.backends.resolve import resolve_backend from sqlalchemy.dialects import sqlite + from orm_loader.backends.resolve import resolve_backend + postgres_session = _FakeSession() backend = resolve_backend(_sess(postgres_session)) assert isinstance(backend, PostgresBackend) @@ -198,7 +216,7 @@ def test_postgres_backend_rejects_dialect_that_drifted_after_resolve_backend(): selectable = sa.select(sa.literal(1).label("n")) with pytest.raises(TypeError, match="received a 'sqlite' connection; expected 'postgresql'"): - backend.create_materialized_view(_sess(sqlite_session), "mv_test", selectable) + backend.create_materialized_view(_bind(sqlite_session), "mv_test", selectable) def test_postgres_backend_quotes_unqualified_materialized_view_name(): @@ -208,12 +226,15 @@ def test_postgres_backend_quotes_unqualified_materialized_view_name(): session = _FakeSession() selectable = sa.select(sa.literal(1).label("n")) - backend.create_materialized_view(_sess(session), 'mv name', selectable) + backend.create_materialized_view(_bind(session), "mv name", selectable) backend.create_materialized_view_index( - _sess(session), 'mv name', MaterializedViewIndex(name="mv_name_idx", columns=("n",)) + _bind(session), "mv name", MaterializedViewIndex(name="mv_name_idx", columns=("n",)) ) - assert any('CREATE MATERIALIZED VIEW IF NOT EXISTS "mv name" as SELECT' in sql for sql in session.statements) + assert any( + 'CREATE MATERIALIZED VIEW IF NOT EXISTS "mv name" as SELECT' in sql + for sql in session.statements + ) assert any('ON "mv name" ("n")' in sql for sql in session.statements) @@ -221,7 +242,7 @@ def test_postgres_backend_create_mv_quotes_name_for_legacy_search_path_resolutio backend = PostgresBackend() session = _FakeSession() backend.create_materialized_view( - _sess(session), "MixedCaseMv", sa.select(sa.literal(1).label("n")) + _bind(session), "MixedCaseMv", sa.select(sa.literal(1).label("n")) ) assert any( @@ -237,7 +258,7 @@ def test_postgres_backend_create_materialized_view_index_emits_expected_sql(): session = _FakeSession(schema_translate_map={Role.PRIMARY.value: "reporting"}) index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",), unique=True) - backend.create_materialized_view_index(_sess(session), "mv_test", index, schema="reporting") + backend.create_materialized_view_index(_bind(session), "mv_test", index, schema="reporting") assert session.statements == [ 'CREATE UNIQUE INDEX IF NOT EXISTS "mv_test_row_id_uq" ON reporting.mv_test ("row_id")' @@ -245,8 +266,8 @@ def test_postgres_backend_create_materialized_view_index_emits_expected_sql(): def test_postgres_backend_create_materialized_view_index_failure_mentions_index_name(): - from orm_loader.mappers.materialised_view_errors import MaterializationError from orm_loader.mappers.materialised_view_contracts import MaterializedViewIndex + from orm_loader.mappers.materialised_view_errors import MaterializationError original = RuntimeError("boom") backend = PostgresBackend() @@ -254,16 +275,16 @@ def test_postgres_backend_create_materialized_view_index_failure_mentions_index_ index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",)) with pytest.raises(MaterializationError, match="mv_test_row_id_uq"): - backend.create_materialized_view_index(_sess(session), "mv_test", index) + backend.create_materialized_view_index(_bind(session), "mv_test", index) def test_postgres_backend_drop_materialized_view_default_args(): backend = PostgresBackend() session = _FakeSession(schema_translate_map={Role.PRIMARY.value: "reporting"}) - backend.drop_materialized_view(_sess(session), "mv_test", schema="reporting") + backend.drop_materialized_view(_bind(session), "mv_test", schema="reporting") - assert session.statements == ['DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test'] + assert session.statements == ["DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test"] def test_postgres_backend_drop_materialized_view_cascade_and_if_exists_false(): @@ -271,14 +292,17 @@ def test_postgres_backend_drop_materialized_view_cascade_and_if_exists_false(): session = _FakeSession(schema_translate_map={Role.PRIMARY.value: "reporting"}) backend.drop_materialized_view( - _sess(session), "mv_test", schema="reporting", if_exists=False, cascade=True + _bind(session), "mv_test", schema="reporting", if_exists=False, cascade=True ) - assert session.statements == ['DROP MATERIALIZED VIEW reporting.mv_test CASCADE'] + assert session.statements == ["DROP MATERIALIZED VIEW reporting.mv_test CASCADE"] def test_postgres_backend_drop_materialized_view_failure_preserves_cause(): - from orm_loader.mappers.materialised_view_errors import MaterializationError, MaterializationOperation + from orm_loader.mappers.materialised_view_errors import ( + MaterializationError, + MaterializationOperation, + ) original = RuntimeError("boom") backend = PostgresBackend() @@ -287,7 +311,7 @@ def test_postgres_backend_drop_materialized_view_failure_preserves_cause(): ) with pytest.raises(MaterializationError) as exc_info: - backend.drop_materialized_view(_sess(session), "mv_test") + backend.drop_materialized_view(_bind(session), "mv_test") assert exc_info.value.__cause__ is original assert exc_info.value.failure.cause is original @@ -295,7 +319,10 @@ def test_postgres_backend_drop_materialized_view_failure_preserves_cause(): def test_postgres_backend_create_materialized_view_failure_preserves_cause(): - from orm_loader.mappers.materialised_view_errors import MaterializationError, MaterializationOperation + from orm_loader.mappers.materialised_view_errors import ( + MaterializationError, + MaterializationOperation, + ) original = RuntimeError("boom") backend = PostgresBackend() @@ -303,7 +330,7 @@ def test_postgres_backend_create_materialized_view_failure_preserves_cause(): with pytest.raises(MaterializationError) as exc_info: backend.create_materialized_view( - _sess(session), "mv_test", sa.select(sa.literal(1).label("n")) + _bind(session), "mv_test", sa.select(sa.literal(1).label("n")) ) assert exc_info.value.__cause__ is original @@ -318,7 +345,7 @@ def test_postgres_backend_refresh_concurrently_without_declared_unique_index_rai session = _FakeSession(schema_translate_map={Role.PRIMARY.value: "reporting"}) with pytest.raises(ConcurrentRefreshNotEligibleError, match="no simple unique index"): - backend.refresh_materialized_view(_sess(session), "mv_test", concurrently=True) + backend.refresh_materialized_view(_bind(session), "mv_test", concurrently=True) assert session.statements == [] @@ -326,8 +353,8 @@ def test_postgres_backend_refresh_concurrently_without_declared_unique_index_rai def test_postgres_backend_refresh_concurrently_declared_but_database_rejects_it_translates_error(): psycopg = pytest.importorskip("psycopg") - from orm_loader.mappers.materialised_view_errors import ConcurrentRefreshNotEligibleError from orm_loader.mappers.materialised_view_contracts import MaterializedViewIndex + from orm_loader.mappers.materialised_view_errors import ConcurrentRefreshNotEligibleError orig = psycopg.errors.ObjectNotInPrerequisiteState( 'cannot refresh materialized view "reporting.mv_test" concurrently\n' @@ -336,24 +363,24 @@ def test_postgres_backend_refresh_concurrently_declared_but_database_rejects_it_ ) backend = PostgresBackend() session = _FakeSession( - raise_on_execute=sa.exc.OperationalError("REFRESH ...", {}, orig), + raise_on_execute=sa_exc.OperationalError("REFRESH ...", {}, orig), schema_translate_map={Role.PRIMARY.value: "reporting"}, ) index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",), unique=True) with pytest.raises(ConcurrentRefreshNotEligibleError, match="cannot refresh") as exc_info: backend.refresh_materialized_view( - _sess(session), + _bind(session), "mv_test", schema="reporting", concurrently=True, declared_indexes=(index,), ) - assert exc_info.value.failure.cause.orig is orig - assert session.statements == [ - 'REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;' - ] + cause = exc_info.value.failure.cause + assert isinstance(cause, sa_exc.OperationalError) + assert cause.orig is orig + assert session.statements == ["REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;"] def test_postgres_backend_refresh_concurrently_unrelated_operational_error_propagates_unchanged(): @@ -364,14 +391,14 @@ def test_postgres_backend_refresh_concurrently_unrelated_operational_error_propa orig = psycopg.errors.QueryCanceled("canceling statement due to statement timeout") backend = PostgresBackend() session = _FakeSession( - raise_on_execute=sa.exc.OperationalError("REFRESH ...", {}, orig), + raise_on_execute=sa_exc.OperationalError("REFRESH ...", {}, orig), schema_translate_map={Role.PRIMARY.value: "reporting"}, ) index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",), unique=True) - with pytest.raises(sa.exc.OperationalError) as exc_info: + with pytest.raises(sa_exc.OperationalError) as exc_info: backend.refresh_materialized_view( - _sess(session), + _bind(session), "mv_test", concurrently=True, declared_indexes=(index,), @@ -388,16 +415,14 @@ def test_postgres_backend_refresh_concurrently_with_declared_index_emits_concurr index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",), unique=True) backend.refresh_materialized_view( - _sess(session), + _bind(session), "mv_test", schema="reporting", concurrently=True, declared_indexes=(index,), ) - assert session.statements[-1] == ( - 'REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;' - ) + assert session.statements[-1] == ("REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;") def test_postgres_backend_materialized_view_lifecycle_is_schema_isolated_with_adversarial_identifiers( @@ -433,20 +458,26 @@ def test_postgres_backend_materialized_view_lifecycle_is_schema_isolated_with_ad backend.drop_materialized_view(conn, name, schema=left_schema) with engine.connect() as conn: - assert conn.execute( - sa.text( - "SELECT EXISTS (SELECT 1 FROM pg_matviews " - "WHERE schemaname = :schema AND matviewname = :name)" - ), - {"schema": left_schema, "name": name}, - ).scalar() is False - assert conn.execute( - sa.text( - "SELECT EXISTS (SELECT 1 FROM pg_matviews " - "WHERE schemaname = :schema AND matviewname = :name)" - ), - {"schema": right_schema, "name": name}, - ).scalar() is True + assert ( + conn.execute( + sa.text( + "SELECT EXISTS (SELECT 1 FROM pg_matviews " + "WHERE schemaname = :schema AND matviewname = :name)" + ), + {"schema": left_schema, "name": name}, + ).scalar() + is False + ) + assert ( + conn.execute( + sa.text( + "SELECT EXISTS (SELECT 1 FROM pg_matviews " + "WHERE schemaname = :schema AND matviewname = :name)" + ), + {"schema": right_schema, "name": name}, + ).scalar() + is True + ) finally: with engine.begin() as cleanup_conn: for schema in (left_schema, right_schema): @@ -456,8 +487,8 @@ def test_postgres_backend_materialized_view_lifecycle_is_schema_isolated_with_ad def test_postgres_backend_refresh_concurrently_raises_when_declared_index_was_never_created(pg_db): - from orm_loader.mappers.materialised_view_errors import ConcurrentRefreshNotEligibleError from orm_loader.mappers.materialised_view_contracts import MaterializedViewIndex + from orm_loader.mappers.materialised_view_errors import ConcurrentRefreshNotEligibleError backend = PostgresBackend() index = MaterializedViewIndex(name="mv_missing_index_test_uq", columns=("row_id",), unique=True) @@ -476,21 +507,19 @@ def test_postgres_backend_refresh_concurrently_raises_when_declared_index_was_ne ) assert "concurrently" in str(exc_info.value).lower() - assert isinstance(exc_info.value.__cause__, sa.exc.OperationalError) + assert isinstance(exc_info.value.__cause__, sa_exc.OperationalError) def test_postgres_backend_materialized_view_legacy_unqualified_path_still_round_trips(pg_db): backend = PostgresBackend() conn = pg_db.connection - backend.create_materialized_view( - conn, "mv_legacy_test", sa.select(sa.literal(1).label("n")) - ) + backend.create_materialized_view(conn, "mv_legacy_test", sa.select(sa.literal(1).label("n"))) backend.refresh_materialized_view(conn, "mv_legacy_test") assert conn.execute(sa.text("SELECT n FROM mv_legacy_test")).scalar() == 1 backend.drop_materialized_view(conn, "mv_legacy_test") - with pytest.raises(sa.exc.ProgrammingError): + with pytest.raises(sa_exc.ProgrammingError): conn.execute(sa.text("SELECT n FROM mv_legacy_test")) @@ -522,22 +551,37 @@ def _write_simple(tmp_path, rows: list[tuple[int, str]]): def _simple_rows(session) -> list[tuple[int, str]]: - return [(r.id, r.name) for r in session.execute(sa.select(SimpleTable).order_by(SimpleTable.id)).scalars()] + return [ + (r.id, r.name) + for r in session.execute(sa.select(SimpleTable).order_by(SimpleTable.id)).scalars() + ] def test_staging_table_has_the_target_columns_and_an_identity_row_number(pg_session): - resolve_backend(pg_session, staging_schema_tag=STAGING_SCHEMA, mapper=SimpleTable).create_staging_table(SimpleTable, pg_session) + resolve_backend( + pg_session, staging_schema_tag=STAGING_SCHEMA, mapper=SimpleTable + ).create_staging_table(_SimpleTableCls, pg_session) staging_name = resolve_backend(pg_session).staging_name_for_table(SimpleTable.__tablename__) - cols = pg_session.execute(sa.text( - "SELECT column_name, data_type FROM information_schema.columns" - " WHERE table_schema = :s AND table_name = :t ORDER BY ordinal_position" - ), {"s": STAGING_SCHEMA, "t": staging_name}).all() - assert [(n, d) for n, d in cols if n != "_rownum"] == [("id", "integer"), ("name", "character varying")] + cols = pg_session.execute( + sa.text( + "SELECT column_name, data_type FROM information_schema.columns" + " WHERE table_schema = :s AND table_name = :t ORDER BY ordinal_position" + ), + {"s": STAGING_SCHEMA, "t": staging_name}, + ).all() + assert [(n, d) for n, d in cols if n != "_rownum"] == [ + ("id", "integer"), + ("name", "character varying"), + ] assert ("_rownum", "bigint") in [(n, d) for n, d in cols] def test_load_file_merges_into_the_target(pg_session, tmp_path): - inserted = SimpleTable.load_file(pg_session, _write_simple(tmp_path, [(1, "alpha"), (2, "beta")]), staging_schema_tag=STAGING_SCHEMA) + inserted = SimpleTable.load_file( + pg_session, + _write_simple(tmp_path, [(1, "alpha"), (2, "beta")]), + staging_schema_tag=STAGING_SCHEMA, + ) pg_session.commit() assert inserted == 2 assert _simple_rows(pg_session) == [(1, "alpha"), (2, "beta")] @@ -545,17 +589,26 @@ def test_load_file_merges_into_the_target(pg_session, tmp_path): def test_a_large_file_loads_in_full(pg_session, tmp_path): rows = [(i, f"name_{i}") for i in range(1, 10_000)] - inserted = SimpleTable.load_file(pg_session, _write_simple(tmp_path, rows), staging_schema_tag=STAGING_SCHEMA) + inserted = SimpleTable.load_file( + pg_session, _write_simple(tmp_path, rows), staging_schema_tag=STAGING_SCHEMA + ) pg_session.commit() assert inserted == 9999 - assert pg_session.execute(sa.select(sa.func.count()).select_from(SimpleTable)).scalar_one() == 9999 + assert ( + pg_session.execute(sa.select(sa.func.count()).select_from(SimpleTable)).scalar_one() == 9999 + ) def test_upsert_does_not_update_existing_rows(pg_session, tmp_path): - SimpleTable.load_file(pg_session, _write_simple(tmp_path, [(1, "alpha")]), staging_schema_tag=STAGING_SCHEMA) + SimpleTable.load_file( + pg_session, _write_simple(tmp_path, [(1, "alpha")]), staging_schema_tag=STAGING_SCHEMA + ) pg_session.commit() SimpleTable.load_file( - pg_session, _write_simple(tmp_path, [(1, "alpha_updated")]), merge_strategy="upsert", staging_schema_tag=STAGING_SCHEMA + pg_session, + _write_simple(tmp_path, [(1, "alpha_updated")]), + merge_strategy="upsert", + staging_schema_tag=STAGING_SCHEMA, ) pg_session.commit() assert _simple_rows(pg_session) == [(1, "alpha")] @@ -563,11 +616,18 @@ def test_upsert_does_not_update_existing_rows(pg_session, tmp_path): def test_insert_loads_new_keys_and_fails_on_an_existing_key(pg_session, tmp_path): path = _write_simple(tmp_path, [(1, "alpha"), (2, "beta")]) - assert SimpleTable.load_file(pg_session, path, merge_strategy="insert", staging_schema_tag=STAGING_SCHEMA) == 2 + assert ( + SimpleTable.load_file( + pg_session, path, merge_strategy="insert", staging_schema_tag=STAGING_SCHEMA + ) + == 2 + ) pg_session.commit() assert _simple_rows(pg_session) == [(1, "alpha"), (2, "beta")] - with pytest.raises(sa.exc.IntegrityError): - SimpleTable.load_file(pg_session, path, merge_strategy="insert", staging_schema_tag=STAGING_SCHEMA) + with pytest.raises(sa_exc.IntegrityError): + SimpleTable.load_file( + pg_session, path, merge_strategy="insert", staging_schema_tag=STAGING_SCHEMA + ) def test_a_native_enum_column_loads_member_names(pg_session, tmp_path): diff --git a/tests/backends/test_postgres_fk_scope.py b/tests/backends/test_postgres_fk_scope.py index 29f4bab..2416666 100644 --- a/tests/backends/test_postgres_fk_scope.py +++ b/tests/backends/test_postgres_fk_scope.py @@ -13,7 +13,10 @@ @pytest.fixture def pooled_pg_engine(pg_db): engine = sa.create_engine( - pg_db.connection.engine.url, pool_size=2, max_overflow=0, pool_use_lifo=False, + pg_db.connection.engine.url, + pool_size=2, + max_overflow=0, + pool_use_lifo=False, ) try: first = engine.connect() @@ -27,7 +30,8 @@ def pooled_pg_engine(pg_db): @pytest.mark.parametrize("end_transaction", ["commit", "rollback"]) def test_relaxation_follows_connection_swaps_without_affecting_other_sessions( - pooled_pg_engine, end_transaction, + pooled_pg_engine, + end_transaction, ): backend = PostgresBackend() with so.Session(pooled_pg_engine) as session: @@ -53,12 +57,14 @@ def test_relaxation_follows_connection_swaps_without_affecting_other_sessions( def test_failed_scope_rolls_back_and_removes_transaction_listener(pooled_pg_engine): with so.Session(pooled_pg_engine) as session: - with pytest.raises(RuntimeError, match="stop the load"): - with PostgresBackend().relaxed_foreign_keys(session): - session.execute(sa.text("SELECT 1")) - session.commit() - assert session.scalar(sa.text("SHOW session_replication_role")) == "replica" - raise RuntimeError("stop the load") + with ( + pytest.raises(RuntimeError, match="stop the load"), + PostgresBackend().relaxed_foreign_keys(session), + ): + session.execute(sa.text("SELECT 1")) + session.commit() + assert session.scalar(sa.text("SHOW session_replication_role")) == "replica" + raise RuntimeError("stop the load") assert session.scalar(sa.text("SHOW session_replication_role")) == "origin" with pooled_pg_engine.connect() as first, pooled_pg_engine.connect() as second: assert first.scalar(sa.text("SHOW session_replication_role")) == "origin" @@ -84,14 +90,16 @@ def test_relaxation_is_limited_to_the_mapper_bind(pooled_pg_engine): other_engine = sa.create_engine(pooled_pg_engine.url) try: - with so.Session(binds={SimpleTable: pooled_pg_engine, EnumTable: other_engine}) as session: - with PostgresBackend().relaxed_foreign_keys(session, mapper=SimpleTable): - for _ in range(2): - target = session.connection(bind_arguments={"mapper": SimpleTable}) - unrelated = session.connection(bind_arguments={"mapper": EnumTable}) - assert target.scalar(sa.text("SHOW session_replication_role")) == "replica" - assert unrelated.scalar(sa.text("SHOW session_replication_role")) == "origin" - session.commit() + with ( + so.Session(binds={SimpleTable: pooled_pg_engine, EnumTable: other_engine}) as session, + PostgresBackend().relaxed_foreign_keys(session, mapper=SimpleTable), + ): + for _ in range(2): + target = session.connection(bind_arguments={"mapper": SimpleTable}) + unrelated = session.connection(bind_arguments={"mapper": EnumTable}) + assert target.scalar(sa.text("SHOW session_replication_role")) == "replica" + assert unrelated.scalar(sa.text("SHOW session_replication_role")) == "origin" + session.commit() finally: other_engine.dispose() @@ -150,6 +158,8 @@ def test_orm_inserts_must_be_flushed_inside_relaxation(pooled_pg_engine, flush_i with pytest.raises(IntegrityError): session.commit() with so.Session(pooled_pg_engine) as verify: - assert verify.scalar(sa.select(sa.func.count()).select_from(_ScopeChild)) == int(flush_inside) + assert verify.scalar(sa.select(sa.func.count()).select_from(_ScopeChild)) == int( + flush_inside + ) finally: _ScopeBase.metadata.drop_all(pooled_pg_engine) diff --git a/tests/backends/test_reserved_schema.py b/tests/backends/test_reserved_schema.py index 92eaed7..58d233d 100644 --- a/tests/backends/test_reserved_schema.py +++ b/tests/backends/test_reserved_schema.py @@ -17,14 +17,15 @@ from __future__ import annotations +import pytest from oa_configurator import ( CDMDatabaseConfig, GenericDatabaseConfig, + ResolvedCDMDatabase, Resolver, SchemaOwnershipError, ) from oa_configurator.testing import reset_schema_registry_rows -import pytest from orm_loader.backends import STAGING_SCHEMA, staging_schema_claim @@ -49,8 +50,10 @@ def test_resolving_cdm_database_with_staging_schema_name_raises(pg_db, cleanup_a engine = resolver.resolve_database("loader").create_engine( schema_claims=[staging_schema_claim()] ) + cdm = resolver.resolve_database(cdm_name) + assert isinstance(cdm, ResolvedCDMDatabase) try: with pytest.raises(SchemaOwnershipError, match=f"{STAGING_SCHEMA!r}.*orm_loader"): - resolver.resolve_database(cdm_name).create_engines() + cdm.create_engines() finally: engine.dispose() diff --git a/tests/backends/test_shared_backend.py b/tests/backends/test_shared_backend.py index a21a91b..b174be4 100644 --- a/tests/backends/test_shared_backend.py +++ b/tests/backends/test_shared_backend.py @@ -1,24 +1,24 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Type, cast +from typing import TYPE_CHECKING, cast import pytest import sqlalchemy as sa - from oa_configurator import Dialect from oa_configurator.testing import DIALECT_PARAMS + from orm_loader.backends import STAGING_SCHEMA, DatabaseBackend from orm_loader.backends.resolve import resolve_backend from tests.conftest import session_for -from tests.models import ComputedColumnTable, CompositeTable +from tests.models import CompositeTable, ComputedColumnTable if TYPE_CHECKING: import sqlalchemy.orm as so from orm_loader.tables.typing import FileTableProtocol -_ComputedTableCls = cast("Type[FileTableProtocol]", ComputedColumnTable) -_CompositeTableCls = cast("Type[FileTableProtocol]", CompositeTable) +_ComputedTableCls = cast("type[FileTableProtocol]", ComputedColumnTable) +_CompositeTableCls = cast("type[FileTableProtocol]", CompositeTable) # merge_replace deletes on sqlite/postgres but updates in place on DuckDB # (see DuckDBBackend.merge_replace); these tests check the delete-only behavior. @@ -26,7 +26,7 @@ @pytest.fixture(params=DIALECT_PARAMS) -def merge_backend(request: pytest.FixtureRequest) -> tuple[DatabaseBackend, "so.Session"]: +def merge_backend(request: pytest.FixtureRequest) -> tuple[DatabaseBackend, so.Session]: """Same merge-method contract exercised against every real backend. DIALECT_PARAMS carries each dialect's own mark plus `forked` directly @@ -41,7 +41,7 @@ def merge_backend(request: pytest.FixtureRequest) -> tuple[DatabaseBackend, "so. @pytest.mark.parametrize("merge_backend", _REPLACE_DELETE_SEMANTICS_PARAMS, indirect=True) -def test_merge_replace_single_pk(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_replace_single_pk(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend staging = backend.create_staging_table(_ComputedTableCls, session) @@ -58,7 +58,7 @@ def test_merge_replace_single_pk(merge_backend: tuple[DatabaseBackend, "so.Sessi @pytest.mark.parametrize("merge_backend", _REPLACE_DELETE_SEMANTICS_PARAMS, indirect=True) -def test_merge_replace_composite_pk(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_replace_composite_pk(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend staging = backend.create_staging_table(_CompositeTableCls, session) @@ -74,7 +74,9 @@ def test_merge_replace_composite_pk(merge_backend: tuple[DatabaseBackend, "so.Se assert remaining == [(2, 2)] -def test_merge_insert_excludes_computed_columns(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_insert_excludes_computed_columns( + merge_backend: tuple[DatabaseBackend, so.Session], +) -> None: backend, session = merge_backend staging = backend.create_staging_table(_ComputedTableCls, session) session.execute(sa.insert(staging), [{"id": 1, "name": "alpha"}]) @@ -85,14 +87,14 @@ def test_merge_insert_excludes_computed_columns(merge_backend: tuple[DatabaseBac assert (row.id, row.name, row.slug) == (1, "alpha", "alpha") -def test_merge_upsert_excludes_computed_columns(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_upsert_excludes_computed_columns( + merge_backend: tuple[DatabaseBackend, so.Session], +) -> None: backend, session = merge_backend staging = backend.create_staging_table(_ComputedTableCls, session) session.execute(sa.insert(ComputedColumnTable), [{"id": 1, "name": "existing"}]) - session.execute( - sa.insert(staging), [{"id": 1, "name": "ignored"}, {"id": 2, "name": "new"}] - ) + session.execute(sa.insert(staging), [{"id": 1, "name": "ignored"}, {"id": 2, "name": "new"}]) backend.merge_upsert(_ComputedTableCls, session, ["id"], staging) @@ -105,9 +107,7 @@ def test_merge_upsert_excludes_computed_columns(merge_backend: tuple[DatabaseBac def test_postgres_merge_insert_paginates_rownums_with_initial_gap(pg_session) -> None: backend = resolve_backend(pg_session, staging_schema_tag=STAGING_SCHEMA) staging = backend.create_staging_table(_ComputedTableCls, pg_session) - pg_session.execute( - sa.insert(staging), [{"id": i, "name": f"row{i}"} for i in range(10)] - ) + pg_session.execute(sa.insert(staging), [{"id": i, "name": f"row{i}"} for i in range(10)]) pg_session.execute( sa.text( 'ALTER TABLE "staging"."_staging_computed_column_table" ' @@ -124,14 +124,16 @@ def test_postgres_merge_insert_paginates_rownums_with_initial_gap(pg_session) -> backend.merge_insert(_ComputedTableCls, pg_session, staging, merge_batch_size=2) - ids = pg_session.execute( - sa.select(ComputedColumnTable.id).order_by(ComputedColumnTable.id) - ).scalars().all() + ids = ( + pg_session.execute(sa.select(ComputedColumnTable.id).order_by(ComputedColumnTable.id)) + .scalars() + .all() + ) assert ids == list(range(10)) @pytest.mark.parametrize("merge_backend", _REPLACE_DELETE_SEMANTICS_PARAMS, indirect=True) -def test_merge_replace_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_replace_paginated_path(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend staging = backend.create_staging_table(_ComputedTableCls, session) @@ -148,7 +150,7 @@ def test_merge_replace_paginated_path(merge_backend: tuple[DatabaseBackend, "so. assert remaining == 0 -def test_merge_insert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_insert_paginated_path(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend staging = backend.create_staging_table(_ComputedTableCls, session) session.execute(sa.insert(staging), [{"id": i, "name": f"row{i}"} for i in range(10)]) @@ -159,13 +161,11 @@ def test_merge_insert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.S assert ids == list(range(10)) -def test_merge_upsert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_upsert_paginated_path(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend staging = backend.create_staging_table(_ComputedTableCls, session) - session.execute( - sa.insert(ComputedColumnTable), [{"id": i, "name": "kept"} for i in range(5)] - ) + session.execute(sa.insert(ComputedColumnTable), [{"id": i, "name": "kept"} for i in range(5)]) session.execute( sa.insert(staging), [{"id": i, "name": "should-not-overwrite"} for i in range(10)] ) @@ -173,4 +173,7 @@ def test_merge_upsert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.S backend.merge_upsert(_ComputedTableCls, session, ["id"], staging, merge_batch_size=3) rows = {r.id: r.name for r in session.execute(sa.select(ComputedColumnTable)).scalars().all()} - assert rows == {**{i: "kept" for i in range(5)}, **{i: "should-not-overwrite" for i in range(5, 10)}} + assert rows == { + **{i: "kept" for i in range(5)}, + **{i: "should-not-overwrite" for i in range(5, 10)}, + } diff --git a/tests/backends/test_sinks.py b/tests/backends/test_sinks.py index d807846..38de905 100644 --- a/tests/backends/test_sinks.py +++ b/tests/backends/test_sinks.py @@ -1,7 +1,7 @@ """The sink contract: every backend writes the same typed batches to staging the same way.""" from datetime import date -from typing import Type, cast +from typing import cast import pyarrow as pa import pytest @@ -12,7 +12,7 @@ from orm_loader.tables.typing import FileTableProtocol from tests.models import DemoRecord -_Demo = cast(Type[FileTableProtocol], DemoRecord) +_Demo = cast(type[FileTableProtocol], DemoRecord) COLUMNS = ["demo_id", "demo_label", "demo_number", "demo_flag", "demo_date"] @@ -20,18 +20,22 @@ def _sink(session, tmp_path): tag = None if session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA backend = resolve_backend(session, staging_schema_tag=tag, mapper=_Demo) staging = backend.create_staging_table(_Demo, session) - ctx = LoaderContext(tableclass=_Demo, session=session, path=tmp_path / "demo_record.csv", backend=backend) + ctx = LoaderContext( + tableclass=_Demo, session=session, path=tmp_path / "demo_record.csv", backend=backend + ) return backend, ctx, staging def _batch(i: int, label: str | None) -> pa.Table: - return pa.table({ - "demo_id": [i], - "demo_label": pa.array([label], pa.string()), - "demo_number": [i * 10], - "demo_flag": [i % 2 == 0], - "demo_date": pa.array([date(2020, 1, i)], pa.date32()), - }) + return pa.table( + { + "demo_id": [i], + "demo_label": pa.array([label], pa.string()), + "demo_number": [i * 10], + "demo_flag": [i % 2 == 0], + "demo_date": pa.array([date(2020, 1, i)], pa.date32()), + } + ) def _stored(session, staging): @@ -40,7 +44,9 @@ def _stored(session, staging): return session.execute(sa.select(staging).order_by(order)).all() -def test_typed_values_and_nulls_are_stored_across_more_batches_than_a_commit_interval(dialect_session, tmp_path): +def test_typed_values_and_nulls_are_stored_across_more_batches_than_a_commit_interval( + dialect_session, tmp_path +): backend, ctx, staging = _sink(dialect_session, tmp_path) batches = [_batch(i, f"row {i}" if i % 2 else None) for i in range(1, 13)] backend.insert_batches(ctx, staging, batches, COLUMNS) @@ -67,7 +73,9 @@ def test_no_batches_write_nothing(dialect_session, tmp_path): @pytest.mark.parametrize( "label", ["", "a,b", 'say "hi"', "line\nbreak", "tab\there", "ünïcode ✓", "NULL", "\\N"] ) -def test_text_is_stored_verbatim_and_an_empty_string_stays_distinct_from_null(dialect_session, tmp_path, label): +def test_text_is_stored_verbatim_and_an_empty_string_stays_distinct_from_null( + dialect_session, tmp_path, label +): backend, ctx, staging = _sink(dialect_session, tmp_path) backend.insert_batches(ctx, staging, [_batch(1, label), _batch(2, None)], COLUMNS) stored = _stored(dialect_session, staging) @@ -87,24 +95,30 @@ def batches(): def _staging_for(session, table): tag = None if session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA - created = resolve_backend(session, staging_schema_tag=tag, mapper=table).create_staging_table(table, session) - return sa.Table(created.name, sa.MetaData(), autoload_with=session.connection(), schema=created.schema) + created = resolve_backend(session, staging_schema_tag=tag, mapper=table).create_staging_table( + table, session + ) + return sa.Table( + created.name, sa.MetaData(), autoload_with=session.connection(), schema=created.schema + ) def test_a_staging_table_mirrors_the_columns_and_nullability_of_the_target(dialect_session): from tests.models import DemoRequiredRecord - staging = _staging_for(dialect_session, cast(Type[FileTableProtocol], DemoRequiredRecord)) + staging = _staging_for(dialect_session, cast(type[FileTableProtocol], DemoRequiredRecord)) target = DemoRequiredRecord.__table__ assert [c.name for c in staging.c if c.name != "_rownum"] == [c.name for c in target.c] - assert {c.name: c.nullable for c in staging.c if c.name != "_rownum"} == {c.name: c.nullable for c in target.c} + assert {c.name: c.nullable for c in staging.c if c.name != "_rownum"} == { + c.name: c.nullable for c in target.c + } assert not target.c.demo_label.nullable def test_a_staging_table_has_no_key_and_no_default(dialect_session): from tests.models import DemoDefaultedRecord - staging = _staging_for(dialect_session, cast(Type[FileTableProtocol], DemoDefaultedRecord)) + staging = _staging_for(dialect_session, cast(type[FileTableProtocol], DemoDefaultedRecord)) assert staging.primary_key.columns.keys() == [] assert all(c.server_default is None for c in staging.c if c.name != "_rownum") @@ -113,6 +127,10 @@ def test_a_staging_table_can_be_dropped_twice_and_created_again(dialect_session) staging = _staging_for(dialect_session, _Demo) tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA for _ in range(2): - resolve_backend(dialect_session, staging_schema_tag=tag, mapper=_Demo).drop_staging_table(_Demo, dialect_session) - assert not sa.inspect(dialect_session.connection()).has_table(staging.name, schema=staging.schema) + resolve_backend(dialect_session, staging_schema_tag=tag, mapper=_Demo).drop_staging_table( + _Demo, dialect_session + ) + assert not sa.inspect(dialect_session.connection()).has_table( + staging.name, schema=staging.schema + ) assert _staging_for(dialect_session, _Demo).name == staging.name diff --git a/tests/backends/test_sqlite_backend.py b/tests/backends/test_sqlite_backend.py index 5b4a477..6379fbe 100644 --- a/tests/backends/test_sqlite_backend.py +++ b/tests/backends/test_sqlite_backend.py @@ -1,9 +1,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Type, cast +from typing import TYPE_CHECKING, cast import pytest import sqlalchemy as sa +import sqlalchemy.event as sae import sqlalchemy.orm as so from sqlalchemy.dialects import sqlite as sqlite_dialect @@ -37,10 +38,12 @@ def __init__(self, value): def scalar(self): return self._value - return _Result(self.scalar_results.pop(0) if len(self.scalar_results) > 1 else self.scalar_results[0]) + return _Result( + self.scalar_results.pop(0) if len(self.scalar_results) > 1 else self.scalar_results[0] + ) -_ComputedTableCls = cast("Type[FileTableProtocol]", ComputedColumnTable) +_ComputedTableCls = cast("type[FileTableProtocol]", ComputedColumnTable) def _sess(s: _FakeSession) -> so.Session: @@ -80,7 +83,9 @@ def test_sqlite_backend_create_staging_table(sqlite_session, sqlite_engine): cols = inspector.get_columns(_STAGING_TABLE) assert [c["name"] for c in cols] == ["id", "name"] target = {c.name: c.nullable for c in ComputedColumnTable.__table__.columns} - assert {c["name"]: c["nullable"] for c in cols} == {name: target[name] for name in ("id", "name")} + assert {c["name"]: c["nullable"] for c in cols} == { + name: target[name] for name in ("id", "name") + } def test_sqlite_backend_disable_fk_reads_sets_then_reads_back(): @@ -182,14 +187,17 @@ def test_sqlite_backend_fk_toggle_round_trip(sqlite_session): previous = backend.disable_fk_check(sqlite_session) assert sqlite_session.execute(sa.text("PRAGMA foreign_keys")).scalar() == 0 + assert previous is not None backend.restore_fk_check(sqlite_session, previous) assert sqlite_session.execute(sa.text("PRAGMA foreign_keys")).scalar() == 1 def test_relaxed_foreign_keys_on_sqlite_logs_the_toggling(sqlite_session, caplog): - with caplog.at_level("INFO", logger="orm_loader.backends.base"): - with SQLiteBackend().relaxed_foreign_keys(sqlite_session): - pass + with ( + caplog.at_level("INFO", logger="orm_loader.backends.base"), + SQLiteBackend().relaxed_foreign_keys(sqlite_session), + ): + pass assert "Foreign key checks disabled" in caplog.text assert "Foreign key checks restored" in caplog.text @@ -198,7 +206,7 @@ def test_relaxed_foreign_keys_on_sqlite_logs_the_toggling(sqlite_session, caplog @pytest.fixture def enforcing_session(sqlite_engine): """A session whose connections come up with foreign key enforcement on.""" - sa.event.listen(sqlite_engine, "connect", lambda conn, _: conn.execute("PRAGMA foreign_keys = ON")) + sae.listen(sqlite_engine, "connect", lambda conn, _: conn.execute("PRAGMA foreign_keys = ON")) sqlite_engine.dispose() with so.Session(sqlite_engine) as session: yield session @@ -211,7 +219,7 @@ def _foreign_keys(session: so.Session) -> int: def test_a_load_relaxes_enforced_foreign_keys_and_restores_them(enforcing_session, tmp_path): path = tmp_path / "referencing_child_table.csv" path.write_text("id,parent_id\n1,1\n") - child = cast("Type[FileTableProtocol]", ReferencingChildTable) + child = cast("type[FileTableProtocol]", ReferencingChildTable) assert child.load_file(enforcing_session, path, staging_schema_tag=None) == 1 @@ -223,7 +231,7 @@ def test_a_load_on_a_session_with_pending_writes_raises_and_keeps_them(enforcing enforcing_session.flush() path = tmp_path / f"{SimpleTable.__tablename__}.csv" path.write_text("id,name\n2,a\n") - simple = cast("Type[FileTableProtocol]", SimpleTable) + simple = cast("type[FileTableProtocol]", SimpleTable) with pytest.raises(RuntimeError, match="inside a transaction"): simple.load_file(enforcing_session, path, staging_schema_tag=None) diff --git a/tests/conftest.py b/tests/conftest.py index 9a04436..1acd41b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -from collections.abc import Iterable, Iterator +from collections.abc import Generator, Iterable from contextlib import contextmanager from pathlib import Path @@ -6,9 +6,14 @@ import sqlalchemy as sa import sqlalchemy.orm as so from dotenv import load_dotenv - from oa_configurator import Dialect, Role -from oa_configurator.testing import DIALECT_PARAMS, ScopedTestSchema, isolated_test_database, scoped_test_schema +from oa_configurator.testing import ( + DIALECT_PARAMS, + ScopedTestSchema, + isolated_test_database, + scoped_test_schema, +) + from orm_loader.backends import staging_schema_claim from orm_loader.config import OrmLoaderConfig from tests.models import Base @@ -19,7 +24,9 @@ @pytest.fixture def sqlite_engine(): with isolated_test_database( - OrmLoaderConfig, "test_orm_db_sqlite", dialect="sqlite", + OrmLoaderConfig, + "test_orm_db_sqlite", + dialect="sqlite", ) as db: sqlite_engine = db.connection.engine Base.metadata.create_all(sqlite_engine) @@ -36,6 +43,7 @@ def sqlite_session(sqlite_engine): # Postgres fixtures # --------------------------------------------------------------------------- + @pytest.fixture def pg_db(request): """Isolated PostgreSQL test database. Everything done through @@ -43,7 +51,10 @@ def pg_db(request): that's rolled back on exit, so concurrent test runs can't collide and nothing needs manual cleanup.""" with isolated_test_database( - OrmLoaderConfig, "test_orm_db_pg", request=request, schema_claims=[staging_schema_claim()], + OrmLoaderConfig, + "test_orm_db_pg", + request=request, + schema_claims=[staging_schema_claim()], ) as db: yield db @@ -62,13 +73,17 @@ def pg_session(pg_db): # DuckDB fixtures # --------------------------------------------------------------------------- + @pytest.fixture def duckdb_db(request): """Isolated DuckDB test database, a fresh disposable tempfile per test. Schema-claimed the same way as pg_db: DuckDB genuinely supports schemas, unlike SQLite.""" with isolated_test_database( - OrmLoaderConfig, "test_orm_db_duckdb", dialect="duckdb", request=request, + OrmLoaderConfig, + "test_orm_db_duckdb", + dialect="duckdb", + request=request, schema_claims=[staging_schema_claim()], ) as db: yield db @@ -87,11 +102,14 @@ def duckdb_session(duckdb_db): @contextmanager def schema_scoped_session( pg_db, table: sa.Table, *, prefix: str, split_roles: Iterable[Role] = () -) -> Iterator[tuple[ScopedTestSchema, so.Session]]: +) -> Generator[tuple[ScopedTestSchema, so.Session]]: """A Session on committed scoped test schemas, with the staging schema claimed and *table* already created.""" with scoped_test_schema( - pg_db.resolved, prefix=prefix, split_roles=split_roles, schema_claims=[staging_schema_claim()] + pg_db.resolved, + prefix=prefix, + split_roles=split_roles, + schema_claims=[staging_schema_claim()], ) as scoped: with scoped.engine.begin() as conn: table.create(conn, checkfirst=True) diff --git a/tests/conversion/test_boolean_handler.py b/tests/conversion/test_boolean_handler.py index 4187f7e..97a4a20 100644 --- a/tests/conversion/test_boolean_handler.py +++ b/tests/conversion/test_boolean_handler.py @@ -6,8 +6,18 @@ @pytest.mark.parametrize( "raw, expected", - [("true", True), ("T", True), ("Yes", True), ("on", True), ("1", True), - ("false", False), ("f", False), ("NO", False), ("off", False), ("0", False)], + [ + ("true", True), + ("T", True), + ("Yes", True), + ("on", True), + ("1", True), + ("false", False), + ("f", False), + ("NO", False), + ("off", False), + ("0", False), + ], ) def test_boolean_words(raw, expected): assert read(raw, sa.Boolean()) is expected diff --git a/tests/conversion/test_date_handlers.py b/tests/conversion/test_date_handlers.py index 1678b9c..facf00b 100644 --- a/tests/conversion/test_date_handlers.py +++ b/tests/conversion/test_date_handlers.py @@ -1,4 +1,4 @@ -from datetime import date, datetime, timezone +from datetime import UTC, date, datetime import numpy as np import pyarrow as pa @@ -31,8 +31,19 @@ def test_both_athena_formats_and_iso_parse_by_default(raw, expected): @pytest.mark.parametrize( "raw", - ["20200231", "19000229", "2020022", "+2020229", "202002299", "abcdefgh", - "31-FEB-2020", "01-XXX-2020", "2019-02-29", "Jan 2 2020", "2020-13-45"], + [ + "20200231", + "19000229", + "2020022", + "+2020229", + "202002299", + "abcdefgh", + "31-FEB-2020", + "01-XXX-2020", + "2019-02-29", + "Jan 2 2020", + "2020-13-45", + ], ) def test_impossible_or_misshapen_dates_are_unreadable(raw): assert unreadable(raw, sa.Date()) @@ -40,8 +51,12 @@ def test_impossible_or_misshapen_dates_are_unreadable(raw): def test_ambiguous_date_needs_a_declared_format(): assert unreadable("02/01/2020", sa.Date()) - assert read("02/01/2020", sa.Date(), ConversionPolicy(date=DateHandler(formats=("dd/mm/yyyy",)))) == date(2020, 1, 2) - assert read("02/01/2020", sa.Date(), ConversionPolicy(date=DateHandler(formats=("mm/dd/yyyy",)))) == date(2020, 2, 1) + assert read( + "02/01/2020", sa.Date(), ConversionPolicy(date=DateHandler(formats=("dd/mm/yyyy",))) + ) == date(2020, 1, 2) + assert read( + "02/01/2020", sa.Date(), ConversionPolicy(date=DateHandler(formats=("mm/dd/yyyy",))) + ) == date(2020, 2, 1) def test_unknown_date_format_name_is_rejected(): @@ -52,7 +67,11 @@ def test_unknown_date_format_name_is_rejected(): def test_dictionary_encoded_dates_parse_the_same(): column = pc.dictionary_encode(pa.array(["20200229", None, "", "20200301", "20200229"])) assert interpret(column, sa.Date())[0] == [ - date(2020, 2, 29), None, None, date(2020, 3, 1), date(2020, 2, 29), + date(2020, 2, 29), + None, + None, + date(2020, 3, 1), + date(2020, 2, 29), ] @@ -70,7 +89,9 @@ def test_datetime_accepts_iso_and_date_only_forms(raw, expected): assert read(raw, sa.DateTime()) == expected -@pytest.mark.parametrize("raw", ["2017-02-30 01:00:00", "2017-08-24 25:00:00", "2017-08-24 something", "garbage"]) +@pytest.mark.parametrize( + "raw", ["2017-02-30 01:00:00", "2017-08-24 25:00:00", "2017-08-24 something", "garbage"] +) def test_datetime_rejects_invalid_values(raw): assert unreadable(raw, sa.DateTime()) @@ -87,7 +108,9 @@ def pad(values: np.ndarray, width: int) -> pa.Array: def test_every_calendar_date_from_year_1_to_9999_parses_to_itself(): - days = np.arange(np.datetime64("0001-01-01"), np.datetime64("10000-01-01"), dtype="datetime64[D]") + days = np.arange( + np.datetime64("0001-01-01"), np.datetime64("10000-01-01"), dtype="datetime64[D]" + ) dates, invalid = parse_dates(_yyyymmdd(days), ("yyyymmdd",)) assert not pc.any(invalid).as_py() assert dates.cast(pa.int32()).to_numpy().tolist() == days.astype(int).tolist() @@ -142,8 +165,8 @@ def test_year_zero_is_unreadable(sa_type): def test_a_timezone_column_requires_an_offset_and_stores_utc(): aware = sa.DateTime(timezone=True) - assert read("2020-01-01T10:00:00+10:00", aware) == datetime(2020, 1, 1, tzinfo=timezone.utc) - assert read("2020-01-01 10:00:00Z", aware) == datetime(2020, 1, 1, 10, tzinfo=timezone.utc) + assert read("2020-01-01T10:00:00+10:00", aware) == datetime(2020, 1, 1, tzinfo=UTC) + assert read("2020-01-01 10:00:00Z", aware) == datetime(2020, 1, 1, 10, tzinfo=UTC) assert unreadable("2020-01-01 10:00:00", aware) assert unreadable("20200101", aware) diff --git a/tests/conversion/test_number_handlers.py b/tests/conversion/test_number_handlers.py index 1f0386e..15b04af 100644 --- a/tests/conversion/test_number_handlers.py +++ b/tests/conversion/test_number_handlers.py @@ -27,7 +27,13 @@ def test_float_rejects_non_numbers(raw): def test_float_accepts_decimal_and_exponent_forms(): - assert [read(v, sa.Float()) for v in ("1.80", "2", ".5", "1e3", "+2.5")] == [1.8, 2.0, 0.5, 1000.0, 2.5] + assert [read(v, sa.Float()) for v in ("1.80", "2", ".5", "1e3", "+2.5")] == [ + 1.8, + 2.0, + 0.5, + 1000.0, + 2.5, + ] def test_numeric_keeps_declared_precision_and_rejects_overflow_of_scale(): @@ -75,7 +81,7 @@ def test_float_rejects_infinity_from_text_overflow(): def test_numeric_precision_without_scale_has_scale_zero(): - assert read("12345678901234567890", sa.Numeric(30)) == Decimal("12345678901234567890") + assert read("12345678901234567890", sa.Numeric(30)) == Decimal(12345678901234567890) assert unreadable("1.5", sa.Numeric(30)) @@ -90,21 +96,54 @@ def _reference_integer(raw: str, bits: int) -> int | None: sign = -1 if text.startswith("-") else 1 digits = text[1:] if text[:1] in ("+", "-") else text whole, _, fraction = digits.partition(".") - if not (whole.isascii() and whole.isdigit()) or (fraction and set(fraction) != {"0"}) or text.count(".") > 1: + if ( + not (whole.isascii() and whole.isdigit()) + or (fraction and set(fraction) != {"0"}) + or text.count(".") > 1 + ): return None value = sign * int(whole) return value if -(2 ** (bits - 1)) <= value < 2 ** (bits - 1) else None EDGE_INTEGERS = [ - "0", "-0", "+0", "007", "-007", "1.0", "1.", ".5", "5.", "1.50", "+7", "--1", "1e3", "", - "32767", "32768", "-32768", "-32769", "2147483647", "2147483648", "-2147483649", - "9223372036854775807", "9223372036854775808", "-9223372036854775808", "-9223372036854775809", - "0" * 30 + "12", "9" * 25, "12abc", " 7 ", "1_000", "١٢", + "0", + "-0", + "+0", + "007", + "-007", + "1.0", + "1.", + ".5", + "5.", + "1.50", + "+7", + "--1", + "1e3", + "", + "32767", + "32768", + "-32768", + "-32769", + "2147483647", + "2147483648", + "-2147483649", + "9223372036854775807", + "9223372036854775808", + "-9223372036854775808", + "-9223372036854775809", + "0" * 30 + "12", + "9" * 25, + "12abc", + " 7 ", + "1_000", + "١٢", ] -@pytest.mark.parametrize("sa_type, bits", [(sa.SmallInteger(), 16), (sa.Integer(), 32), (sa.BigInteger(), 64)]) +@pytest.mark.parametrize( + "sa_type, bits", [(sa.SmallInteger(), 16), (sa.Integer(), 32), (sa.BigInteger(), 64)] +) def test_integer_parsing_matches_a_per_element_reference(sa_type, bits): values, invalid = interpret(EDGE_INTEGERS, sa_type) for raw, value, bad in zip(EDGE_INTEGERS, values, invalid): @@ -119,7 +158,7 @@ def test_one_bad_value_in_a_large_batch_does_not_slow_the_whole_batch(): raw = pa.array([str(i) for i in range(200_000)]) raw = pa.concat_arrays([raw.slice(0, 100_000), pa.array(["9" * 25]), raw.slice(100_000)]) started = time.perf_counter() - values, invalid = interpret(raw, sa.BigInteger()) + _, invalid = interpret(raw, sa.BigInteger()) assert time.perf_counter() - started < 0.5 assert sum(invalid) == 1 and invalid[100_000] @@ -134,28 +173,64 @@ def test_numeric_fractions_longer_than_the_decimal_do_not_wrap_around(): def test_numeric_with_scale_zero_accepts_a_zero_fraction_only(): assert interpret(["2", "2.0", "2.5", "."], sa.Numeric(30)) == ( - [Decimal("2"), Decimal("2"), None, None], + [Decimal(2), Decimal(2), None, None], [False, False, True, True], ) -@pytest.mark.parametrize("raw, expected", [ - ("1e3", "1000.00"), ("1.5E3", "1500.00"), ("1.5E+003", "1500.00"), ("5e005", "500000.00"), ("2.50e1", "25.00"), ("1e-2", "0.01"), ("-2.5E+2", "-250.00"), - (".05", "0.05"), ("5.", "5.00"), (".0", "0.00"), ("+7", "7.00"), ("1.500", "1.50"), ("007.50", "7.50"), -]) +@pytest.mark.parametrize( + "raw, expected", + [ + ("1e3", "1000.00"), + ("1.5E3", "1500.00"), + ("1.5E+003", "1500.00"), + ("5e005", "500000.00"), + ("2.50e1", "25.00"), + ("1e-2", "0.01"), + ("-2.5E+2", "-250.00"), + (".05", "0.05"), + ("5.", "5.00"), + (".0", "0.00"), + ("+7", "7.00"), + ("1.500", "1.50"), + ("007.50", "7.50"), + ], +) def test_numeric_reads_plain_and_exponent_forms(raw, expected): assert read(raw, sa.Numeric(12, 2)) == Decimal(expected) -@pytest.mark.parametrize("raw", [ - ".", "+", "-", "+.", "-.", "e5", "1e", "1e+", "1e+-5", "--1", "1.2.3", "0x1F", "1,5", "abc", - "1.555", "1e-3", "1.5e-3", "1e10", "123456789012.00", -]) +@pytest.mark.parametrize( + "raw", + [ + ".", + "+", + "-", + "+.", + "-.", + "e5", + "1e", + "1e+", + "1e+-5", + "--1", + "1.2.3", + "0x1F", + "1,5", + "abc", + "1.555", + "1e-3", + "1.5e-3", + "1e10", + "123456789012.00", + ], +) def test_numeric_rejects_malformed_and_inexact_values(raw): assert unreadable(raw, sa.Numeric(12, 2)) -@pytest.mark.parametrize("raw", ["5e-50", "1e-40", "1.0e-39", "1" + "0" * 39, "9" * 39, "1e99999", "5e-9999"]) +@pytest.mark.parametrize( + "raw", ["5e-50", "1e-40", "1.0e-39", "1" + "0" * 39, "9" * 39, "1e99999", "5e-9999"] +) def test_numeric_values_beyond_what_arrow_parses_exactly_are_unreadable(raw): assert unreadable(raw, sa.Numeric(38, 2)) assert unreadable(raw, sa.Numeric(38, 0)) @@ -174,20 +249,33 @@ def test_numeric_fills_the_declared_precision_exactly(): def _reference(raw: str, precision: int, scale: int) -> Fraction | None: """Exact value of a decimal or exponent literal, or None when it does not fit the column.""" match = re.fullmatch(r"([+-]?)(\d*)\.?(\d*)(?:[eE]([+-]?\d+))?", raw) - if not match or not (match.group(2) or match.group(3)) or not re.fullmatch(r"[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?", raw): + if ( + not match + or not (match.group(2) or match.group(3)) + or not re.fullmatch(r"[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?", raw) + ): return None - sign, whole, fraction, exponent = match.group(1), match.group(2), match.group(3), int(match.group(4) or 0) + sign, whole, fraction, exponent = ( + match.group(1), + match.group(2), + match.group(3), + int(match.group(4) or 0), + ) value = Fraction(int((whole + fraction) or "0"), 10 ** len(fraction)) * Fraction(10) ** exponent - scaled = (-value if sign == "-" else value) * 10 ** scale - if scaled.denominator != 1 or abs(scaled) >= 10 ** precision: + scaled = (-value if sign == "-" else value) * 10**scale + if scaled.denominator != 1 or abs(scaled) >= 10**precision: return None - return Fraction(scaled.numerator, 10 ** scale) + return Fraction(scaled.numerator, 10**scale) @pytest.mark.parametrize("precision, scale", [(10, 2), (12, 0), (38, 2), (6, 6)]) def test_numeric_reads_every_string_exactly_or_refuses_it(precision, scale): """A string is read to its exact value or refused, and no string over this alphabet is refused though valid.""" - raws = ["".join(chars) for length in range(1, 6) for chars in itertools.product("+-.05e9", repeat=length)] + raws = [ + "".join(chars) + for length in range(1, 6) + for chars in itertools.product("+-.05e9", repeat=length) + ] raws += [ f"{mantissa}e{sign}{exponent}" for mantissa in ("1", "1.5", "9.99", ".5") @@ -205,7 +293,10 @@ def test_numeric_reads_every_string_exactly_or_refuses_it(precision, scale): assert refused_valid == [], refused_valid[:10] -@pytest.mark.parametrize("declared", [sa.Numeric(), sa.Numeric(39, 2), sa.Numeric(50, 2), sa.Numeric(0, 0), sa.Numeric(2, 5)]) +@pytest.mark.parametrize( + "declared", + [sa.Numeric(), sa.Numeric(39, 2), sa.Numeric(50, 2), sa.Numeric(0, 0), sa.Numeric(2, 5)], +) def test_numeric_declarations_an_arrow_decimal_cannot_hold_are_refused(declared): from orm_loader.conversion.handlers import NumericHandler @@ -213,7 +304,9 @@ def test_numeric_declarations_an_arrow_decimal_cannot_hold_are_refused(declared) NumericHandler().validate(declared) -@pytest.mark.parametrize("declared", [sa.Numeric(1, 0), sa.Numeric(38, 38), sa.Numeric(18), sa.Numeric(10, 2)]) +@pytest.mark.parametrize( + "declared", [sa.Numeric(1, 0), sa.Numeric(38, 38), sa.Numeric(18), sa.Numeric(10, 2)] +) def test_numeric_declarations_an_arrow_decimal_holds_are_accepted(declared): from orm_loader.conversion.handlers import NumericHandler diff --git a/tests/conversion/test_policy.py b/tests/conversion/test_policy.py index ccdf64b..211bb18 100644 --- a/tests/conversion/test_policy.py +++ b/tests/conversion/test_policy.py @@ -59,7 +59,10 @@ def test_a_type_without_a_handler_is_an_error_naming_the_column(): def test_passthrough_is_an_explicit_choice_for_a_column(): policy = ConversionPolicy(columns={("t", "c"): PassthroughHandler()}) raw = pa.array([b"a", None]) - assert interpret(raw, sa.LargeBinary(), policy, table_name="t", column_name="c")[0] == [b"a", None] + assert interpret(raw, sa.LargeBinary(), policy, table_name="t", column_name="c")[0] == [ + b"a", + None, + ] def test_a_type_decorator_is_handled_as_the_type_it_stores(): @@ -108,9 +111,9 @@ def test_column_handlers_win_over_type_handlers(): def test_the_handler_and_column_mappings_cannot_be_changed_afterwards(): policy = ConversionPolicy(columns={("t", "c"): IntegerHandler()}) with pytest.raises(TypeError): - policy.columns[("t", "d")] = IntegerHandler() # type: ignore[index] + policy.columns[("t", "d")] = IntegerHandler() # ty: ignore[invalid-assignment] def test_unknown_error_mode_is_rejected(): with pytest.raises(ValueError, match="on_error"): - ErrorPolicy(on_error="ignore") # type: ignore[arg-type] + ErrorPolicy(on_error="ignore") # ty: ignore[invalid-argument-type] diff --git a/tests/conversion/test_typed_input.py b/tests/conversion/test_typed_input.py index 8aca5b7..4a2d8a3 100644 --- a/tests/conversion/test_typed_input.py +++ b/tests/conversion/test_typed_input.py @@ -1,6 +1,6 @@ """Input that already has a type, such as a Parquet column, is cast and checked like parsed text.""" -from datetime import date, datetime, timezone +from datetime import UTC, date, datetime from decimal import Decimal import pyarrow as pa @@ -51,7 +51,9 @@ def test_float_nan_is_missing_and_infinity_is_unreadable(): def test_decimals_that_lose_digits_are_unreadable(): - values, invalid = _result(pa.array([Decimal("1.25"), Decimal("1.234")], pa.decimal128(10, 3)), sa.Numeric(10, 2)) + values, invalid = _result( + pa.array([Decimal("1.25"), Decimal("1.234")], pa.decimal128(10, 3)), sa.Numeric(10, 2) + ) assert values == [Decimal("1.25"), None] assert invalid == [False, True] @@ -63,9 +65,12 @@ def test_an_all_null_column_of_unknown_type_is_missing_values(): def test_timestamps_with_and_without_a_time_zone_match_their_column(): - aware = pa.array([datetime(2020, 1, 1, 10, tzinfo=timezone.utc)], pa.timestamp("us", "UTC")) + aware = pa.array([datetime(2020, 1, 1, 10, tzinfo=UTC)], pa.timestamp("us", "UTC")) naive = pa.array([datetime(2020, 1, 1, 10)], pa.timestamp("us")) - assert _result(aware, sa.DateTime(timezone=True)) == ([datetime(2020, 1, 1, 10, tzinfo=timezone.utc)], [False]) + assert _result(aware, sa.DateTime(timezone=True)) == ( + [datetime(2020, 1, 1, 10, tzinfo=UTC)], + [False], + ) assert _result(naive, sa.DateTime()) == ([datetime(2020, 1, 1, 10)], [False]) @@ -76,23 +81,29 @@ def test_year_zero_dates_are_unreadable_on_typed_input(): def test_the_time_zone_rule_of_the_column_applies_to_typed_timestamps(): - aware = pa.array([datetime(2020, 1, 1, 10, tzinfo=timezone.utc), None], pa.timestamp("us", "UTC")) + aware = pa.array([datetime(2020, 1, 1, 10, tzinfo=UTC), None], pa.timestamp("us", "UTC")) naive = pa.array([datetime(2020, 1, 1, 10), None], pa.timestamp("us")) assert _result(aware, sa.DateTime())[1] == [True, False] assert _result(naive, sa.DateTime(timezone=True))[1] == [True, False] - assert _result(pa.array([date(2020, 1, 1)], pa.date32()), sa.DateTime(timezone=True))[1] == [True] + assert _result(pa.array([date(2020, 1, 1)], pa.date32()), sa.DateTime(timezone=True))[1] == [ + True + ] assert _result(pa.array([date(2020, 1, 1)], pa.date32()), sa.DateTime())[1] == [False] @pytest.mark.parametrize("source", [pa.decimal128(10, 3), pa.decimal256(55, 3)]) def test_decimals_of_any_width_are_read_exactly_or_refused(source): - values, invalid = _result(pa.array([Decimal("1.250"), Decimal("1.555"), None], source), sa.Numeric(12, 2)) + values, invalid = _result( + pa.array([Decimal("1.250"), Decimal("1.555"), None], source), sa.Numeric(12, 2) + ) assert values == [Decimal("1.25"), None, None] assert invalid == [False, True, False] def test_floats_and_integers_in_a_numeric_column_must_fit_exactly(): - values, invalid = _result(pa.array([0.1, 1.555, float("inf"), float("nan"), 2.5]), sa.Numeric(12, 2)) + values, invalid = _result( + pa.array([0.1, 1.555, float("inf"), float("nan"), 2.5]), sa.Numeric(12, 2) + ) assert values == [Decimal("0.10"), None, None, None, Decimal("2.50")] assert invalid == [False, True, True, False, False] values, invalid = _result(pa.array([5, 10**11]), sa.Numeric(12, 2)) diff --git a/tests/helpers/test_bootstrap.py b/tests/helpers/test_bootstrap.py index c383411..d1a92be 100644 --- a/tests/helpers/test_bootstrap.py +++ b/tests/helpers/test_bootstrap.py @@ -2,7 +2,14 @@ import pytest import sqlalchemy as sa -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Resolver, SchemaClaim, StackConfig +from oa_configurator import ( + CDMDatabaseConfig, + ConnectionConfig, + ResolvedCDMDatabase, + Resolver, + SchemaClaim, + StackConfig, +) from orm_loader.helpers import bootstrap, create_db, create_tables @@ -86,7 +93,9 @@ def test_create_tables_crossing_subset_resolves_local_fk_to_existing_table(tmp_p parent = sa.Table( "local_parent", metadata, sa.Column("id", sa.Integer, primary_key=True), schema="primary" ) - sa.Table("remote_parent", metadata, sa.Column("id", sa.Integer, primary_key=True), schema="vocab") + sa.Table( + "remote_parent", metadata, sa.Column("id", sa.Integer, primary_key=True), schema="vocab" + ) child = sa.Table( "crossing_child", metadata, @@ -102,9 +111,9 @@ def test_create_tables_crossing_subset_resolves_local_fk_to_existing_table(tmp_p create_tables(connection, [child], resolved=resolved) assert sa.inspect(engine).has_table("crossing_child") - assert [fk["referred_table"] for fk in sa.inspect(engine).get_foreign_keys("crossing_child")] == [ - "local_parent" - ] + assert [ + fk["referred_table"] for fk in sa.inspect(engine).get_foreign_keys("crossing_child") + ] == ["local_parent"] engine.dispose() @@ -175,6 +184,7 @@ def test_create_tables_resolves_a_schema_claims_only_tag_on_duckdb(): }, ) resolved = Resolver(stack).resolve_database("test_cdm") + assert isinstance(resolved, ResolvedCDMDatabase) engine, _ = resolved.create_engines( schema_claims=[SchemaClaim(schema_tag="custom", physical_schema="cdm")] ) diff --git a/tests/loaders/test_dedupe.py b/tests/loaders/test_dedupe.py index 3887630..258aef2 100644 --- a/tests/loaders/test_dedupe.py +++ b/tests/loaders/test_dedupe.py @@ -1,4 +1,4 @@ -from typing import Type, cast +from typing import cast import pandas as pd import sqlalchemy as sa @@ -16,13 +16,13 @@ class Base(DeclarativeBase): class DedupTable(Base, FileLoadableTableInterface): __tablename__ = "dedup_table" - __table_args__ = {"schema": Role.PRIMARY.value} + __table_args__ = ({"schema": Role.PRIMARY.value},) id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) value: so.Mapped[str] = so.mapped_column(sa.String, nullable=False) -_DedupTable = cast(Type[FileTableProtocol], DedupTable) +_DedupTable = cast(type[FileTableProtocol], DedupTable) def test_internal_deduplication(sqlite_session, sqlite_engine, tmp_path): @@ -44,4 +44,4 @@ def test_internal_deduplication(sqlite_session, sqlite_engine, tmp_path): ) sqlite_session.commit() - assert inserted == 2 \ No newline at end of file + assert inserted == 2 diff --git a/tests/loaders/test_defaults.py b/tests/loaders/test_defaults.py index ce6c056..fa343f6 100644 --- a/tests/loaders/test_defaults.py +++ b/tests/loaders/test_defaults.py @@ -2,7 +2,7 @@ from datetime import date, datetime from pathlib import Path -from typing import Type, cast +from typing import cast import pytest import sqlalchemy as sa @@ -13,7 +13,7 @@ from orm_loader.tables.typing import FileTableProtocol from tests.models import DemoDefaultedRecord, DemoDefaultKinds -_Table = cast(Type[FileTableProtocol], DemoDefaultedRecord) +_Table = cast(type[FileTableProtocol], DemoDefaultedRecord) FULL = "demo_id,demo_status,demo_priority,demo_stamp,demo_note\n" @@ -29,7 +29,13 @@ def _load(session, tmp_path: Path, text: str, **kwargs) -> int: def _rows(session) -> list[tuple]: model = DemoDefaultedRecord - columns = (model.demo_id, model.demo_status, model.demo_priority, model.demo_stamp, model.demo_note) + columns = ( + model.demo_id, + model.demo_status, + model.demo_priority, + model.demo_stamp, + model.demo_note, + ) return [tuple(r) for r in session.execute(sa.select(*columns).order_by(model.demo_id))] @@ -65,8 +71,10 @@ def test_a_missing_callable_default_can_be_rejected_by_row(dialect_session, tmp_ def test_a_reload_applies_the_default_to_an_existing_row(dialect_session, tmp_path, strategy): _load(dialect_session, tmp_path, FULL + "1,closed,9,s,kept\n") _load(dialect_session, tmp_path, FULL + "1,,,t,\n2,,,u,\n", merge_strategy=strategy) - expected = {"replace": [(1, "open", 5, "t", "n"), (2, "open", 5, "u", "n")], - "upsert": [(1, "closed", 9, "s", "kept"), (2, "open", 5, "u", "n")]} + expected = { + "replace": [(1, "open", 5, "t", "n"), (2, "open", 5, "u", "n")], + "upsert": [(1, "closed", 9, "s", "kept"), (2, "open", 5, "u", "n")], + } assert _rows(dialect_session) == expected[strategy] @@ -74,7 +82,9 @@ def test_every_kind_of_default_is_applied_and_read_back_on_every_dialect(dialect path = tmp_path / "demo_default_kinds.csv" path.write_text("kind_id\n1\n2\n") tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA - cast(Type[FileTableProtocol], DemoDefaultKinds).load_file(dialect_session, path, staging_schema_tag=tag) + cast(type[FileTableProtocol], DemoDefaultKinds).load_file( + dialect_session, path, staging_schema_tag=tag + ) dialect_session.commit() dialect_session.expire_all() rows = dialect_session.execute(sa.select(DemoDefaultKinds)).scalars().all() @@ -82,7 +92,11 @@ def test_every_kind_of_default_is_applied_and_read_back_on_every_dialect(dialect for row in rows: assert (row.kind_text, row.kind_int, row.kind_bool) == ("it's text", 7, True) assert (row.kind_date, row.kind_stamp) == (date(2000, 1, 2), datetime(2000, 1, 2, 10, 30)) - assert (row.kind_expression, row.kind_py_int, row.kind_py_date) == ("expr", 5, date(2001, 2, 3)) + assert (row.kind_expression, row.kind_py_int, row.kind_py_date) == ( + "expr", + 5, + date(2001, 2, 3), + ) assert isinstance(row.kind_now, datetime) @@ -92,10 +106,14 @@ def _keyed(session, tmp_path: Path, text: str, **kwargs) -> list[tuple]: path = tmp_path / "demo_keyed_record.csv" path.write_text(text) tag = None if session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA - cast(Type[FileTableProtocol], DemoKeyedRecord).load_file(session, path, staging_schema_tag=tag, **kwargs) + cast(type[FileTableProtocol], DemoKeyedRecord).load_file( + session, path, staging_schema_tag=tag, **kwargs + ) session.commit() columns = (DemoKeyedRecord.keyed_id, DemoKeyedRecord.keyed_name) - return [tuple(r) for r in session.execute(sa.select(*columns).order_by(DemoKeyedRecord.keyed_id))] + return [ + tuple(r) for r in session.execute(sa.select(*columns).order_by(DemoKeyedRecord.keyed_id)) + ] def test_a_primary_key_the_file_lacks_is_a_missing_required_column(dialect_session, tmp_path): @@ -104,7 +122,10 @@ def test_a_primary_key_the_file_lacks_is_a_missing_required_column(dialect_sessi def test_a_key_in_the_file_is_loaded_and_may_not_be_blank(dialect_session, tmp_path): - assert _keyed(dialect_session, tmp_path, "keyed_id,keyed_name\n10,a\n20,b\n") == [(10, "a"), (20, "b")] + assert _keyed(dialect_session, tmp_path, "keyed_id,keyed_name\n10,a\n20,b\n") == [ + (10, "a"), + (20, "b"), + ] with pytest.raises(RequiredValueError, match="keyed_id"): _keyed(dialect_session, tmp_path, "keyed_id,keyed_name\n,a\n") @@ -114,18 +135,18 @@ def test_expression_defaults_fill_nulls_across_reader_batches(dialect_session, t path = tmp_path / "demo_default_kinds.csv" path.write_text( - "kind_id,kind_now\n1,2020-01-01 00:00:00\n" - + "".join(f"{i},\n" for i in range(2, 201)) + "kind_id,kind_now\n1,2020-01-01 00:00:00\n" + "".join(f"{i},\n" for i in range(2, 201)) ) tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA - table = cast(Type[FileTableProtocol], DemoDefaultKinds) + table = cast(type[FileTableProtocol], DemoDefaultKinds) reader = DelimitedTextReader(path, DemoDefaultKinds.model_columns(), block_size=256) assert len(list(reader.batches())) > 1 table.load_file(dialect_session, path, staging_schema_tag=tag, reader=reader) dialect_session.commit() rows = dialect_session.execute( - sa.select(DemoDefaultKinds.kind_id, DemoDefaultKinds.kind_now) - .order_by(DemoDefaultKinds.kind_id) + sa.select(DemoDefaultKinds.kind_id, DemoDefaultKinds.kind_now).order_by( + DemoDefaultKinds.kind_id + ) ).all() assert len(rows) == 200 assert rows[0].kind_now == datetime(2020, 1, 1) @@ -140,4 +161,6 @@ def test_a_default_the_handler_cannot_read_is_an_error_naming_the_column(dialect path.write_text("bad_id\n1\n") tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA with pytest.raises(ConversionError, match="bad_count"): - cast(Type[FileTableProtocol], DemoBadDefaultRecord).load_file(dialect_session, path, staging_schema_tag=tag) + cast(type[FileTableProtocol], DemoBadDefaultRecord).load_file( + dialect_session, path, staging_schema_tag=tag + ) diff --git a/tests/loaders/test_helpers.py b/tests/loaders/test_helpers.py index 615d23f..9049f4d 100644 --- a/tests/loaders/test_helpers.py +++ b/tests/loaders/test_helpers.py @@ -12,11 +12,13 @@ def test_infer_delim_csv(tmp_path): p.write_text("a,b,c\n1,2,3\n") assert infer_delim(p) == "," + def test_infer_delim_tsv(tmp_path): p = tmp_path / "x.tsv" p.write_text("a\tb\tc\n1\t2\t3\n") assert infer_delim(p) == "\t" + def test_infer_encoding_promotes_ascii_to_utf8(tmp_path): p = tmp_path / "ascii.csv" p.write_bytes(b"id,name\n1,alpha\n") @@ -34,6 +36,7 @@ def test_infer_encoding_utf8(tmp_path): # resolve_quote_mode # --------------------------------------------------------------------------- + def test_resolve_quote_mode_explicit_csv_passthrough(): # Explicit modes are returned unchanged and never read the file assert resolve_quote_mode("csv", ",") == "csv" diff --git a/tests/loaders/test_load_policy.py b/tests/loaders/test_load_policy.py index 715ed1d..8cd3d13 100644 --- a/tests/loaders/test_load_policy.py +++ b/tests/loaders/test_load_policy.py @@ -3,29 +3,30 @@ import os from datetime import date from pathlib import Path -from typing import Type, cast +from typing import cast import pyarrow as pa import pyarrow.parquet as pq import pytest import sqlalchemy as sa -from orm_loader.backends import STAGING_SCHEMA +from orm_loader.backends import STAGING_SCHEMA, DedupeResult from orm_loader.conversion import ConversionPolicy, DateHandler, ErrorPolicy, NullPolicy from orm_loader.helpers import ConversionError, IngestError, RejectLimitError, RequiredValueError -from orm_loader.backends import DedupeResult from orm_loader.loaders import LoadReport from orm_loader.tables.typing import FileTableProtocol from tests.models import DemoRecord, DemoRequiredRecord -_Demo = cast(Type[FileTableProtocol], DemoRecord) -_DemoRequired = cast(Type[FileTableProtocol], DemoRequiredRecord) +_Demo = cast(type[FileTableProtocol], DemoRecord) +_DemoRequired = cast(type[FileTableProtocol], DemoRequiredRecord) SEVEN_ROWS = 'demo_id,demo_label\n1,NULL\n2,"NULL"\n3,"""NULL"""\n4,\n5,""\n6,"alpha,beta"\n7,"alpha\nbeta"\n' MARKER = ConversionPolicy(nulls=NullPolicy(text_markers=frozenset({"NULL"}))) -def _load(dialect_session, tmp_path: Path, table, text: str, name: str | None = None, **kwargs) -> int: +def _load( + dialect_session, tmp_path: Path, table, text: str, name: str | None = None, **kwargs +) -> int: path = tmp_path / f"{name or table.__tablename__}.csv" path.write_text(text, encoding="utf-8", newline="") tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA @@ -36,14 +37,22 @@ def _load(dialect_session, tmp_path: Path, table, text: str, name: str | None = def _labels(dialect_session, model) -> dict[int, str | None]: - rows = dialect_session.execute(sa.select(model.demo_id, model.demo_label).order_by(model.demo_id)).all() + rows = dialect_session.execute( + sa.select(model.demo_id, model.demo_label).order_by(model.demo_id) + ).all() return {r[0]: r[1] for r in rows} def test_text_marker_is_missing_whether_or_not_quoted(dialect_session, tmp_path): _load(dialect_session, tmp_path, _Demo, SEVEN_ROWS, policy=MARKER) assert _labels(dialect_session, DemoRecord) == { - 1: None, 2: None, 3: '"NULL"', 4: None, 5: None, 6: "alpha,beta", 7: "alpha\nbeta", + 1: None, + 2: None, + 3: '"NULL"', + 4: None, + 5: None, + 6: "alpha,beta", + 7: "alpha\nbeta", } @@ -51,7 +60,12 @@ def test_text_marker_matches_exactly_and_ignores_padding(dialect_session, tmp_pa text = "demo_id,demo_label\n1,NULL\n2, NULL \n3,Null\n4,null\n5,NULLIFY\n6,Contains NULL here\n" _load(dialect_session, tmp_path, _Demo, text, policy=MARKER) assert _labels(dialect_session, DemoRecord) == { - 1: None, 2: None, 3: "Null", 4: "null", 5: "NULLIFY", 6: "Contains NULL here", + 1: None, + 2: None, + 3: "Null", + 4: "null", + 5: "NULLIFY", + 6: "Contains NULL here", } @@ -69,7 +83,9 @@ def test_empty_text_is_kept_when_the_policy_says_so(dialect_session, tmp_path, s tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA policy = ConversionPolicy(nulls=NullPolicy(empty_is_null=False)) _Demo.load_file(dialect_session, path, staging_schema_tag=tag, policy=policy) - rows = dialect_session.execute(sa.select(DemoRecord.demo_label, DemoRecord.demo_number).order_by(DemoRecord.demo_id)).all() + rows = dialect_session.execute( + sa.select(DemoRecord.demo_label, DemoRecord.demo_number).order_by(DemoRecord.demo_id) + ).all() assert [tuple(r) for r in rows] == [("", None), ("x", 5)] @@ -84,9 +100,15 @@ def test_markers_ignore_padding_in_a_tab_delimited_file(dialect_session, tmp_pat def test_both_athena_date_formats_and_iso_load_in_one_file(dialect_session, tmp_path): text = "demo_id,demo_date\n1,20200229\n2,01-JAN-2020\n3,2020-01-02\n4,\n5,NA\n" _load(dialect_session, tmp_path, _Demo, text) - rows = dialect_session.execute(sa.select(DemoRecord.demo_id, DemoRecord.demo_date).order_by(DemoRecord.demo_id)).all() + rows = dialect_session.execute( + sa.select(DemoRecord.demo_id, DemoRecord.demo_date).order_by(DemoRecord.demo_id) + ).all() assert [tuple(r) for r in rows] == [ - (1, date(2020, 2, 29)), (2, date(2020, 1, 1)), (3, date(2020, 1, 2)), (4, None), (5, None), + (1, date(2020, 2, 29)), + (2, date(2020, 1, 1)), + (3, date(2020, 1, 2)), + (4, None), + (5, None), ] @@ -94,7 +116,13 @@ def test_ambiguous_date_needs_a_declared_format(dialect_session, tmp_path): text = "demo_id,demo_date\n1,02/01/2020\n" with pytest.raises(ConversionError, match="DateHandler"): _load(dialect_session, tmp_path, _Demo, text) - _load(dialect_session, tmp_path, _Demo, text, policy=ConversionPolicy(date=DateHandler(formats=("dd/mm/yyyy",)))) + _load( + dialect_session, + tmp_path, + _Demo, + text, + policy=ConversionPolicy(date=DateHandler(formats=("dd/mm/yyyy",))), + ) assert dialect_session.execute(sa.select(DemoRecord.demo_date)).scalar_one() == date(2020, 1, 2) @@ -108,7 +136,8 @@ def test_ambiguous_date_needs_a_declared_format(dialect_session, tmp_path): ) -def _rejects(path: Path) -> list[dict]: +def _rejects(path: Path | None) -> list[dict]: + assert path is not None import pyarrow.csv as pv header = [h.strip('"') for h in path.read_text().splitlines()[0].split(",")] @@ -137,20 +166,34 @@ def test_reject_row_also_rejects_missing_required_values(dialect_session, tmp_pa report = LoadReport() text = "demo_id,demo_label\n1,ok\n2,\n3,NA\n" loaded = _load( - dialect_session, tmp_path, _DemoRequired, text, policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), report=report + dialect_session, + tmp_path, + _DemoRequired, + text, + policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), + report=report, ) assert loaded == 2 assert _labels(dialect_session, DemoRequiredRecord) == {1: "ok", 3: "NA"} [rejected] = _rejects(report.rejects_path) - assert (rejected["row"], rejected["column"], rejected["reason"]) == ("2", "demo_label", "required value is missing") + assert (rejected["row"], rejected["column"], rejected["reason"]) == ( + "2", + "demo_label", + "required value is missing", + ) def test_rejects_go_to_the_requested_directory(dialect_session, tmp_path): target = tmp_path / "elsewhere" report = LoadReport() _load( - dialect_session, tmp_path, _Demo, MESSY, - policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), rejects_dir=target, report=report, + dialect_session, + tmp_path, + _Demo, + MESSY, + policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), + rejects_dir=target, + report=report, ) assert report.rejects_path == target / "demo_record.rejects.csv" assert len(_rejects(report.rejects_path)) == 2 @@ -166,19 +209,30 @@ def test_unwritable_rejects_directory_fails_before_loading(dialect_session, tmp_ pytest.skip("directory permissions are not enforced for this user") with pytest.raises(IngestError, match="not writable"): _load( - dialect_session, tmp_path, _Demo, MESSY, - policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), rejects_dir=blocked / "inner", + dialect_session, + tmp_path, + _Demo, + MESSY, + policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), + rejects_dir=blocked / "inner", ) finally: blocked.chmod(0o700) - assert dialect_session.execute(sa.select(sa.func.count()).select_from(DemoRecord)).scalar_one() == 0 + assert ( + dialect_session.execute(sa.select(sa.func.count()).select_from(DemoRecord)).scalar_one() + == 0 + ) def test_clean_file_writes_no_rejects_file(dialect_session, tmp_path): report = LoadReport() _load( - dialect_session, tmp_path, _Demo, "demo_id,demo_label\n1,a\n", - policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), report=report, + dialect_session, + tmp_path, + _Demo, + "demo_id,demo_label\n1,a\n", + policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), + report=report, ) assert report.rejects_path is None assert not (tmp_path / "rejects").exists() @@ -200,13 +254,21 @@ def test_max_rejects_allows_a_load_within_the_limit(dialect_session, tmp_path, l def test_default_policy_raises_on_the_first_bad_batch(dialect_session, tmp_path): with pytest.raises(ConversionError, match="demo_number"): _load(dialect_session, tmp_path, _Demo, MESSY) - assert dialect_session.execute(sa.select(sa.func.count()).select_from(DemoRecord)).scalar_one() == 0 + assert ( + dialect_session.execute(sa.select(sa.func.count()).select_from(DemoRecord)).scalar_one() + == 0 + ) def test_report_counts_a_clean_load(dialect_session, tmp_path): report = LoadReport() _load(dialect_session, tmp_path, _Demo, "demo_id,demo_label\n1,a\n2,b\n", report=report) - assert (report.rows_read, report.rows_loaded, report.rows_rejected, report.rows_deduplicated) == (2, 2, 0, 0) + assert ( + report.rows_read, + report.rows_loaded, + report.rows_rejected, + report.rows_deduplicated, + ) == (2, 2, 0, 0) assert report.dedupe is None @@ -224,15 +286,29 @@ def test_dedupe_keeps_the_first_row_and_reports_what_it_removed(dialect_session, def test_dedupe_reports_zero_when_nothing_is_duplicated(dialect_session, tmp_path): report = LoadReport() - _load(dialect_session, tmp_path, _Demo, "demo_id,demo_label\n1,a\n2,b\n", dedupe=True, report=report) + _load( + dialect_session, + tmp_path, + _Demo, + "demo_id,demo_label\n1,a\n2,b\n", + dedupe=True, + report=report, + ) assert report.dedupe == DedupeResult(("demo_id",), rows_before=2, rows_removed=0) def test_dedupe_works_across_read_batches(sqlite_session, tmp_path): - rows = ["demo_id,demo_label"] + [f"{i},abcdefghijkl" for i in range(1, 450_001)] + ["1,again", "450000,again"] + rows = ( + ["demo_id,demo_label"] + + [f"{i},abcdefghijkl" for i in range(1, 450_001)] + + ["1,again", "450000,again"] + ) report = LoadReport() - loaded = _load(sqlite_session, tmp_path, _Demo, "\n".join(rows) + "\n", dedupe=True, report=report) + loaded = _load( + sqlite_session, tmp_path, _Demo, "\n".join(rows) + "\n", dedupe=True, report=report + ) assert loaded == 450_000 + assert report.dedupe is not None assert report.dedupe.rows_removed == 2 assert report.accounted labels = _labels(sqlite_session, DemoRecord) @@ -242,26 +318,42 @@ def test_dedupe_works_across_read_batches(sqlite_session, tmp_path): def test_dedupe_counts_rejected_rows_separately(dialect_session, tmp_path): report = LoadReport() text = "demo_id,demo_number\n1,1\n1,2\n2,x\n" - _load(dialect_session, tmp_path, _Demo, text, dedupe=True, report=report, policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row"))) - assert (report.rows_read, report.rows_rejected, report.rows_deduplicated, report.rows_loaded) == (3, 1, 1, 1) + _load( + dialect_session, + tmp_path, + _Demo, + text, + dedupe=True, + report=report, + policy=ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")), + ) + assert ( + report.rows_read, + report.rows_rejected, + report.rows_deduplicated, + report.rows_loaded, + ) == (3, 1, 1, 1) assert report.accounted def test_parquet_goes_through_the_same_policy(dialect_session, tmp_path): - table = pa.table({ - "demo_id": pa.array([1, 2, 3, 4], pa.int64()), - "demo_label": pa.array(["NULL", "x", None, ""], pa.string()), - "demo_number": pa.array([0, None, 3, 4], pa.int64()), - "demo_date": pa.array([date(2020, 2, 29), None, date(2020, 1, 1), None], pa.date32()), - }) + table = pa.table( + { + "demo_id": pa.array([1, 2, 3, 4], pa.int64()), + "demo_label": pa.array(["NULL", "x", None, ""], pa.string()), + "demo_number": pa.array([0, None, 3, 4], pa.int64()), + "demo_date": pa.array([date(2020, 2, 29), None, date(2020, 1, 1), None], pa.date32()), + } + ) path = tmp_path / "demo_record.parquet" pq.write_table(table, path) tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA _Demo.load_file(dialect_session, path, staging_schema_tag=tag, policy=MARKER) dialect_session.commit() rows = dialect_session.execute( - sa.select(DemoRecord.demo_id, DemoRecord.demo_label, DemoRecord.demo_number, DemoRecord.demo_date) - .order_by(DemoRecord.demo_id) + sa.select( + DemoRecord.demo_id, DemoRecord.demo_label, DemoRecord.demo_number, DemoRecord.demo_date + ).order_by(DemoRecord.demo_id) ).all() assert [tuple(r) for r in rows] == [ (1, None, 0, date(2020, 2, 29)), diff --git a/tests/loaders/test_loader_e2e.py b/tests/loaders/test_loader_e2e.py index 39f79fb..83593bf 100644 --- a/tests/loaders/test_loader_e2e.py +++ b/tests/loaders/test_loader_e2e.py @@ -1,10 +1,11 @@ -from typing import Type, cast - import logging +from typing import ClassVar, cast + import pandas as pd import pytest import sqlalchemy as sa import sqlalchemy.event as sae +import sqlalchemy.exc as sa_exc import sqlalchemy.orm as so from oa_configurator import Role as SchemaRole @@ -25,12 +26,12 @@ # Typed aliases: Pylance cannot verify SQLAlchemy metaclass-generated attrs # satisfy FileTableProtocol structurally, so we cast once per class here. -_SimpleTable = cast(Type[FileTableProtocol], SimpleTable) -_RequiredTable = cast(Type[FileTableProtocol], RequiredTable) -_CompositeTable = cast(Type[FileTableProtocol], CompositeTable) -_EnumTable = cast(Type[FileTableProtocol], EnumTable) -_VocabSchemaTable = cast(Type[FileTableProtocol], VocabSchemaTable) -_CycleATable = cast(Type[FileTableProtocol], CycleATable) +_SimpleTable = cast(type[FileTableProtocol], SimpleTable) +_RequiredTable = cast(type[FileTableProtocol], RequiredTable) +_CompositeTable = cast(type[FileTableProtocol], CompositeTable) +_EnumTable = cast(type[FileTableProtocol], EnumTable) +_VocabSchemaTable = cast(type[FileTableProtocol], VocabSchemaTable) +_CycleATable = cast(type[FileTableProtocol], CycleATable) def test_initial_file_load(sqlite_session, tmp_path): @@ -71,20 +72,20 @@ def test_initial_file_load_for_a_non_primary_schema_table(sqlite_session, tmp_pa schema-tag coverage.""" csv_path = tmp_path / "test_vocab_role_table.csv" - pd.DataFrame( - [{"id": 1, "name": "alpha"}, {"id": 2, "name": "beta"}] - ).to_csv(csv_path, index=False, sep="\t") - - inserted = _VocabSchemaTable.load_file( - sqlite_session, csv_path, dedupe=False + pd.DataFrame([{"id": 1, "name": "alpha"}, {"id": 2, "name": "beta"}]).to_csv( + csv_path, index=False, sep="\t" ) + + inserted = _VocabSchemaTable.load_file(sqlite_session, csv_path, dedupe=False) sqlite_session.commit() assert inserted == 2 - rows = sqlite_session.execute( - sa.select(VocabSchemaTable).order_by(VocabSchemaTable.id) - ).scalars().all() + rows = ( + sqlite_session.execute(sa.select(VocabSchemaTable).order_by(VocabSchemaTable.id)) + .scalars() + .all() + ) assert [(r.id, r.name) for r in rows] == [(1, "alpha"), (2, "beta")] @@ -229,7 +230,9 @@ def test_composite_pk_dedup(sqlite_session, tmp_path): ), ], ) -def test_merge_strategies(sqlite_session, tmp_path, merge_strategy, expected_rows, expected_inserted): +def test_merge_strategies( + sqlite_session, tmp_path, merge_strategy, expected_rows, expected_inserted +): csv_path = tmp_path / "test_table.csv" pd.DataFrame( @@ -270,11 +273,13 @@ def test_merge_strategies(sqlite_session, tmp_path, merge_strategy, expected_row def test_a_load_resolves_its_backend_once(sqlite_session, tmp_path, monkeypatch): - import orm_loader.tables.loadable_table as loadable_table + from orm_loader.tables import loadable_table calls = [] original = loadable_table.resolve_backend - monkeypatch.setattr(loadable_table, "resolve_backend", lambda *a, **k: calls.append(1) or original(*a, **k)) + monkeypatch.setattr( + loadable_table, "resolve_backend", lambda *a, **k: calls.append(1) or original(*a, **k) + ) csv_path = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv_path, index=False, sep="\t") @@ -311,7 +316,9 @@ def test_insert_merge_strategy(sqlite_session, tmp_path): ] -def test_insert_adds_new_keys_to_a_populated_target_and_fails_on_an_existing_key(sqlite_session, sqlite_engine, tmp_path): +def test_insert_adds_new_keys_to_a_populated_target_and_fails_on_an_existing_key( + sqlite_session, sqlite_engine, tmp_path +): csv_path = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv_path, index=False, sep="\t") @@ -321,11 +328,13 @@ def test_insert_adds_new_keys_to_a_populated_target_and_fails_on_an_existing_key pd.DataFrame([{"id": 2, "name": "beta"}]).to_csv(csv_path, index=False, sep="\t") _SimpleTable.load_file(sqlite_session, csv_path, dedupe=False, merge_strategy="insert") sqlite_session.commit() - rows = sqlite_session.execute(sa.select(SimpleTable.id, SimpleTable.name).order_by(SimpleTable.id)).all() + rows = sqlite_session.execute( + sa.select(SimpleTable.id, SimpleTable.name).order_by(SimpleTable.id) + ).all() assert [tuple(r) for r in rows] == [(1, "alpha"), (2, "beta")] pd.DataFrame([{"id": 2, "name": "again"}]).to_csv(csv_path, index=False, sep="\t") - with pytest.raises(sa.exc.IntegrityError): + with pytest.raises(sa_exc.IntegrityError): _SimpleTable.load_file(sqlite_session, csv_path, dedupe=False, merge_strategy="insert") inspector = sa.inspect(sqlite_engine) @@ -416,7 +425,9 @@ def test_composite_pk_replace_merge(sqlite_session, tmp_path): sqlite_session.commit() rows = ( - sqlite_session.execute(sa.select(CompositeTable).order_by(CompositeTable.a, CompositeTable.b)) + sqlite_session.execute( + sa.select(CompositeTable).order_by(CompositeTable.a, CompositeTable.b) + ) .scalars() .all() ) @@ -441,13 +452,13 @@ def test_filename_must_match_tablename(sqlite_session, tmp_path): def test_nullable_column_with_nan_does_not_crash(sqlite_session, sqlite_engine, tmp_path): class NullableTable(Base, FileLoadableTableInterface): __tablename__ = "nullable_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) flag: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) Base.metadata.create_all(sqlite_engine) - _NullableTable = cast(Type[FileTableProtocol], NullableTable) + _NullableTable = cast(type[FileTableProtocol], NullableTable) csv = tmp_path / "nullable_table.csv" pd.DataFrame( @@ -466,7 +477,9 @@ class NullableTable(Base, FileLoadableTableInterface): assert inserted == 2 - rows = sqlite_session.execute(sa.select(NullableTable).order_by(NullableTable.id)).scalars().all() + rows = ( + sqlite_session.execute(sa.select(NullableTable).order_by(NullableTable.id)).scalars().all() + ) assert [(r.id, r.flag) for r in rows] == [ (1, "S"), @@ -477,13 +490,13 @@ class NullableTable(Base, FileLoadableTableInterface): def test_embedded_newline_in_field_is_preserved(sqlite_session, sqlite_engine, tmp_path): class TextTable(Base, FileLoadableTableInterface): __tablename__ = "text_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String) Base.metadata.create_all(sqlite_engine) - _TextTable = cast(Type[FileTableProtocol], TextTable) + _TextTable = cast(type[FileTableProtocol], TextTable) csv = tmp_path / "text_table.csv" @@ -505,13 +518,13 @@ class TextTable(Base, FileLoadableTableInterface): def test_embedded_tab_in_field(sqlite_session, sqlite_engine, tmp_path): class TextTable2(Base, FileLoadableTableInterface): __tablename__ = "tab_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String) Base.metadata.create_all(sqlite_engine) - _TextTable2 = cast(Type[FileTableProtocol], TextTable2) + _TextTable2 = cast(type[FileTableProtocol], TextTable2) csv = tmp_path / "tab_table.csv" csv.write_text('id\tname\n1\t"foo\tbar"\n') @@ -538,7 +551,7 @@ def _make_ddl_tracker(sqlite_engine): def _capture(*args): statement: str = args[2] upper = statement.strip().upper() - if upper.startswith("DROP INDEX") or upper.startswith("CREATE INDEX"): + if upper.startswith(("DROP INDEX", "CREATE INDEX")): ddl_log.append(statement.strip()) sae.listen(sqlite_engine, "before_cursor_execute", _capture) @@ -596,27 +609,38 @@ def test_explicit_drop_rebuild_on_sqlite_restores_index(sqlite_session, sqlite_e assert "ix_test_table_name" in {idx["name"] for idx in inspector.get_indexes("test_table")} -def test_a_failed_index_rebuild_raises_after_the_merge_is_committed(sqlite_session, tmp_path, monkeypatch): +def test_a_failed_index_rebuild_raises_after_the_merge_is_committed( + sqlite_session, tmp_path, monkeypatch +): csv_path = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv_path, index=False, sep="\t") def failing_create_index(index): - raise sa.exc.OperationalError("CREATE INDEX", {}, Exception("disk full")) + raise sa_exc.OperationalError("CREATE INDEX", {}, Exception("disk full")) monkeypatch.setattr(sa.schema, "CreateIndex", failing_create_index) with pytest.raises(IngestError, match="ix_test_table_name"): _SimpleTable.load_file(sqlite_session, csv_path, index_strategy="drop_rebuild") - assert sqlite_session.execute(sa.select(sa.func.count()).select_from(SimpleTable)).scalar_one() == 1 + assert ( + sqlite_session.execute(sa.select(sa.func.count()).select_from(SimpleTable)).scalar_one() + == 1 + ) -def test_a_failed_merge_is_not_masked_by_a_failed_index_rebuild(sqlite_session, tmp_path, monkeypatch): +def test_a_failed_merge_is_not_masked_by_a_failed_index_rebuild( + sqlite_session, tmp_path, monkeypatch +): csv_path = tmp_path / "test_table.csv" - pd.DataFrame([{"id": 1, "name": "a"}, {"id": 1, "name": "b"}]).to_csv(csv_path, index=False, sep="\t") - monkeypatch.setattr(sa.schema, "CreateIndex", lambda index: (_ for _ in ()).throw(RuntimeError("no index"))) + pd.DataFrame([{"id": 1, "name": "a"}, {"id": 1, "name": "b"}]).to_csv( + csv_path, index=False, sep="\t" + ) + monkeypatch.setattr( + sa.schema, "CreateIndex", lambda index: (_ for _ in ()).throw(RuntimeError("no index")) + ) - with pytest.raises(sa.exc.IntegrityError): + with pytest.raises(sa_exc.IntegrityError): _SimpleTable.load_file(sqlite_session, csv_path, index_strategy="drop_rebuild") @@ -734,7 +758,9 @@ def test_invalid_index_strategy_raises(sqlite_session, tmp_path): def test_sqlite_failed_batched_replace_keeps_original_target_rows(sqlite_session, tmp_path): - sqlite_session.add_all([SimpleTable(id=1, name="original-1"), SimpleTable(id=2, name="original-2")]) + sqlite_session.add_all( + [SimpleTable(id=1, name="original-1"), SimpleTable(id=2, name="original-2")] + ) sqlite_session.commit() csv_path = tmp_path / "test_table.csv" pd.DataFrame( @@ -745,7 +771,7 @@ def test_sqlite_failed_batched_replace_keeps_original_target_rows(sqlite_session ] ).to_csv(csv_path, index=False, sep="\t") - with pytest.raises(sa.exc.IntegrityError): + with pytest.raises(sa_exc.IntegrityError): SimpleTable.load_file( sqlite_session, csv_path, @@ -761,7 +787,9 @@ def test_sqlite_failed_batched_replace_keeps_original_target_rows(sqlite_session @pytest.mark.duckdb @pytest.mark.db_dialect -def test_validate_deferred_fks_catches_an_orphan_introduced_by_load_file(duckdb_session, duckdb_db, tmp_path): +def test_validate_deferred_fks_catches_an_orphan_introduced_by_load_file( + duckdb_session, duckdb_db, tmp_path +): """cycle_a_table.b_id is part of a genuine FK cycle with cycle_b_table, so DuckDB never got a real constraint for it (oa-configurator's _skip_deferred_fk_on_duckdb no-oped the ALTER TABLE ADD CONSTRAINT): a @@ -771,7 +799,7 @@ def test_validate_deferred_fks_catches_an_orphan_introduced_by_load_file(duckdb_ Notes ----- Rollback on duckdb_session wipes out data that has already been committed. - Rollback-after-commit on this fixture's session is therefore unreliable for + Rollback-after-commit on this fixture's session is therefore unreliable for asserting target-row survival. """ csv_path = tmp_path / "cycle_a_table.csv" @@ -794,7 +822,9 @@ def test_validate_deferred_fks_catches_an_orphan_introduced_by_load_file(duckdb_ @pytest.mark.duckdb @pytest.mark.db_dialect -def test_load_file_rejects_replace_with_keep_index_strategy_on_duckdb_upfront(duckdb_session, tmp_path): +def test_load_file_rejects_replace_with_keep_index_strategy_on_duckdb_upfront( + duckdb_session, tmp_path +): """merge_strategy='replace' does an UPDATE in place on DuckDB, which breaks with a live index on an updated column. This combination must be rejected before the staging load even starts, not surface as DuckDB's @@ -802,7 +832,9 @@ def test_load_file_rejects_replace_with_keep_index_strategy_on_duckdb_upfront(du csv_path = tmp_path / f"{SimpleTable.__tablename__}.csv" pd.DataFrame([{"id": 1, "name": "a"}]).to_csv(csv_path, index=False, sep="\t") - with pytest.raises(NotImplementedError, match="merge_strategy='replace' with index_strategy='keep'"): + with pytest.raises( + NotImplementedError, match="merge_strategy='replace' with index_strategy='keep'" + ): SimpleTable.load_file( duckdb_session, csv_path, diff --git a/tests/loaders/test_malformed_files.py b/tests/loaders/test_malformed_files.py index e78b4c5..539bcd9 100644 --- a/tests/loaders/test_malformed_files.py +++ b/tests/loaders/test_malformed_files.py @@ -2,7 +2,7 @@ import logging from pathlib import Path -from typing import Type, cast +from typing import cast import pyarrow as pa import pyarrow.csv as pv @@ -17,8 +17,8 @@ from orm_loader.tables.typing import FileTableProtocol from tests.models import DemoBinaryRecord, DemoRecord, DemoRequiredRecord -_Demo = cast(Type[FileTableProtocol], DemoRecord) -_DemoRequired = cast(Type[FileTableProtocol], DemoRequiredRecord) +_Demo = cast(type[FileTableProtocol], DemoRecord) +_DemoRequired = cast(type[FileTableProtocol], DemoRequiredRecord) REJECT = ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")) COLUMNS = DemoRecord.model_columns() @@ -33,11 +33,16 @@ def _load(session, tmp_path: Path, text: str, **kwargs) -> int: def _ids(session) -> list[int]: - return [r[0] for r in session.execute(sa.select(DemoRecord.demo_id).order_by(DemoRecord.demo_id))] + return [ + r[0] for r in session.execute(sa.select(DemoRecord.demo_id).order_by(DemoRecord.demo_id)) + ] -def _rejects(path: Path) -> list[dict]: - options = pv.ConvertOptions(column_types={"row": pa.int64(), "column": pa.string(), "reason": pa.string()}) +def _rejects(path: Path | None) -> list[dict]: + assert path is not None + options = pv.ConvertOptions( + column_types={"row": pa.int64(), "column": pa.string(), "reason": pa.string()} + ) return pv.read_csv(path, convert_options=options).to_pylist() @@ -66,7 +71,10 @@ def test_row_numbers_count_the_malformed_lines_before_them(dialect_session, tmp_ report = LoadReport() text = "demo_id,demo_label,demo_number\n1,a,1\n2,b\n3,c,3\n4,d,x\n5,e,5\n" assert _load(dialect_session, tmp_path, text, policy=REJECT, report=report) == 3 - assert [(r["row"], r["column"]) for r in _rejects(report.rejects_path)] == [(2, ""), (4, "demo_number")] + assert [(r["row"], r["column"]) for r in _rejects(report.rejects_path)] == [ + (2, ""), + (4, "demo_number"), + ] def test_a_required_value_error_names_the_row_in_the_file(dialect_session, tmp_path): @@ -83,7 +91,10 @@ def test_a_required_value_error_names_the_row_in_the_file(dialect_session, tmp_p def test_a_file_of_only_ragged_lines_is_rejected_not_silently_empty(dialect_session, tmp_path): report = LoadReport() - assert _load(dialect_session, tmp_path, "demo_id,demo_label\n1\n2\n", policy=REJECT, report=report) == 0 + assert ( + _load(dialect_session, tmp_path, "demo_id,demo_label\n1\n2\n", policy=REJECT, report=report) + == 0 + ) assert report.rows_read == 2 and report.rows_rejected == 2 and report.accounted @@ -99,6 +110,7 @@ def test_undecodable_bytes_fail_with_the_file_name(tmp_path): def test_a_clean_reload_removes_the_rejects_file_of_the_previous_load(dialect_session, tmp_path): first = LoadReport() _load(dialect_session, tmp_path, RAGGED, policy=REJECT, report=first) + assert first.rejects_path is not None assert first.rejects_path.exists() second = LoadReport() _load(dialect_session, tmp_path, "demo_id,demo_label\n1,a\n", policy=REJECT, report=second) @@ -133,7 +145,11 @@ def test_parquet_column_names_are_normalised_like_delimited_headers(tmp_path): def test_a_hash_suffix_is_stripped_only_at_the_end_of_a_name(tmp_path): path = tmp_path / "demo_record.csv" path.write_text("demo_id,my_hash_value,demo_label_hash\n1,x,a\n") - assert DelimitedTextReader(path, COLUMNS).file_columns == ["demo_id", "my_hash_value", "demo_label"] + assert DelimitedTextReader(path, COLUMNS).file_columns == [ + "demo_id", + "my_hash_value", + "demo_label", + ] def test_duplicate_names_after_normalisation_are_an_error(tmp_path): @@ -153,11 +169,13 @@ def test_invalid_utf8_is_reported_without_the_quote_mode_hint(tmp_path): assert "quote_mode" not in str(error.value) -def test_a_column_type_without_a_handler_is_fine_when_values_are_not_interpreted(dialect_session, tmp_path): +def test_a_column_type_without_a_handler_is_fine_when_values_are_not_interpreted( + dialect_session, tmp_path +): path = tmp_path / "demo_binary_record.csv" path.write_text("demo_id,demo_blob\n1,abc\n") tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA - table = cast(Type[FileTableProtocol], DemoBinaryRecord) + table = cast(type[FileTableProtocol], DemoBinaryRecord) with pytest.raises(PolicyError, match="demo_blob"): table.load_file(dialect_session, path, staging_schema_tag=tag) assert table.load_file(dialect_session, path, staging_schema_tag=tag, normalise=False) == 1 diff --git a/tests/loaders/test_merge_semantics.py b/tests/loaders/test_merge_semantics.py index 111f25e..aab38bd 100644 --- a/tests/loaders/test_merge_semantics.py +++ b/tests/loaders/test_merge_semantics.py @@ -1,18 +1,19 @@ """A merge strategy gives the same result whether or not the target already has rows.""" from pathlib import Path -from typing import Type, cast +from typing import cast import pytest import sqlalchemy as sa - +import sqlalchemy.exc as sa_exc from oa_configurator import Dialect + from orm_loader.backends import STAGING_SCHEMA from orm_loader.tables.typing import FileTableProtocol from tests.models import CompositeTable, SimpleTable -_Simple = cast(Type[FileTableProtocol], SimpleTable) -_Composite = cast(Type[FileTableProtocol], CompositeTable) +_Simple = cast(type[FileTableProtocol], SimpleTable) +_Composite = cast(type[FileTableProtocol], CompositeTable) DUPLICATED = "id\tname\n1\tfirst\n1\tsecond\n2\tother\n" @@ -34,7 +35,12 @@ def _load(session, tmp_path: Path, text: str, options: dict, **extra) -> None: def _rows(session) -> list[tuple]: - return [tuple(r) for r in session.execute(sa.select(SimpleTable.id, SimpleTable.name).order_by(SimpleTable.id))] + return [ + tuple(r) + for r in session.execute( + sa.select(SimpleTable.id, SimpleTable.name).order_by(SimpleTable.id) + ) + ] def test_upsert_keeps_the_first_duplicate_in_an_empty_target(dialect_session, tmp_path, options): @@ -63,10 +69,14 @@ def test_replace_with_a_composite_key_overwrites_matching_rows(dialect_session, path = tmp_path / "composite_table.csv" for text in ("a\tb\tvalue\n1\t1\told\n1\t2\tkept\n", "a\tb\tvalue\n1\t1\tnew\n2\t2\tadded\n"): path.write_text(text) - _Composite.load_file(dialect_session, path, dedupe=False, merge_strategy="replace", **options) + _Composite.load_file( + dialect_session, path, dedupe=False, merge_strategy="replace", **options + ) dialect_session.commit() rows = dialect_session.execute( - sa.select(CompositeTable.a, CompositeTable.b, CompositeTable.value).order_by(CompositeTable.a, CompositeTable.b) + sa.select(CompositeTable.a, CompositeTable.b, CompositeTable.value).order_by( + CompositeTable.a, CompositeTable.b + ) ) assert [tuple(r) for r in rows] == [(1, 1, "new"), (1, 2, "kept"), (2, 2, "added")] @@ -75,5 +85,5 @@ def test_insert_fails_on_an_existing_key_and_adds_new_ones(dialect_session, tmp_ _load(dialect_session, tmp_path, "id\tname\n1\ta\n", options, merge_strategy="insert") _load(dialect_session, tmp_path, "id\tname\n2\tb\n", options, merge_strategy="insert") assert _rows(dialect_session) == [(1, "a"), (2, "b")] - with pytest.raises(sa.exc.IntegrityError): + with pytest.raises(sa_exc.IntegrityError): _load(dialect_session, tmp_path, "id\tname\n2\tagain\n", options, merge_strategy="insert") diff --git a/tests/loaders/test_null_handling.py b/tests/loaders/test_null_handling.py index c97d7f5..2f4601e 100644 --- a/tests/loaders/test_null_handling.py +++ b/tests/loaders/test_null_handling.py @@ -6,7 +6,7 @@ from datetime import date from pathlib import Path -from typing import Type, cast +from typing import cast import pyarrow as pa import pytest @@ -18,12 +18,23 @@ from tests.models import DemoRecord, DemoRequiredRecord from tests.support import interpret, read -_Demo = cast(Type[FileTableProtocol], DemoRecord) -_DemoRequired = cast(Type[FileTableProtocol], DemoRequiredRecord) +_Demo = cast(type[FileTableProtocol], DemoRecord) +_DemoRequired = cast(type[FileTableProtocol], DemoRequiredRecord) MARKER_WORDS = [ - "Null", "NULL", "null", "None", "none", "NA", "N/A", "NaN", "nan", - "#N/A", "", "Contains NULL here", '"NULL"', + "Null", + "NULL", + "null", + "None", + "none", + "NA", + "N/A", + "NaN", + "nan", + "#N/A", + "", + "Contains NULL here", + '"NULL"', ] SEVEN_ROWS = 'demo_id,demo_label\n1,NULL\n2,"NULL"\n3,"""NULL"""\n4,\n5,""\n6,"alpha,beta"\n7,"alpha\nbeta"\n' @@ -32,11 +43,17 @@ '"6","alpha,beta"\n"7","alpha\nbeta"\n' ) SEVEN_EXPECTED = { - 1: "NULL", 2: "NULL", 3: '"NULL"', 4: None, 5: None, 6: "alpha,beta", 7: "alpha\nbeta", + 1: "NULL", + 2: "NULL", + 3: '"NULL"', + 4: None, + 5: None, + 6: "alpha,beta", + 7: "alpha\nbeta", } -def _write(tmp_path: Path, table: type, text: str) -> Path: +def _write(tmp_path: Path, table: type[FileTableProtocol], text: str) -> Path: path = tmp_path / f"{table.__tablename__}.csv" path.write_text(text, encoding="utf-8", newline="") return path @@ -44,19 +61,24 @@ def _write(tmp_path: Path, table: type, text: str) -> Path: def _load(dialect_session, tmp_path: Path, table, text: str, **kwargs) -> int: tag = None if dialect_session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA - n = table.load_file(dialect_session, _write(tmp_path, table, text), staging_schema_tag=tag, **kwargs) + n = table.load_file( + dialect_session, _write(tmp_path, table, text), staging_schema_tag=tag, **kwargs + ) dialect_session.commit() dialect_session.expire_all() return n def _labels(dialect_session, model) -> dict[int, str | None]: - rows = dialect_session.execute(sa.select(model.demo_id, model.demo_label).order_by(model.demo_id)).all() + rows = dialect_session.execute( + sa.select(model.demo_id, model.demo_label).order_by(model.demo_id) + ).all() return {r[0]: r[1] for r in rows} # Marker-like text through each entry point + @pytest.mark.parametrize("word", MARKER_WORDS) def test_text_keeps_marker_words(word): out, _ = interpret([word, "plain"], DemoRecord.__table__.c.demo_label.type) @@ -65,6 +87,7 @@ def test_text_keeps_marker_words(word): # Actual missing objects + @pytest.mark.parametrize("value", [None, "", " "]) def test_missing_text_is_null(value): assert read(value, sa.String()) is None @@ -78,6 +101,7 @@ def test_arrow_null_bitmap_and_nan_are_null(): # Whitespace and identifiers + def test_text_is_stripped_by_default(): assert read(" x ", sa.String()) == "x" @@ -89,6 +113,7 @@ def test_text_is_never_canonicalised_as_a_number(code): # Typed columns: valid value, missing word, or error + @pytest.mark.parametrize("word", ["null", "None", "NA", "N/A", "NaN", "#N/A"]) def test_typed_missing_words_are_null(word): assert read(word, sa.Integer()) is None @@ -102,6 +127,7 @@ def test_typed_float_missing_words_are_null(word): # Persisted round trips + def test_seven_row_fixture_preserves_text(dialect_session, tmp_path): _load(dialect_session, tmp_path, _Demo, SEVEN_ROWS) assert _labels(dialect_session, DemoRecord) == SEVEN_EXPECTED @@ -121,7 +147,9 @@ def test_marker_words_survive_tab_delimited_load(dialect_session, tmp_path): def test_literal_quotes_survive_tab_delimited_load(dialect_session, tmp_path): values = ['"NULL"', '"word" label', "Null", "NA"] - text = "demo_id\tdemo_label\n" + "".join(f"{i}\t{v}\n" for i, v in enumerate(values, 1)) + "5\t\n" + text = ( + "demo_id\tdemo_label\n" + "".join(f"{i}\t{v}\n" for i, v in enumerate(values, 1)) + "5\t\n" + ) _load(dialect_session, tmp_path, _Demo, text, quote_mode="literal") assert _labels(dialect_session, DemoRecord) == {**dict(enumerate(values, 1)), 5: None} @@ -134,9 +162,11 @@ def test_typed_values_round_trip(dialect_session, tmp_path): "3,c,7,true,2020-01-02\n" ) _load(dialect_session, tmp_path, _Demo, text) - rows = dialect_session.execute(sa.select( - DemoRecord.demo_id, DemoRecord.demo_number, DemoRecord.demo_flag, DemoRecord.demo_date - ).order_by(DemoRecord.demo_id)).all() + rows = dialect_session.execute( + sa.select( + DemoRecord.demo_id, DemoRecord.demo_number, DemoRecord.demo_flag, DemoRecord.demo_date + ).order_by(DemoRecord.demo_id) + ).all() assert [tuple(r) for r in rows] == [ (1, 0, False, date(2020, 2, 29)), (2, None, None, None), @@ -161,11 +191,15 @@ def test_unreadable_typed_value_raises_and_loads_nothing(dialect_session, tmp_pa text = f"demo_id,demo_label,demo_number,demo_date\n1,a,,\n2,b,{cols['demo_number']},{cols['demo_date']}\n" with pytest.raises(IngestError, match=column): _load(dialect_session, tmp_path, _Demo, text) - assert dialect_session.execute(sa.select(sa.func.count()).select_from(DemoRecord)).scalar_one() == 0 + assert ( + dialect_session.execute(sa.select(sa.func.count()).select_from(DemoRecord)).scalar_one() + == 0 + ) # Required columns + def test_required_text_accepts_marker_words(dialect_session, tmp_path): words = ["Null", "None", "NA", "N/A", "NaN"] text = "demo_id,demo_label\n" + "".join(f"{i},{w}\n" for i, w in enumerate(words, 1)) @@ -178,11 +212,21 @@ def test_required_text_empty_raises_and_loads_nothing(dialect_session, tmp_path, text = f"demo_id,demo_label\n1,ok\n2,{missing}\n" with pytest.raises(IngestError, match="demo_label"): _load(dialect_session, tmp_path, _DemoRequired, text) - assert dialect_session.execute(sa.select(sa.func.count()).select_from(DemoRequiredRecord)).scalar_one() == 0 + assert ( + dialect_session.execute( + sa.select(sa.func.count()).select_from(DemoRequiredRecord) + ).scalar_one() + == 0 + ) def test_required_empty_raises_atomically_with_batching(dialect_session, tmp_path): text = "demo_id,demo_label\n1,a\n2,b\n3,\n" with pytest.raises(IngestError, match="demo_label"): _load(dialect_session, tmp_path, _DemoRequired, text, merge_batch_size=1) - assert dialect_session.execute(sa.select(sa.func.count()).select_from(DemoRequiredRecord)).scalar_one() == 0 + assert ( + dialect_session.execute( + sa.select(sa.func.count()).select_from(DemoRequiredRecord) + ).scalar_one() + == 0 + ) diff --git a/tests/loaders/test_numeric_columns.py b/tests/loaders/test_numeric_columns.py index 78a17af..90d7fbb 100644 --- a/tests/loaders/test_numeric_columns.py +++ b/tests/loaders/test_numeric_columns.py @@ -50,7 +50,9 @@ def _load(session, path: Path, model, **kwargs) -> int: model.__table__.create(session.connection(), checkfirst=True) session.commit() tag = None if session.get_bind().dialect.name == "sqlite" else STAGING_SCHEMA - loaded = cast(type[FileTableProtocol], model).load_file(session, path, staging_schema_tag=tag, **kwargs) + loaded = cast(type[FileTableProtocol], model).load_file( + session, path, staging_schema_tag=tag, **kwargs + ) session.commit() return loaded @@ -60,32 +62,71 @@ def _amounts(session, model=Amount) -> dict[int, Decimal | None]: return {key: None if value is None else Decimal(str(value)) for key, value in rows} -def _rejected(path: Path) -> list[tuple[str, str]]: +def _rejected(path: Path | None) -> list[tuple[str, str]]: + assert path is not None header = [h.strip('"') for h in path.read_text().splitlines()[0].split(",")] - table = pv.read_csv(path, convert_options=pv.ConvertOptions(column_types={name: pa.string() for name in header})) + table = pv.read_csv( + path, convert_options=pv.ConvertOptions(column_types={name: pa.string() for name in header}) + ) return [(row["row"], row["amount"]) for row in table.to_pylist()] def test_text_amounts_load_exactly_and_the_rest_is_rejected_with_its_row(dialect_session, tmp_path): - cells = ["1.50", " 2.5 ", ".05", "1.5E3", "1e-2", "007.50", "+7", "3.100", "5.", ".0", "1.5E+003", - ".", "abc", "1.555", "1e10", "5e-50", "0e+-0", "1" + "0" * 40] + cells = [ + "1.50", + " 2.5 ", + ".05", + "1.5E3", + "1e-2", + "007.50", + "+7", + "3.100", + "5.", + ".0", + "1.5E+003", + ".", + "abc", + "1.555", + "1e10", + "5e-50", + "0e+-0", + "1" + "0" * 40, + ] path = tmp_path / "numeric_amount.csv" - path.write_text("record_id,amount\n" + "".join(f"{i},{cell}\n" for i, cell in enumerate(cells, 1))) + path.write_text( + "record_id,amount\n" + "".join(f"{i},{cell}\n" for i, cell in enumerate(cells, 1)) + ) report = LoadReport() assert _load(dialect_session, path, Amount, policy=REJECT, report=report) == 11 assert _amounts(dialect_session) == { - 1: Decimal("1.50"), 2: Decimal("2.50"), 3: Decimal("0.05"), 4: Decimal("1500.00"), 5: Decimal("0.01"), - 6: Decimal("7.50"), 7: Decimal("7.00"), 8: Decimal("3.10"), 9: Decimal("5.00"), 10: Decimal("0.00"), + 1: Decimal("1.50"), + 2: Decimal("2.50"), + 3: Decimal("0.05"), + 4: Decimal("1500.00"), + 5: Decimal("0.01"), + 6: Decimal("7.50"), + 7: Decimal("7.00"), + 8: Decimal("3.10"), + 9: Decimal("5.00"), + 10: Decimal("0.00"), 11: Decimal("1500.00"), } assert (report.rows_read, report.rows_loaded, report.rows_rejected) == (18, 11, 7) assert report.accounted assert _rejected(report.rejects_path) == [ - ("12", "."), ("13", "abc"), ("14", "1.555"), ("15", "1e10"), ("16", "5e-50"), ("17", "0e+-0"), ("18", "1" + "0" * 40), + ("12", "."), + ("13", "abc"), + ("14", "1.555"), + ("15", "1e10"), + ("16", "5e-50"), + ("17", "0e+-0"), + ("18", "1" + "0" * 40), ] -def test_a_lone_point_raises_a_conversion_error_before_it_reaches_the_database(dialect_session, tmp_path): +def test_a_lone_point_raises_a_conversion_error_before_it_reaches_the_database( + dialect_session, tmp_path +): path = tmp_path / "numeric_amount.csv" path.write_text("record_id,amount\n1,.\n2,2.50\n") with pytest.raises(ConversionError, match="amount"): @@ -93,45 +134,79 @@ def test_a_lone_point_raises_a_conversion_error_before_it_reaches_the_database(d assert _amounts(dialect_session) == {} -@pytest.mark.parametrize("column, loaded, rejected", [ - (pa.array([5, 10**11, 7], pa.int64()), {1: "5.00", 3: "7.00"}, ["2"]), - (pa.array([5, 10**11], pa.uint64()), {1: "5.00"}, ["2"]), - (pa.array([Decimal("1.250"), Decimal("1.555"), Decimal("2.5")], pa.decimal256(55, 3)), {1: "1.25", 3: "2.50"}, ["2"]), - (pa.array([Decimal("1.25"), Decimal("1.5")], pa.decimal128(10, 2)), {1: "1.25", 2: "1.50"}, []), - (pa.array([708693.7, 0.1 + 0.2, 2.5, float("inf"), float("nan")]), {1: "708693.70", 3: "2.50", 5: None}, ["2", "4"]), - (pa.array(np.array([-549424.4, 0.1], np.float32)), {1: "-549424.40", 2: "0.10"}, []), -], ids=["int64", "uint64", "decimal256", "decimal128", "float64", "float32"]) -def test_parquet_columns_load_exactly_or_are_rejected(dialect_session, tmp_path, column, loaded, rejected): +@pytest.mark.parametrize( + "column, loaded, rejected", + [ + (pa.array([5, 10**11, 7], pa.int64()), {1: "5.00", 3: "7.00"}, ["2"]), + (pa.array([5, 10**11], pa.uint64()), {1: "5.00"}, ["2"]), + ( + pa.array([Decimal("1.250"), Decimal("1.555"), Decimal("2.5")], pa.decimal256(55, 3)), + {1: "1.25", 3: "2.50"}, + ["2"], + ), + ( + pa.array([Decimal("1.25"), Decimal("1.5")], pa.decimal128(10, 2)), + {1: "1.25", 2: "1.50"}, + [], + ), + ( + pa.array([708693.7, 0.1 + 0.2, 2.5, float("inf"), float("nan")]), + {1: "708693.70", 3: "2.50", 5: None}, + ["2", "4"], + ), + (pa.array(np.array([-549424.4, 0.1], np.float32)), {1: "-549424.40", 2: "0.10"}, []), + ], + ids=["int64", "uint64", "decimal256", "decimal128", "float64", "float32"], +) +def test_parquet_columns_load_exactly_or_are_rejected( + dialect_session, tmp_path, column, loaded, rejected +): path = tmp_path / "numeric_amount.parquet" - pq.write_table(pa.table({"record_id": pa.array(range(1, len(column) + 1), pa.int32()), "amount": column}), path) + pq.write_table( + pa.table({"record_id": pa.array(range(1, len(column) + 1), pa.int32()), "amount": column}), + path, + ) report = LoadReport() _load(dialect_session, path, Amount, policy=REJECT, report=report) - assert _amounts(dialect_session) == {key: None if value is None else Decimal(value) for key, value in loaded.items()} + assert _amounts(dialect_session) == { + key: None if value is None else Decimal(value) for key, value in loaded.items() + } assert report.rows_rejected == len(rejected) if rejected: assert [row for row, _ in _rejected(report.rejects_path)] == rejected assert report.accounted -@pytest.mark.parametrize("model, declared", [(WideAmount, r"\(50, 2\)"), (UnboundedAmount, r"\(None, None\)")]) +@pytest.mark.parametrize( + "model, declared", [(WideAmount, r"\(50, 2\)"), (UnboundedAmount, r"\(None, None\)")] +) def test_a_numeric_column_an_arrow_decimal_cannot_hold_is_refused_before_any_row_is_read( sqlite_session, tmp_path, model, declared ): path = tmp_path / f"{model.__tablename__}.csv" path.write_text("record_id,amount\n1,2.50\n") - with pytest.raises(PolicyError, match=rf"{model.__tablename__}.amount.*{declared}.*PassthroughHandler"): + with pytest.raises( + PolicyError, match=rf"{model.__tablename__}.amount.*{declared}.*PassthroughHandler" + ): _load(sqlite_session, path, model) assert sqlite_session.scalar(sa.select(sa.func.count()).select_from(model)) == 0 -@pytest.mark.parametrize("model, expected", [ - (WideAmount, ["1.50", "0.05", "1500.00", "1.56", "1" + "0" * 40 + ".25"]), - (UnboundedAmount, ["1.50", "0.05", "1500", "1.555", "1" + "0" * 40 + ".25"]), -]) -def test_a_column_the_loader_cannot_convert_loads_when_left_to_the_database(pg_session, tmp_path, model, expected): +@pytest.mark.parametrize( + "model, expected", + [ + (WideAmount, ["1.50", "0.05", "1500.00", "1.56", "1" + "0" * 40 + ".25"]), + (UnboundedAmount, ["1.50", "0.05", "1500", "1.555", "1" + "0" * 40 + ".25"]), + ], +) +def test_a_column_the_loader_cannot_convert_loads_when_left_to_the_database( + pg_session, tmp_path, model, expected +): cells = ["1.50", ".05", "1.5E3", "1.555", "1" + "0" * 40 + ".25"] path = tmp_path / f"{model.__tablename__}.csv" - path.write_text("record_id,amount\n" + "".join(f"{i},{cell}\n" for i, cell in enumerate(cells, 1))) + path.write_text( + "record_id,amount\n" + "".join(f"{i},{cell}\n" for i, cell in enumerate(cells, 1)) + ) policy = ConversionPolicy(columns={(model.__tablename__, "amount"): PassthroughHandler()}) assert _load(pg_session, path, model, policy=policy) == len(cells) assert list(_amounts(pg_session, model).values()) == [Decimal(value) for value in expected] diff --git a/tests/loaders/test_parquet_loader.py b/tests/loaders/test_parquet_loader.py index 6d172a9..c266ef5 100644 --- a/tests/loaders/test_parquet_loader.py +++ b/tests/loaders/test_parquet_loader.py @@ -1,11 +1,12 @@ +from typing import cast + +import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -import pandas as pd import sqlalchemy as sa import sqlalchemy.orm as so from oa_configurator import Role from sqlalchemy.orm import DeclarativeBase -from typing import cast, Type from orm_loader.tables.loadable_table import FileLoadableTableInterface from orm_loader.tables.typing import FileTableProtocol @@ -17,13 +18,13 @@ class Base(DeclarativeBase): class ParquetTable(Base, FileLoadableTableInterface): __tablename__ = "parquet_table" - __table_args__ = {"schema": Role.PRIMARY.value} + __table_args__ = ({"schema": Role.PRIMARY.value},) id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) value: so.Mapped[int] = so.mapped_column(sa.Integer, nullable=False) -_ParquetTable = cast(Type[FileTableProtocol], ParquetTable) +_ParquetTable = cast(type[FileTableProtocol], ParquetTable) def test_parquet_loader(sqlite_session, sqlite_engine, tmp_path): @@ -53,14 +54,14 @@ class NullSentinelTable(Base, FileLoadableTableInterface): """Nullable text columns, for checking that marker-like text survives a Parquet load.""" __tablename__ = "null_sentinel_table" - __table_args__ = {"schema": Role.PRIMARY.value} + __table_args__ = ({"schema": Role.PRIMARY.value},) id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) role: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) -_NullSentinelTable = cast(Type[FileTableProtocol], NullSentinelTable) +_NullSentinelTable = cast(type[FileTableProtocol], NullSentinelTable) _BLANKS = [" ", ""] _MARKER_TEXT = ["NULL", "null", "n/a", "N/A", "NaN", "none", "None"] @@ -78,6 +79,8 @@ def test_parquet_keeps_marker_text_and_nulls_blanks(sqlite_session, sqlite_engin _NullSentinelTable.load_file(sqlite_session, path, dedupe=False) sqlite_session.commit() - stored = dict(sqlite_session.execute(sa.select(NullSentinelTable.id, NullSentinelTable.name)).all()) + stored = dict( + sqlite_session.execute(sa.select(NullSentinelTable.id, NullSentinelTable.name)).all() + ) assert [stored[i] for i in range(1, len(values) + 1)] == _MARKER_TEXT + [None] * len(_BLANKS) assert stored[99] == "keep me" diff --git a/tests/loaders/test_pipeline.py b/tests/loaders/test_pipeline.py index b161470..82f3cb3 100644 --- a/tests/loaders/test_pipeline.py +++ b/tests/loaders/test_pipeline.py @@ -2,27 +2,32 @@ import csv import io -from datetime import date, datetime, timezone +from datetime import UTC, date, datetime from decimal import Decimal from pathlib import Path -from typing import Type, cast +from typing import cast import pyarrow as pa import pyarrow.parquet as pq import pytest import sqlalchemy as sa -from orm_loader.backends import resolve_backend -from orm_loader.backends import STAGING_SCHEMA +from orm_loader.backends import STAGING_SCHEMA, resolve_backend from orm_loader.conversion import ConversionPolicy, ErrorPolicy from orm_loader.helpers import ConversionError, RequiredValueError -from orm_loader.loaders import DelimitedTextReader, LoaderContext, ParquetReader, open_reader, stage_file +from orm_loader.loaders import ( + DelimitedTextReader, + LoaderContext, + ParquetReader, + open_reader, + stage_file, +) from orm_loader.tables.typing import FileTableProtocol from tests.models import DemoRecord, DemoRequiredRecord, DemoTypedRecord -_Demo = cast(Type[FileTableProtocol], DemoRecord) -_DemoRequired = cast(Type[FileTableProtocol], DemoRequiredRecord) -_Typed = cast(Type[FileTableProtocol], DemoTypedRecord) +_Demo = cast(type[FileTableProtocol], DemoRecord) +_DemoRequired = cast(type[FileTableProtocol], DemoRequiredRecord) +_Typed = cast(type[FileTableProtocol], DemoTypedRecord) HEADER = ["demo_id", "demo_label", "demo_number", "demo_flag", "demo_date"] REJECT = ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")) @@ -37,10 +42,17 @@ def _write(tmp_path: Path, table, fmt: str, rows: list[tuple], header: list[str] """Write the same rows as comma or tab delimited text, or as Parquet text columns.""" path = tmp_path / f"{table.__tablename__}.{'parquet' if fmt == 'parquet' else fmt}" if fmt == "parquet": - pq.write_table(pa.table({name: pa.array([r[i] for r in rows], pa.string()) for i, name in enumerate(header)}), path) + pq.write_table( + pa.table( + {name: pa.array([r[i] for r in rows], pa.string()) for i, name in enumerate(header)} + ), + path, + ) else: buffer = io.StringIO() - csv.writer(buffer, delimiter="," if fmt == "csv" else "\t", lineterminator="\n").writerows([header, *rows]) + csv.writer(buffer, delimiter="," if fmt == "csv" else "\t", lineterminator="\n").writerows( + [header, *rows] + ) path.write_text(buffer.getvalue(), encoding="utf-8", newline="") return path @@ -68,7 +80,10 @@ def _reflected(session, table) -> sa.Table: def _staged(session, staging, *columns: str) -> list[tuple]: order = staging.c._rownum if "_rownum" in staging.c else sa.literal_column("rowid") - return [tuple(r) for r in session.execute(sa.select(*(staging.c[c] for c in columns)).order_by(order)).all()] + return [ + tuple(r) + for r in session.execute(sa.select(*(staging.c[c] for c in columns)).order_by(order)).all() + ] def test_rows_are_converted_to_the_column_types(dialect_session, tmp_path, fmt): @@ -86,14 +101,22 @@ def test_marker_words_survive_and_only_blanks_are_null(dialect_session, tmp_path rows = [(str(i), label, "", "", "") for i, label in enumerate(labels, start=1)] _, _, staging = _stage(dialect_session, _Demo, _write(tmp_path, DemoRecord, fmt, rows)) assert [r[0] for r in _staged(dialect_session, staging, "demo_label")] == [ - "NULL", "None", "N/A", "NaN", None, None, "padded", + "NULL", + "None", + "N/A", + "NaN", + None, + None, + "padded", ] def test_missing_words_in_typed_columns_are_null(dialect_session, tmp_path, fmt): rows = [("1", "x", "NA", "null", "N/A")] _, _, staging = _stage(dialect_session, _Demo, _write(tmp_path, DemoRecord, fmt, rows)) - assert _staged(dialect_session, staging, "demo_number", "demo_flag", "demo_date") == [(None, None, None)] + assert _staged(dialect_session, staging, "demo_number", "demo_flag", "demo_date") == [ + (None, None, None) + ] def test_an_unreadable_value_raises_and_stages_nothing(dialect_session, tmp_path, fmt): @@ -105,15 +128,26 @@ def test_an_unreadable_value_raises_and_stages_nothing(dialect_session, tmp_path def test_a_missing_required_value_raises(dialect_session, tmp_path, fmt): - path = _write(tmp_path, DemoRequiredRecord, fmt, [("1", "ok"), ("2", "")], ["demo_id", "demo_label"]) + path = _write( + tmp_path, DemoRequiredRecord, fmt, [("1", "ok"), ("2", "")], ["demo_id", "demo_label"] + ) with pytest.raises(RequiredValueError): _stage(dialect_session, _DemoRequired, path) def test_reject_row_loads_the_rest_and_accounts_for_every_row(dialect_session, tmp_path, fmt): - rows = [("1", "a", "1", "", ""), ("2", "b", "abc", "", ""), ("3", "c", "3", "", "02/01/2020"), ("4", "d", "", "", "")] + rows = [ + ("1", "a", "1", "", ""), + ("2", "b", "abc", "", ""), + ("3", "c", "3", "", "02/01/2020"), + ("4", "d", "", "", ""), + ] loaded, ctx, staging = _stage( - dialect_session, _Demo, _write(tmp_path, DemoRecord, fmt, rows), policy=REJECT, rejects_dir=tmp_path / "rej" + dialect_session, + _Demo, + _write(tmp_path, DemoRecord, fmt, rows), + policy=REJECT, + rejects_dir=tmp_path / "rej", ) assert loaded == 2 assert (ctx.report.rows_read, ctx.report.rows_loaded, ctx.report.rows_rejected) == (4, 2, 2) @@ -124,22 +158,31 @@ def test_reject_row_loads_the_rest_and_accounts_for_every_row(dialect_session, t def test_dedupe_keeps_the_first_row_for_each_key(dialect_session, tmp_path, fmt): rows = [("1", "first", "", "", ""), ("2", "x", "", "", ""), ("1", "second", "", "", "")] - loaded, ctx, staging = _stage(dialect_session, _Demo, _write(tmp_path, DemoRecord, fmt, rows), dedupe=True) + loaded, ctx, staging = _stage( + dialect_session, _Demo, _write(tmp_path, DemoRecord, fmt, rows), dedupe=True + ) assert loaded == 2 and ctx.report.rows_deduplicated == 1 and ctx.report.accounted - assert sorted(_staged(dialect_session, staging, "demo_id", "demo_label")) == [(1, "first"), (2, "x")] + assert sorted(_staged(dialect_session, staging, "demo_id", "demo_label")) == [ + (1, "first"), + (2, "x"), + ] def test_a_file_in_many_batches_loads_every_row_in_order(dialect_session, tmp_path, fmt): rows = [(str(i), f"label {i}", str(i), "", "") for i in range(1, 1501)] path = _write(tmp_path, DemoRecord, fmt, rows) columns = DemoRecord.model_columns() - reader = ParquetReader(path, columns, chunksize=100) if fmt == "parquet" else DelimitedTextReader(path, columns, block_size=1 << 10) + reader = ( + ParquetReader(path, columns, chunksize=100) + if fmt == "parquet" + else DelimitedTextReader(path, columns, block_size=1 << 10) + ) loaded, ctx, staging = _stage(dialect_session, _Demo, path, reader=reader) assert loaded == 1500 and ctx.report.accounted assert [r[0] for r in _staged(dialect_session, staging, "demo_id")] == list(range(1, 1501)) -def test_a_failing_reader_is_raised_and_stages_nothing(dialect_session, tmp_path, fmt): +def test_a_failing_reader_is_raised_and_stages_nothing(dialect_session, tmp_path, fmt, monkeypatch): path = _write(tmp_path, DemoRecord, fmt, [("1", "a", "", "", "")]) reader = open_reader(path, _Demo.model_columns()) @@ -147,17 +190,23 @@ def broken(): raise RuntimeError("boom") yield # pragma: no cover - reader.batches = broken # type: ignore[method-assign] + monkeypatch.setattr(reader, "batches", broken) with pytest.raises(RuntimeError, match="boom"): _stage(dialect_session, _Demo, path, reader=reader) assert _staged(dialect_session, _reflected(dialect_session, _Demo), "demo_id") == [] TYPED_HEADER = [ - "typed_id", "typed_small", "typed_float", "typed_numeric", "typed_flag", - "typed_date", "typed_stamp", "typed_stamp_utc", "typed_label", + "typed_id", + "typed_small", + "typed_float", + "typed_numeric", + "typed_flag", + "typed_date", + "typed_stamp", + "typed_stamp_utc", + "typed_label", ] -UTC = timezone.utc def _typed_file(tmp_path: Path, **overrides: pa.Array) -> Path: @@ -169,7 +218,9 @@ def _typed_file(tmp_path: Path, **overrides: pa.Array) -> Path: "typed_flag": pa.array([True, None], pa.bool_()), "typed_date": pa.array([date(2020, 2, 29), None], pa.date32()), "typed_stamp": pa.array([datetime(2020, 1, 1, 10, 30), None], pa.timestamp("us")), - "typed_stamp_utc": pa.array([datetime(2020, 1, 1, 10, 30, tzinfo=UTC), None], pa.timestamp("us", "UTC")), + "typed_stamp_utc": pa.array( + [datetime(2020, 1, 1, 10, 30, tzinfo=UTC), None], pa.timestamp("us", "UTC") + ), "typed_label": pa.array(["alpha", None], pa.string()), **overrides, } @@ -183,8 +234,15 @@ def test_typed_parquet_columns_keep_their_values_on_every_dialect(dialect_sessio assert loaded == 2 and ctx.report.accounted rows = _staged(dialect_session, staging, *TYPED_HEADER) assert rows[0] == ( - 1, 7, 1.5, Decimal("12.34"), True, date(2020, 2, 29), - datetime(2020, 1, 1, 10, 30), datetime(2020, 1, 1, 10, 30, tzinfo=UTC), "alpha", + 1, + 7, + 1.5, + Decimal("12.34"), + True, + date(2020, 2, 29), + datetime(2020, 1, 1, 10, 30), + datetime(2020, 1, 1, 10, 30, tzinfo=UTC), + "alpha", ) assert rows[1] == (2, None, None, None, None, None, None, None, None) @@ -193,16 +251,23 @@ def test_typed_parquet_columns_keep_their_values_on_every_dialect(dialect_sessio "column, bad", [ ("typed_flag", pa.array([1, 2], pa.int64())), - ("typed_date", pa.array([datetime(2020, 1, 1), datetime(2020, 1, 1, 12)], pa.timestamp("us"))), + ( + "typed_date", + pa.array([datetime(2020, 1, 1), datetime(2020, 1, 1, 12)], pa.timestamp("us")), + ), ("typed_small", pa.array([1, 70_000], pa.int64())), ("typed_label", pa.array([1, 12345678901], pa.int64())), ("typed_numeric", pa.array([Decimal("1.25"), Decimal("1.234")], pa.decimal128(10, 3))), ], ) -def test_typed_parquet_values_that_would_change_are_unreadable(dialect_session, tmp_path, column, bad): +def test_typed_parquet_values_that_would_change_are_unreadable( + dialect_session, tmp_path, column, bad +): with pytest.raises(ConversionError, match=column): _stage(dialect_session, _Typed, _typed_file(tmp_path, **{column: bad})) - loaded, ctx, _ = _stage(dialect_session, _Typed, _typed_file(tmp_path, **{column: bad}), policy=REJECT) + loaded, ctx, _ = _stage( + dialect_session, _Typed, _typed_file(tmp_path, **{column: bad}), policy=REJECT + ) assert loaded == 1 and ctx.report.rows_rejected == 1 and ctx.report.accounted @@ -210,9 +275,15 @@ def _typed_rows(**cells: str) -> list[tuple]: return [tuple(cells.get(name, "1" if name == "typed_id" else "") for name in TYPED_HEADER)] -def test_offsets_are_stored_as_utc_in_a_time_zone_column_and_rejected_in_a_naive_one(dialect_session, tmp_path): - rows = _typed_rows(typed_stamp_utc="2020-01-01T10:00:00+10:00", typed_stamp="2020-01-01 10:00:00") - _, _, staging = _stage(dialect_session, _Typed, _write(tmp_path, DemoTypedRecord, "csv", rows, TYPED_HEADER)) +def test_offsets_are_stored_as_utc_in_a_time_zone_column_and_rejected_in_a_naive_one( + dialect_session, tmp_path +): + rows = _typed_rows( + typed_stamp_utc="2020-01-01T10:00:00+10:00", typed_stamp="2020-01-01 10:00:00" + ) + _, _, staging = _stage( + dialect_session, _Typed, _write(tmp_path, DemoTypedRecord, "csv", rows, TYPED_HEADER) + ) assert _staged(dialect_session, staging, "typed_stamp", "typed_stamp_utc") == [ (datetime(2020, 1, 1, 10), datetime(2020, 1, 1, 0, tzinfo=UTC)) ] @@ -225,7 +296,9 @@ def test_offsets_are_stored_as_utc_in_a_time_zone_column_and_rejected_in_a_naive @pytest.mark.parametrize("policy", [None, REJECT], ids=["raise", "reject_row"]) -def test_an_integer_too_wide_for_the_column_never_reaches_the_database(dialect_session, tmp_path, policy): +def test_an_integer_too_wide_for_the_column_never_reaches_the_database( + dialect_session, tmp_path, policy +): rows = [*_typed_rows(typed_small="32767"), *_typed_rows(typed_id="2", typed_small="32768")] path = _write(tmp_path, DemoTypedRecord, "csv", rows, TYPED_HEADER) if policy is None: @@ -240,4 +313,6 @@ def test_an_integer_too_wide_for_the_column_never_reaches_the_database(dialect_s def test_year_zero_is_rejected_on_every_dialect(dialect_session, tmp_path): rows = _typed_rows(typed_date="0000-01-01", typed_stamp="0000-01-01 00:00:00") with pytest.raises(ConversionError, match="typed_date"): - _stage(dialect_session, _Typed, _write(tmp_path, DemoTypedRecord, "csv", rows, TYPED_HEADER)) + _stage( + dialect_session, _Typed, _write(tmp_path, DemoTypedRecord, "csv", rows, TYPED_HEADER) + ) diff --git a/tests/loaders/test_processor.py b/tests/loaders/test_processor.py index 74fc878..eabe263 100644 --- a/tests/loaders/test_processor.py +++ b/tests/loaders/test_processor.py @@ -4,6 +4,7 @@ import logging from datetime import date from pathlib import Path +from typing import cast import pyarrow as pa import pytest @@ -12,8 +13,11 @@ from orm_loader.helpers import ConversionError, IngestError, RejectLimitError, RequiredValueError from orm_loader.loaders import LoadReport from orm_loader.loaders.processor import BatchProcessor +from orm_loader.tables.typing import FileTableProtocol from tests.models import DemoRecord, DemoRequiredRecord +_DemoRecord = cast(type[FileTableProtocol], DemoRecord) + COLUMNS = ["demo_id", "demo_label", "demo_number", "demo_date"] REJECT = ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")) @@ -22,9 +26,13 @@ def _batch(*rows: tuple, columns: list[str] = COLUMNS) -> pa.RecordBatch: return pa.RecordBatch.from_pylist([dict(zip(columns, row)) for row in rows]) -def _processor(tmp_path: Path, policy: ConversionPolicy | None = None, table=DemoRecord, columns=COLUMNS, **kw): +def _processor( + tmp_path: Path, policy: ConversionPolicy | None = None, table=_DemoRecord, columns=COLUMNS, **kw +): report = LoadReport() - return BatchProcessor(table, columns, policy or ConversionPolicy(), report, tmp_path / "rejects.csv", **kw), report + return BatchProcessor( + table, columns, policy or ConversionPolicy(), report, tmp_path / "rejects.csv", **kw + ), report def _rejects(tmp_path: Path) -> list[dict]: @@ -51,7 +59,9 @@ def test_without_normalise_the_batch_passes_through_as_read(tmp_path): def test_unreadable_values_raise_with_the_count_and_first_three_samples(tmp_path): processor, _ = _processor(tmp_path) - batch = _batch(*[(str(i), "a", bad, "") for i, bad in enumerate(["1", "a", "b", "c", "d", "2"])]) + batch = _batch( + *[(str(i), "a", bad, "") for i, bad in enumerate(["1", "a", "b", "c", "d", "2"])] + ) with pytest.raises(ConversionError) as exc: processor.process(batch) assert (exc.value.count, exc.value.samples) == (4, ["a", "b", "c"]) @@ -64,18 +74,24 @@ def test_an_unreadable_date_names_the_option_that_declares_a_format(tmp_path): processor.process(_batch(("1", "a", "", "02/01/2020"))) declared = ConversionPolicy(date=DateHandler(formats=("dd/mm/yyyy",))) processor, _ = _processor(tmp_path, declared) - assert processor.process(_batch(("1", "a", "", "02/01/2020"))).column("demo_date").to_pylist() == [date(2020, 1, 2)] + assert processor.process(_batch(("1", "a", "", "02/01/2020"))).column( + "demo_date" + ).to_pylist() == [date(2020, 1, 2)] def test_a_missing_required_value_raises_with_the_count_and_row_numbers(tmp_path): processor, _ = _processor(tmp_path, table=DemoRequiredRecord, columns=["demo_id", "demo_label"]) with pytest.raises(RequiredValueError) as exc: - processor.process(_batch(("1", "ok"), ("2", ""), ("3", None), columns=["demo_id", "demo_label"])) + processor.process( + _batch(("1", "ok"), ("2", ""), ("3", None), columns=["demo_id", "demo_label"]) + ) assert (exc.value.count, exc.value.rows) == (2, [2, 3]) def test_row_numbers_continue_across_batches(tmp_path): - processor, report = _processor(tmp_path, table=DemoRequiredRecord, columns=["demo_id", "demo_label"]) + processor, report = _processor( + tmp_path, table=DemoRequiredRecord, columns=["demo_id", "demo_label"] + ) processor.process(_batch(("1", "ok"), ("2", "ok"), columns=["demo_id", "demo_label"])) with pytest.raises(RequiredValueError) as exc: processor.process(_batch(("3", ""), columns=["demo_id", "demo_label"])) @@ -85,7 +101,9 @@ def test_row_numbers_continue_across_batches(tmp_path): def test_reject_row_keeps_good_rows_and_writes_the_rest_with_raw_text_and_reason(tmp_path): processor, report = _processor(tmp_path, REJECT) - kept = processor.process(_batch(("1", "a", "10", ""), ("2", "b", "x", ""), ("3", "c", "", "bad date"))) + kept = processor.process( + _batch(("1", "a", "10", ""), ("2", "b", "x", ""), ("3", "c", "", "bad date")) + ) processor.rejects.close() assert kept.column("demo_id").to_pylist() == [1] rejected = _rejects(tmp_path) @@ -94,11 +112,17 @@ def test_reject_row_keeps_good_rows_and_writes_the_rest_with_raw_text_and_reason ("3", "demo_date", "", "bad date"), ] assert all("cannot be read" in r["reason"] for r in rejected) - assert (report.rows_read, report.rows_rejected, report.rejects_path) == (3, 2, tmp_path / "rejects.csv") + assert (report.rows_read, report.rows_rejected, report.rejects_path) == ( + 3, + 2, + tmp_path / "rejects.csv", + ) def test_reject_row_also_rejects_missing_required_values(tmp_path): - processor, report = _processor(tmp_path, REJECT, table=DemoRequiredRecord, columns=["demo_id", "demo_label"]) + processor, report = _processor( + tmp_path, REJECT, table=DemoRequiredRecord, columns=["demo_id", "demo_label"] + ) kept = processor.process(_batch(("1", "ok"), ("2", ""), columns=["demo_id", "demo_label"])) processor.rejects.close() assert kept.num_rows == 1 and report.rows_rejected == 1 @@ -112,8 +136,12 @@ def test_a_clean_batch_writes_no_rejects_file(tmp_path): assert not (tmp_path / "rejects.csv").exists() -@pytest.mark.parametrize("limit, rows_bad, should_stop", [(1, 2, True), (2, 2, False), (0.5, 3, True), (0.5, 2, False)]) -def test_max_rejects_stops_a_load_over_a_count_or_a_fraction(tmp_path, limit, rows_bad, should_stop): +@pytest.mark.parametrize( + "limit, rows_bad, should_stop", [(1, 2, True), (2, 2, False), (0.5, 3, True), (0.5, 2, False)] +) +def test_max_rejects_stops_a_load_over_a_count_or_a_fraction( + tmp_path, limit, rows_bad, should_stop +): policy = ConversionPolicy(errors=ErrorPolicy(on_error="reject_row", max_rejects=limit)) processor, _ = _processor(tmp_path, policy) rows = [(str(i), "a", "x" if i <= rows_bad else "1", "") for i in range(1, 5)] @@ -173,7 +201,9 @@ def test_a_clean_stream_logs_no_warning(tmp_path, caplog): def test_an_unreadable_value_and_a_missing_value_keep_their_own_reasons(tmp_path): - processor, _ = _processor(tmp_path, REJECT, table=DemoRequiredRecord, columns=["demo_id", "demo_label"]) + processor, _ = _processor( + tmp_path, REJECT, table=DemoRequiredRecord, columns=["demo_id", "demo_label"] + ) processor.process(_batch(("1", "a"), ("x", "b"), ("2", ""), columns=["demo_id", "demo_label"])) processor.rejects.close() assert [(r["row"], r["column"], r["reason"]) for r in _rejects(tmp_path)] == [ @@ -196,4 +226,4 @@ def test_an_unwritable_rejects_directory_fails_before_any_batch_is_read(tmp_path blocker = tmp_path / "file" blocker.write_text("") with pytest.raises(IngestError, match="not writable"): - BatchProcessor(DemoRecord, COLUMNS, REJECT, LoadReport(), blocker / "sub" / "rejects.csv") + BatchProcessor(_DemoRecord, COLUMNS, REJECT, LoadReport(), blocker / "sub" / "rejects.csv") diff --git a/tests/loaders/test_readers.py b/tests/loaders/test_readers.py index ead44fb..2f324b0 100644 --- a/tests/loaders/test_readers.py +++ b/tests/loaders/test_readers.py @@ -26,7 +26,9 @@ def test_open_reader_selects_the_reader_by_suffix(tmp_path): pq.write_table(pa.table({"demo_id": [1]}), tmp_path / "demo_record.parquet") assert isinstance(open_reader(tmp_path / "demo_record.parquet", COLUMNS), ParquetReader) for name in ("demo_record.csv", "demo_record.tsv", "demo_record.txt"): - assert isinstance(open_reader(_write(tmp_path, "demo_id\n1\n", name), COLUMNS), DelimitedTextReader) + assert isinstance( + open_reader(_write(tmp_path, "demo_id\n1\n", name), COLUMNS), DelimitedTextReader + ) def test_delimited_text_is_read_as_raw_strings_of_the_model_columns_in_file_order(tmp_path): @@ -51,17 +53,21 @@ def test_headers_are_lowercased_and_stripped_of_quotes_and_hash_suffix(tmp_path) def test_a_quoted_and_an_unquoted_empty_field_are_both_empty_strings(tmp_path): path = _write(tmp_path, 'demo_id,demo_label\n1,\n2,""\n') - assert [r["demo_label"] for r in _rows(DelimitedTextReader(path, COLUMNS, quote_mode="csv"))] == ["", ""] + assert [ + r["demo_label"] for r in _rows(DelimitedTextReader(path, COLUMNS, quote_mode="csv")) + ] == ["", ""] def test_embedded_newlines_and_tabs_survive(tmp_path): path = _write(tmp_path, 'demo_id,demo_label\n1,"a\nb"\n2,"c\td"\n') - assert [r["demo_label"] for r in _rows(DelimitedTextReader(path, COLUMNS, quote_mode="csv"))] == ["a\nb", "c\td"] + assert [ + r["demo_label"] for r in _rows(DelimitedTextReader(path, COLUMNS, quote_mode="csv")) + ] == ["a\nb", "c\td"] def test_quote_modes_decide_what_double_quotes_mean(tmp_path): path = _write(tmp_path, 'demo_id\tdemo_label\n1\t"x"\n', "demo_record.tsv") - label = lambda mode: _rows(DelimitedTextReader(path, COLUMNS, quote_mode=mode))[0]["demo_label"] # noqa: E731 + label = lambda mode: _rows(DelimitedTextReader(path, COLUMNS, quote_mode=mode))[0]["demo_label"] assert label("csv") == "x" assert label("literal") == '"x"' assert label("by_delimiter") == '"x"' @@ -69,7 +75,11 @@ def test_quote_modes_decide_what_double_quotes_mean(tmp_path): def test_a_quote_far_into_a_tab_delimited_file_is_read_literally_by_default(tmp_path): lines = "".join(f"{i}\tlabel {i}\n" for i in range(1, 5001)) - path = _write(tmp_path, "demo_id\tdemo_label\n" + lines + '5001\t"Sick" headaches\n5002\tnext\n', "demo_record.tsv") + path = _write( + tmp_path, + "demo_id\tdemo_label\n" + lines + '5001\t"Sick" headaches\n5002\tnext\n', + "demo_record.tsv", + ) rows = _rows(DelimitedTextReader(path, COLUMNS)) assert [r["demo_label"] for r in rows[-2:]] == ['"Sick" headaches', "next"] @@ -80,21 +90,33 @@ def test_a_file_is_read_in_several_batches_without_losing_rows(tmp_path): reader = DelimitedTextReader(path, COLUMNS, block_size=64 << 10) batches = list(reader.batches()) assert len(batches) > 1 - assert [row["demo_id"] for batch in batches for row in batch.to_pylist()] == [str(i) for i in range(1, 20_001)] + assert [row["demo_id"] for batch in batches for row in batch.to_pylist()] == [ + str(i) for i in range(1, 20_001) + ] def test_an_empty_file_has_no_content_and_a_header_only_file_has_no_rows(tmp_path): empty = DelimitedTextReader(_write(tmp_path, "", "demo_record.csv"), COLUMNS) assert empty.empty and list(empty.batches()) == [] - header_only = DelimitedTextReader(_write(tmp_path, "demo_id,demo_label\n", "other.csv"), COLUMNS) + header_only = DelimitedTextReader( + _write(tmp_path, "demo_id,demo_label\n", "other.csv"), COLUMNS + ) assert not header_only.empty and list(header_only.batches()) == [] -@pytest.mark.parametrize("name, quote_mode", [("demo_record.csv", "csv"), ("demo_record.tsv", "literal")]) +@pytest.mark.parametrize( + "name, quote_mode", [("demo_record.csv", "csv"), ("demo_record.tsv", "literal")] +) def test_a_delimited_reader_nulls_nothing(tmp_path, name, quote_mode): delimiter = "," if name.endswith("csv") else "\t" - path = _write(tmp_path, f"demo_id{delimiter}demo_label\n1{delimiter}\n2{delimiter}NULL\n3{delimiter} NA \n", name) - labels = [r["demo_label"] for r in _rows(DelimitedTextReader(path, COLUMNS, quote_mode=quote_mode))] + path = _write( + tmp_path, + f"demo_id{delimiter}demo_label\n1{delimiter}\n2{delimiter}NULL\n3{delimiter} NA \n", + name, + ) + labels = [ + r["demo_label"] for r in _rows(DelimitedTextReader(path, COLUMNS, quote_mode=quote_mode)) + ] assert labels == ["", "NULL", " NA "] @@ -105,7 +127,10 @@ def test_parquet_keeps_stored_types_and_ignores_other_columns(tmp_path): assert reader.columns == ["demo_id", "demo_label"] batch = next(iter(reader.batches())) assert batch.schema.types == [pa.int64(), pa.string()] - assert batch.to_pylist() == [{"demo_id": 1, "demo_label": "a"}, {"demo_id": 2, "demo_label": None}] + assert batch.to_pylist() == [ + {"demo_id": 1, "demo_label": "a"}, + {"demo_id": 2, "demo_label": None}, + ] def test_parquet_is_read_in_batches_of_chunksize_rows(tmp_path): diff --git a/tests/loaders/test_schema_translate_map.py b/tests/loaders/test_schema_translate_map.py index 5ced078..c6bc9f2 100644 --- a/tests/loaders/test_schema_translate_map.py +++ b/tests/loaders/test_schema_translate_map.py @@ -23,13 +23,15 @@ from oa_configurator import Role as SchemaRole from orm_loader.backends import STAGING_SCHEMA - from tests.conftest import schema_scoped_session from tests.models import SimpleTable, VocabSchemaTable def test_load_file_respects_non_default_schema_end_to_end(pg_db, tmp_path): - with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_primary") as (scoped, session): + with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_primary") as ( + scoped, + session, + ): schema = scoped.schemas[SchemaRole.PRIMARY] csv_path = tmp_path / "test_table.csv" pd.DataFrame( @@ -60,7 +62,10 @@ def test_replace_merge_respects_non_default_schema_end_to_end(pg_db, tmp_path): """A second load_file() call with merge_strategy="replace" against the same non-default schema. Proves the merge path itself, not just the initial insert, qualifies correctly.""" - with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_replace") as (scoped, session): + with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_replace") as ( + scoped, + session, + ): schema = scoped.schemas[SchemaRole.PRIMARY] def _write_and_load(rows: list[dict], path_name: str) -> int: @@ -108,7 +113,9 @@ def test_load_file_respects_non_primary_schema_tag_end_to_end(pg_db, tmp_path): with scoped.engine.connect() as conn: rows = conn.execute( - sa.text(f'SELECT id, name FROM "{vocab_schema}"."test_vocab_role_table" ORDER BY id') + sa.text( + f'SELECT id, name FROM "{vocab_schema}"."test_vocab_role_table" ORDER BY id' + ) ).fetchall() leaked = conn.execute( sa.text(f"SELECT to_regclass('{primary_schema}.test_vocab_role_table')") diff --git a/tests/loaders/test_special_columns.py b/tests/loaders/test_special_columns.py index 915dc5e..c1d4dbd 100644 --- a/tests/loaders/test_special_columns.py +++ b/tests/loaders/test_special_columns.py @@ -37,7 +37,11 @@ class RequiredComputedRecord(SpecialBase, FileLoadableTableInterface): class KeyedRecord(SpecialBase, FileLoadableTableInterface): __tablename__ = "special_keyed" record_id: so.Mapped[int] = so.mapped_column( - "id_col", sa.Integer, key="id_key", primary_key=True, autoincrement=False, + "id_col", + sa.Integer, + key="id_key", + primary_key=True, + autoincrement=False, ) value: so.Mapped[int] = so.mapped_column("val_col", sa.Integer, key="val_key") @@ -45,10 +49,18 @@ class KeyedRecord(SpecialBase, FileLoadableTableInterface): class CompositeKeyedRecord(SpecialBase, FileLoadableTableInterface): __tablename__ = "special_composite_keyed" record_id: so.Mapped[int] = so.mapped_column( - "id_col", sa.Integer, key="id_key", primary_key=True, autoincrement=False, + "id_col", + sa.Integer, + key="id_key", + primary_key=True, + autoincrement=False, ) group_id: so.Mapped[int] = so.mapped_column( - "group_col", sa.Integer, key="group_key", primary_key=True, autoincrement=False, + "group_col", + sa.Integer, + key="group_key", + primary_key=True, + autoincrement=False, ) value: so.Mapped[int] = so.mapped_column("val_col", sa.Integer, key="val_key") @@ -62,7 +74,11 @@ class ExpressionDefaultRecord(SpecialBase, FileLoadableTableInterface): class DefaultKeyRecord(SpecialBase, FileLoadableTableInterface): __tablename__ = "special_default_key" record_id: so.Mapped[int] = so.mapped_column( - sa.Integer, primary_key=True, nullable=True, default=99, autoincrement=False, + sa.Integer, + primary_key=True, + nullable=True, + default=99, + autoincrement=False, ) note: so.Mapped[str | None] = so.mapped_column(sa.String) @@ -118,8 +134,12 @@ def test_nonnullable_computed_column_can_be_absent_from_the_file(tmp_path, norma table = cast(type[FileTableProtocol], RequiredComputedRecord) assert table.required_columns() == {"record_id", "score"} processor = BatchProcessor( - table, ["record_id", "score"], ConversionPolicy(), LoadReport(), - tmp_path / "rejects.csv", normalise=normalise, + table, + ["record_id", "score"], + ConversionPolicy(), + LoadReport(), + tmp_path / "rejects.csv", + normalise=normalise, ) output = processor.process(pa.record_batch({"record_id": ["1"], "score": ["2"]})) assert output.column_names == ["record_id", "score"] @@ -130,33 +150,48 @@ def test_nonnullable_computed_column_can_be_absent_from_the_file(tmp_path, norma @pytest.mark.parametrize("strategy", ["insert", "upsert", "replace"]) @pytest.mark.parametrize("batch_size", [None, 1]) def test_merge_uses_database_names_when_column_keys_differ( - dialect_session, tmp_path, model, strategy, batch_size, + dialect_session, + tmp_path, + model, + strategy, + batch_size, ): _create(dialect_session, model) composite = model is CompositeKeyedRecord header = "id_col,group_col,val_col\n" if composite else "id_col,val_col\n" + def row(key, value): return f"{key},1,{value}\n" if composite else f"{key},{value}\n" _load(dialect_session, tmp_path, model, header + row(1, 5), merge_strategy="insert") data = row(2, 20) if strategy == "insert" else row(1, 10) + row(2, 20) _load( - dialect_session, tmp_path, model, header + data, - merge_strategy=strategy, merge_batch_size=batch_size, + dialect_session, + tmp_path, + model, + header + data, + merge_strategy=strategy, + merge_batch_size=batch_size, + ) + rows = dialect_session.execute( + sa.select(model.record_id, model.value).order_by(model.record_id) ) - rows = dialect_session.execute(sa.select(model.record_id, model.value).order_by(model.record_id)) assert [tuple(row) for row in rows] == [(1, 10 if strategy == "replace" else 5), (2, 20)] @pytest.mark.parametrize("strategy", ["insert", "upsert", "replace"]) @pytest.mark.parametrize("batch_size", [None, 2]) -def test_sql_expression_defaults_are_evaluated_for_each_merged_row(dialect_session, tmp_path, strategy, batch_size): +def test_sql_expression_defaults_are_evaluated_for_each_merged_row( + dialect_session, tmp_path, strategy, batch_size +): _create(dialect_session, ExpressionDefaultRecord) table = ExpressionDefaultRecord if strategy != "insert": _load(dialect_session, tmp_path, table, "record_id,token\n1,42\n", merge_strategy="insert") text = "record_id,token\n" + "".join(f"{i},\n" for i in range(1, 11)) + "11,123\n" - _load(dialect_session, tmp_path, table, text, merge_strategy=strategy, merge_batch_size=batch_size) + _load( + dialect_session, tmp_path, table, text, merge_strategy=strategy, merge_batch_size=batch_size + ) tokens = dialect_session.scalars(sa.select(table.token).order_by(table.record_id)).all() assert len(tokens) == 11 assert all(value is not None for value in tokens) @@ -167,7 +202,9 @@ def test_sql_expression_defaults_are_evaluated_for_each_merged_row(dialect_sessi @pytest.mark.parametrize("normalise", [False, True]) -def test_a_sql_expression_default_is_evaluated_for_each_row_of_a_missing_column(dialect_session, tmp_path, normalise): +def test_a_sql_expression_default_is_evaluated_for_each_row_of_a_missing_column( + dialect_session, tmp_path, normalise +): _create(dialect_session, ExpressionDefaultRecord) text = "record_id\n" + "".join(f"{i}\n" for i in range(1, 11)) _load(dialect_session, tmp_path, ExpressionDefaultRecord, text, normalise=normalise) @@ -176,7 +213,9 @@ def test_a_sql_expression_default_is_evaluated_for_each_row_of_a_missing_column( @pytest.mark.parametrize("normalise", [False, True]) -def test_a_primary_key_with_a_default_must_still_be_in_the_file(dialect_session, tmp_path, normalise): +def test_a_primary_key_with_a_default_must_still_be_in_the_file( + dialect_session, tmp_path, normalise +): _create(dialect_session, DefaultKeyRecord) with pytest.raises(IngestError, match="record_id"): _load(dialect_session, tmp_path, DefaultKeyRecord, "note\na\n", normalise=normalise) @@ -186,8 +225,12 @@ def test_a_blank_primary_key_is_rejected_instead_of_taking_its_default(dialect_s _create(dialect_session, DefaultKeyRecord) policy = ConversionPolicy(errors=ErrorPolicy(on_error="reject_row")) _load( - dialect_session, tmp_path, DefaultKeyRecord, - "record_id,note\n,a\n1,b\n", policy=policy, rejects_dir=tmp_path, + dialect_session, + tmp_path, + DefaultKeyRecord, + "record_id,note\n,a\n1,b\n", + policy=policy, + rejects_dir=tmp_path, ) assert dialect_session.scalars(sa.select(DefaultKeyRecord.record_id)).all() == [1] @@ -197,12 +240,20 @@ def test_a_primary_key_is_required_and_takes_no_default(): assert "record_id" not in DefaultKeyRecord.column_defaults() -@pytest.mark.parametrize("model, declared", [ - (InvalidNumericRecord, r"\(2, 5\)"), (UnboundedNumericRecord, r"\(None, None\)"), (WideNumericRecord, r"\(50, 2\)"), -]) +@pytest.mark.parametrize( + "model, declared", + [ + (InvalidNumericRecord, r"\(2, 5\)"), + (UnboundedNumericRecord, r"\(None, None\)"), + (WideNumericRecord, r"\(50, 2\)"), + ], +) def test_unsupported_numeric_declaration_names_the_table_and_column(tmp_path, model, declared): with pytest.raises(PolicyError, match=rf"{model.__tablename__}.amount.*{declared}"): BatchProcessor( - cast(type[FileTableProtocol], model), ["record_id", "amount"], - ConversionPolicy(), LoadReport(), tmp_path / "rejects.csv", + cast(type[FileTableProtocol], model), + ["record_id", "amount"], + ConversionPolicy(), + LoadReport(), + tmp_path / "rejects.csv", ) diff --git a/tests/loaders/test_split_connection.py b/tests/loaders/test_split_connection.py index 0a78b06..0bc21aa 100644 --- a/tests/loaders/test_split_connection.py +++ b/tests/loaders/test_split_connection.py @@ -1,12 +1,17 @@ -"""Tests split CDM/vocab connection instances. -""" +"""Tests split CDM/vocab connection instances.""" from __future__ import annotations import pandas as pd import sqlalchemy as sa import sqlalchemy.orm as so -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Resolver, StackConfig +from oa_configurator import ( + CDMDatabaseConfig, + ConnectionConfig, + ResolvedCDMDatabase, + Resolver, + StackConfig, +) from oa_configurator import Role as SchemaRole from sqlalchemy.exc import UnboundExecutionError @@ -32,9 +37,8 @@ def test_vocab_file_load_routes_every_session_bind_to_vocab_file(tmp_path): ) ) resolved = resolver.resolve_database("test_cdm") - primary_engine, vocab_engine = resolved.create_engines( - schema_claims=[staging_schema_claim()] - ) + assert isinstance(resolved, ResolvedCDMDatabase) + primary_engine, vocab_engine = resolved.create_engines(schema_claims=[staging_schema_claim()]) VocabSchemaTable.__table__.create(vocab_engine) class MapperRoutedSession(so.Session): @@ -46,7 +50,9 @@ def get_bind(self, mapper=None, clause=None, **kwargs): if mapper is None and clause is None: raise UnboundExecutionError("a mapper or clause is required") if mapper is not None: - sqlite_engine = vocab_engine if mapper.__table__.schema == SchemaRole.VOCAB else primary_engine + sqlite_engine = ( + vocab_engine if mapper.__table__.schema == SchemaRole.VOCAB else primary_engine + ) self._last_mapper_engine = sqlite_engine return sqlite_engine table = getattr(clause, "table", None) @@ -70,9 +76,7 @@ def get_bind(self, mapper=None, clause=None, **kwargs): sqlite_session.close() with vocab_engine.connect() as connection: - rows = connection.execute( - sa.select(VocabSchemaTable.id, VocabSchemaTable.name) - ).all() + rows = connection.execute(sa.select(VocabSchemaTable.id, VocabSchemaTable.name)).all() assert rows == [(7, "vocab-row")] assert not sa.inspect(primary_engine).has_table("test_vocab_role_table") primary_engine.dispose() @@ -84,10 +88,15 @@ def test_load_file_across_two_genuinely_separate_connections(pg_db, sqlite_sessi SQLite, vocab connection, via the module-level ``sqlite_engine``/``sqlite_session`` fixtures) are two entirely different engines against two entirely different database systems.""" - with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_split") as (scoped, primary_session): + with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_split") as ( + scoped, + primary_session, + ): primary_schema = scoped.schemas[SchemaRole.PRIMARY] primary_csv = tmp_path / "test_table.csv" - pd.DataFrame([{"id": 1, "name": "primary-alpha"}]).to_csv(primary_csv, index=False, sep="\t") + pd.DataFrame([{"id": 1, "name": "primary-alpha"}]).to_csv( + primary_csv, index=False, sep="\t" + ) vocab_csv = tmp_path / "test_vocab_role_table.csv" pd.DataFrame([{"id": 1, "name": "vocab-alpha"}]).to_csv(vocab_csv, index=False, sep="\t") @@ -112,7 +121,9 @@ def test_load_file_across_two_genuinely_separate_connections(pg_db, sqlite_sessi assert primary_rows == [(1, "primary-alpha")] assert leaked_vocab_table_in_pg is None - vocab_rows = sqlite_session.execute( - sa.select(VocabSchemaTable).order_by(VocabSchemaTable.id) - ).scalars().all() + vocab_rows = ( + sqlite_session.execute(sa.select(VocabSchemaTable).order_by(VocabSchemaTable.id)) + .scalars() + .all() + ) assert [(r.id, r.name) for r in vocab_rows] == [(1, "vocab-alpha")] diff --git a/tests/loaders/test_stage_file.py b/tests/loaders/test_stage_file.py index 3c6ae0d..646e795 100644 --- a/tests/loaders/test_stage_file.py +++ b/tests/loaders/test_stage_file.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import cast import pytest import sqlalchemy as sa @@ -6,8 +7,11 @@ from orm_loader.backends import resolve_backend from orm_loader.helpers import IngestError from orm_loader.loaders import LoaderContext, open_reader, stage_file +from orm_loader.tables.typing import FileTableProtocol from tests.models import FileLoaderTable +_FileLoaderTable = cast(type[FileTableProtocol], FileLoaderTable) + def _reader(ctx, **options): return open_reader(ctx.path, ctx.tableclass.model_columns(), **options) @@ -16,9 +20,9 @@ def _reader(ctx, **options): def test_loader_context_fields(sqlite_session): ctx = LoaderContext( - tableclass=FileLoaderTable, + tableclass=_FileLoaderTable, session=sqlite_session, - backend=resolve_backend(sqlite_session, mapper=FileLoaderTable), + backend=resolve_backend(sqlite_session, mapper=_FileLoaderTable), path=Path("file.csv"), normalise=False, dedupe=True, @@ -27,6 +31,7 @@ def test_loader_context_fields(sqlite_session): assert ctx.normalise is False assert ctx.dedupe is True + def test_stage_file_case_insensitive_headers(tmp_path, sqlite_session): """ The loader should accept mixed-case CSV headers and map them @@ -34,24 +39,20 @@ def test_stage_file_case_insensitive_headers(tmp_path, sqlite_session): """ csv = tmp_path / "test_case_headers.csv" - csv.write_text( - "ID,Value\n" - "1,alpha\n" - "2,beta\n" - ) + csv.write_text("ID,Value\n1,alpha\n2,beta\n") ctx = LoaderContext( - tableclass=FileLoaderTable, + tableclass=_FileLoaderTable, session=sqlite_session, - backend=resolve_backend(sqlite_session, mapper=FileLoaderTable), + backend=resolve_backend(sqlite_session, mapper=_FileLoaderTable), path=csv, dedupe=True, ) - n = stage_file(ctx, _reader(ctx), FileLoaderTable.__table__) + n = stage_file(ctx, _reader(ctx), _FileLoaderTable.__table__) rows = sqlite_session.execute( - sa.text('SELECT id, value FROM test_file_loader ORDER BY id') + sa.text("SELECT id, value FROM test_file_loader ORDER BY id") ).all() assert n == 2 @@ -66,24 +67,20 @@ def test_stage_file_literal_mode_preserves_quotes(tmp_path, sqlite_session): """ csv = tmp_path / "quoted.csv" - csv.write_text( - 'id,value\n' - '1,"hello"\n' - '2,"world"\n' - ) + csv.write_text('id,value\n1,"hello"\n2,"world"\n') ctx = LoaderContext( - tableclass=FileLoaderTable, + tableclass=_FileLoaderTable, session=sqlite_session, - backend=resolve_backend(sqlite_session, mapper=FileLoaderTable), + backend=resolve_backend(sqlite_session, mapper=_FileLoaderTable), path=csv, dedupe=True, ) - n = stage_file(ctx, _reader(ctx, quote_mode="literal"), FileLoaderTable.__table__) + n = stage_file(ctx, _reader(ctx, quote_mode="literal"), _FileLoaderTable.__table__) rows = sqlite_session.execute( - sa.text('SELECT id, value FROM test_file_loader ORDER BY id') + sa.text("SELECT id, value FROM test_file_loader ORDER BY id") ).all() assert n == 2 @@ -97,24 +94,20 @@ def test_stage_file_csv_mode_strips_quotes(tmp_path, sqlite_session): """ csv = tmp_path / "quoted.csv" - csv.write_text( - 'id,value\n' - '1,"hello"\n' - '2,"world"\n' - ) + csv.write_text('id,value\n1,"hello"\n2,"world"\n') ctx = LoaderContext( - tableclass=FileLoaderTable, + tableclass=_FileLoaderTable, session=sqlite_session, - backend=resolve_backend(sqlite_session, mapper=FileLoaderTable), + backend=resolve_backend(sqlite_session, mapper=_FileLoaderTable), path=csv, dedupe=True, ) - n = stage_file(ctx, _reader(ctx, quote_mode="csv"), FileLoaderTable.__table__) + n = stage_file(ctx, _reader(ctx, quote_mode="csv"), _FileLoaderTable.__table__) rows = sqlite_session.execute( - sa.text('SELECT id, value FROM test_file_loader ORDER BY id') + sa.text("SELECT id, value FROM test_file_loader ORDER BY id") ).all() assert n == 2 @@ -128,24 +121,20 @@ def test_stage_file_by_delimiter_tab_preserves_quotes(tmp_path, sqlite_session): """ tsv = tmp_path / "tab.tsv" - tsv.write_text( - 'id\tvalue\n' - '1\t"hi"\n' - '2\tbye\n' - ) + tsv.write_text('id\tvalue\n1\t"hi"\n2\tbye\n') ctx = LoaderContext( - tableclass=FileLoaderTable, + tableclass=_FileLoaderTable, session=sqlite_session, - backend=resolve_backend(sqlite_session, mapper=FileLoaderTable), + backend=resolve_backend(sqlite_session, mapper=_FileLoaderTable), path=tsv, dedupe=True, ) - n = stage_file(ctx, _reader(ctx, quote_mode="by_delimiter"), FileLoaderTable.__table__) + n = stage_file(ctx, _reader(ctx, quote_mode="by_delimiter"), _FileLoaderTable.__table__) rows = sqlite_session.execute( - sa.text('SELECT id, value FROM test_file_loader ORDER BY id') + sa.text("SELECT id, value FROM test_file_loader ORDER BY id") ).all() assert n == 2 @@ -157,12 +146,12 @@ def test_file_without_data_rows_loads_nothing_whatever_its_header(tmp_path, sqli csv = tmp_path / "no_rows.csv" csv.write_text(text) ctx = LoaderContext( - tableclass=FileLoaderTable, + tableclass=_FileLoaderTable, session=sqlite_session, - backend=resolve_backend(sqlite_session, mapper=FileLoaderTable), + backend=resolve_backend(sqlite_session, mapper=_FileLoaderTable), path=csv, ) - assert stage_file(ctx, _reader(ctx), FileLoaderTable.__table__) == 0 + assert stage_file(ctx, _reader(ctx), _FileLoaderTable.__table__) == 0 assert ctx.report.rows_read == 0 @@ -171,13 +160,13 @@ def test_accounting_is_checked_against_the_staging_table(tmp_path, sqlite_sessio csv = tmp_path / "test_file_loader.csv" csv.write_text("id,value\n1,a\n2,b\n3,c\n") ctx = LoaderContext( - tableclass=FileLoaderTable, + tableclass=_FileLoaderTable, session=sqlite_session, - backend=resolve_backend(sqlite_session, mapper=FileLoaderTable), + backend=resolve_backend(sqlite_session, mapper=_FileLoaderTable), path=csv, dedupe=False, ) - backend = resolve_backend(sqlite_session, mapper=FileLoaderTable) + backend = resolve_backend(sqlite_session, mapper=_FileLoaderTable) insert = type(backend).insert_batches def losing(self, ctx, staging, batches, columns): @@ -185,4 +174,4 @@ def losing(self, ctx, staging, batches, columns): monkeypatch.setattr(type(backend), "insert_batches", losing) with pytest.raises(IngestError, match="read 3 rows but loaded 0"): - stage_file(ctx, _reader(ctx), FileLoaderTable.__table__) + stage_file(ctx, _reader(ctx), _FileLoaderTable.__table__) diff --git a/tests/mappers/test_materialised_view_contracts.py b/tests/mappers/test_materialised_view_contracts.py index b5c8a45..8f30077 100644 --- a/tests/mappers/test_materialised_view_contracts.py +++ b/tests/mappers/test_materialised_view_contracts.py @@ -15,8 +15,7 @@ def test_mappers_package_exports_materialized_view_api(): - import orm_loader.mappers as mappers - + from orm_loader import mappers from orm_loader.mappers.materialised_view_errors import MaterializationError assert mappers.MaterializedViewIndex is MaterializedViewIndex @@ -55,7 +54,9 @@ def test_materialized_view_index_defaults_to_non_unique(): def test_drop_materialized_view_default_compiles_with_if_exists(): ddl = DropMaterializedView("reporting.mv_test") - assert str(ddl.compile(dialect=_DIALECT)) == "DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test" + assert ( + str(ddl.compile(dialect=_DIALECT)) == "DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test" + ) def test_drop_materialized_view_if_exists_false_and_cascade(): @@ -69,8 +70,7 @@ def test_create_materialized_view_index_compiles_unique_index_ddl(): ddl = CreateMaterializedViewIndex("reporting.mv_test", index) assert str(ddl.compile(dialect=_DIALECT)) == ( - 'CREATE UNIQUE INDEX IF NOT EXISTS "mv_test_id_uq" ' - 'ON reporting.mv_test ("id", "person_id")' + 'CREATE UNIQUE INDEX IF NOT EXISTS "mv_test_id_uq" ON reporting.mv_test ("id", "person_id")' ) diff --git a/tests/mappers/test_materialised_view_mixin.py b/tests/mappers/test_materialised_view_mixin.py index afb8df5..2b0f3b0 100644 --- a/tests/mappers/test_materialised_view_mixin.py +++ b/tests/mappers/test_materialised_view_mixin.py @@ -1,14 +1,14 @@ from __future__ import annotations import importlib -from typing import Any +from typing import Any, ClassVar import pytest import sqlalchemy as sa import sqlalchemy.orm as so - from oa_configurator import Role from oa_configurator.testing import scoped_test_schema + from orm_loader.mappers.materialised_view_contracts import MaterializedViewIndex from orm_loader.mappers.materialised_view_mixin import ( MaterializedViewBackend, @@ -49,7 +49,11 @@ def test_refresh_all_mvs_resolves_each_views_own_schema_tag(pg_db) -> None: class _IndexedPgMv(MaterializedViewMixin): __mv_name__ = "mv_indexed_pg_test" __mv_select__ = sa.select(sa.literal(1).label("row_id")) - __mv_indexes__ = (MaterializedViewIndex(name="mv_indexed_pg_test_row_id_uq", columns=("row_id",), unique=True),) + __mv_indexes__ = ( + MaterializedViewIndex( + name="mv_indexed_pg_test_row_id_uq", columns=("row_id",), unique=True + ), + ) def test_create_mv_with_declared_index_against_postgres(pg_db) -> None: @@ -64,7 +68,9 @@ def test_create_mv_with_declared_index_against_postgres(pg_db) -> None: with scoped.engine.connect() as conn: inspector = sa.inspect(conn) assert inspector.has_table("mv_indexed_pg_test", schema=schema) - index_names = {idx["name"] for idx in inspector.get_indexes("mv_indexed_pg_test", schema=schema)} + index_names = { + idx["name"] for idx in inspector.get_indexes("mv_indexed_pg_test", schema=schema) + } assert "mv_indexed_pg_test_row_id_uq" in index_names @@ -88,13 +94,15 @@ def test_drop_mv_cascade_against_postgres(pg_db) -> None: def test_refresh_mv_concurrently_against_postgres(pg_db) -> None: """refresh_mv(concurrently=True) through the mixin succeeds against a real database when a unique index is declared. - The eligibility check and error translation are already proven at the - backend level directly in test_postgres_backend.py. + The eligibility check and error translation are already proven at the + backend level directly in test_postgres_backend.py. This proves the mixin wires into that correctly end to end.""" - with scoped_test_schema(pg_db.resolved, prefix="mv_refresh") as scoped: - with scoped.engine.begin() as conn: - _IndexedPgMv.create_mv(conn) - _IndexedPgMv.refresh_mv(conn, concurrently=True) + with ( + scoped_test_schema(pg_db.resolved, prefix="mv_refresh") as scoped, + scoped.engine.begin() as conn, + ): + _IndexedPgMv.create_mv(conn) + _IndexedPgMv.refresh_mv(conn, concurrently=True) class _FakeBackend(MaterializedViewBackend): @@ -161,8 +169,9 @@ def test_create_mv_forwards_default_args_to_backend(fake_backend: _FakeBackend, def test_create_mv_rejects_invalid_bind(fake_backend: _FakeBackend): + invalid_bind: Any = "bind" with pytest.raises(TypeError, match="bind must be a SQLAlchemy Engine or Connection"): - _NoIndexMv.create_mv("bind") + _NoIndexMv.create_mv(invalid_bind) assert fake_backend.calls == [] @@ -208,7 +217,7 @@ class _MappedVocabMv(_MappedMvBase, MaterializedViewMixin): __mv_name__ = "mv_mapped_vocab" __mv_select__ = _SELECT __tablename__ = "mv_mapped_vocab" - __table_args__ = {"schema": Role.VOCAB.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": Role.VOCAB.value} row_id = sa.Column(sa.Integer, primary_key=True) @@ -265,12 +274,12 @@ def test_create_mv_forwards_if_not_exists_to_declared_indexes(fake_backend: _Fak assert fake_backend.calls[1][2] == {"schema": None, "if_not_exists": False} -def test_create_mv_engine_uses_one_transaction_for_view_and_indexes(monkeypatch: pytest.MonkeyPatch): +def test_create_mv_engine_uses_one_transaction_for_view_and_indexes( + monkeypatch: pytest.MonkeyPatch, +): engine = sa.create_engine("sqlite:///:memory:", future=True) with engine.begin() as connection: - connection.execute( - sa.text("CREATE TABLE materialization_steps (step TEXT NOT NULL)") - ) + connection.execute(sa.text("CREATE TABLE materialization_steps (step TEXT NOT NULL)")) connections: list[sa.engine.Connection] = [] fail_once = True @@ -278,16 +287,12 @@ def test_create_mv_engine_uses_one_transaction_for_view_and_indexes(monkeypatch: class TransactionalBackend(_FakeBackend): def create_materialized_view(self, bind, name, selectable, **kwargs): connections.append(bind) - bind.execute( - sa.text("INSERT INTO materialization_steps (step) VALUES ('view')") - ) + bind.execute(sa.text("INSERT INTO materialization_steps (step) VALUES ('view')")) def create_materialized_view_index(self, bind, name, index, **kwargs): nonlocal fail_once connections.append(bind) - bind.execute( - sa.text("INSERT INTO materialization_steps (step) VALUES ('index')") - ) + bind.execute(sa.text("INSERT INTO materialization_steps (step) VALUES ('index')")) if fail_once: fail_once = False raise RuntimeError("index failed") @@ -306,9 +311,10 @@ class TransactionalMv(MaterializedViewMixin): assert connections[0] is connections[1] with engine.connect() as connection: - assert connection.execute( - sa.text("SELECT COUNT(*) FROM materialization_steps") - ).scalar_one() == 0 + assert ( + connection.execute(sa.text("SELECT COUNT(*) FROM materialization_steps")).scalar_one() + == 0 + ) TransactionalMv.create_mv(engine) with engine.connect() as connection: @@ -370,13 +376,13 @@ class _MvA(MaterializedViewMixin): class _MvB(MaterializedViewMixin): __mv_name__ = "mv_b" __mv_select__ = _SELECT - __mv_dependencies__ = {"mv_a"} + __mv_dependencies__ = frozenset({"mv_a"}) class _MvC(MaterializedViewMixin): __mv_name__ = "mv_c" __mv_select__ = _SELECT - __mv_dependencies__ = {"mv_a", "mv_b"} + __mv_dependencies__ = frozenset({"mv_a", "mv_b"}) def test_resolve_mv_refresh_order_linear_dependency(): @@ -393,7 +399,7 @@ def test_resolve_mv_refresh_order_ignores_dependencies_outside_the_given_set(): class _StandaloneMv(MaterializedViewMixin): __mv_name__ = "mv_standalone" __mv_select__ = _SELECT - __mv_dependencies__ = {"some_other_table_not_in_this_registry"} + __mv_dependencies__ = frozenset({"some_other_table_not_in_this_registry"}) assert resolve_mv_refresh_order([_StandaloneMv]) == [_StandaloneMv] @@ -402,12 +408,12 @@ def test_resolve_mv_refresh_order_raises_on_cycle(): class _CycleA(MaterializedViewMixin): __mv_name__ = "mv_cycle_a" __mv_select__ = _SELECT - __mv_dependencies__ = {"mv_cycle_b"} + __mv_dependencies__ = frozenset({"mv_cycle_b"}) class _CycleB(MaterializedViewMixin): __mv_name__ = "mv_cycle_b" __mv_select__ = _SELECT - __mv_dependencies__ = {"mv_cycle_a"} + __mv_dependencies__ = frozenset({"mv_cycle_a"}) with pytest.raises(RuntimeError, match="Cycle detected"): resolve_mv_refresh_order([_CycleA, _CycleB]) @@ -421,7 +427,9 @@ def test_refresh_all_mvs_refreshes_in_dependency_order(fake_backend: _FakeBacken def test_a_backend_without_materialized_views_is_refused(bind): - with pytest.raises(NotImplementedError, match="Backend 'sqlite' does not support materialized views"): + with pytest.raises( + NotImplementedError, match="Backend 'sqlite' does not support materialized views" + ): _NoIndexMv.create_mv(bind) with pytest.raises(NotImplementedError, match="does not support materialized views"): _NoIndexMv.refresh_mv(bind) diff --git a/tests/models.py b/tests/models.py index a6e9425..9e0eb11 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,7 +1,7 @@ - from datetime import date, datetime from decimal import Decimal from enum import Enum +from typing import ClassVar import sqlalchemy as sa import sqlalchemy.orm as so @@ -18,10 +18,9 @@ class Role(str, Enum): LAST_AUTHOR = "last author" - class FileLoaderTable(FileLoadableTableInterface, Base): __tablename__ = "test_file_loader" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id = sa.Column(sa.Integer, primary_key=True) value = sa.Column(sa.String, nullable=False) @@ -39,7 +38,7 @@ class SimpleTable(Base, FileLoadableTableInterface): class RequiredTable(Base, FileLoadableTableInterface): __tablename__ = "required_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String, nullable=False) @@ -47,7 +46,7 @@ class RequiredTable(Base, FileLoadableTableInterface): class CompositeTable(Base, FileLoadableTableInterface): __tablename__ = "composite_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} a: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) b: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) @@ -59,7 +58,7 @@ class CycleATable(Base, FileLoadableTableInterface): DuckDB can't execute the ALTER TABLE ADD CONSTRAINT this cycle forces.""" __tablename__ = "cycle_a_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) b_id: so.Mapped[int | None] = so.mapped_column( @@ -69,7 +68,7 @@ class CycleATable(Base, FileLoadableTableInterface): class CycleBTable(Base, FileLoadableTableInterface): __tablename__ = "cycle_b_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) a_id: so.Mapped[int | None] = so.mapped_column( @@ -82,7 +81,7 @@ class EnumTable(Base, FileLoadableTableInterface): sa.Enum stores by default.""" __tablename__ = "enum_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) role: so.Mapped[Role | None] = so.mapped_column(sa.Enum(Role), nullable=True) @@ -95,7 +94,7 @@ class ComputedColumnTable(Base, FileLoadableTableInterface): FileLoadableTableInterface.""" __tablename__ = "computed_column_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String) @@ -110,7 +109,7 @@ class ReferencedParentTable(Base, FileLoadableTableInterface): dropped.""" __tablename__ = "referenced_parent_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) grandparent_id: so.Mapped[int] = so.mapped_column( @@ -121,7 +120,7 @@ class ReferencedParentTable(Base, FileLoadableTableInterface): class UnreferencedGrandparentTable(Base, FileLoadableTableInterface): __tablename__ = "unreferenced_grandparent_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String) @@ -129,7 +128,7 @@ class UnreferencedGrandparentTable(Base, FileLoadableTableInterface): class ReferencingChildTable(Base, FileLoadableTableInterface): __tablename__ = "referencing_child_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) parent_id: so.Mapped[int] = so.mapped_column( @@ -159,7 +158,7 @@ class VocabReferencingTable(Base, FileLoadableTableInterface): reached a different way.""" __tablename__ = "vocab_referencing_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) vocab_id: so.Mapped[int | None] = so.mapped_column( @@ -176,7 +175,7 @@ class SelfReferentialTable(Base, FileLoadableTableInterface): self-reference.""" __tablename__ = "self_referential_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) parent_id: so.Mapped[int | None] = so.mapped_column( @@ -189,7 +188,7 @@ class DemoRecord(Base, FileLoadableTableInterface): """Optional text label plus one column of each typed kind, for null-handling tests.""" __tablename__ = "demo_record" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} demo_id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) demo_label: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) @@ -202,7 +201,7 @@ class DemoRequiredRecord(Base, FileLoadableTableInterface): """Same shape as DemoRecord with a required text label.""" __tablename__ = "demo_required_record" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} demo_id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) demo_label: so.Mapped[str] = so.mapped_column(sa.String, nullable=False) @@ -212,7 +211,7 @@ class DemoDefaultedRecord(Base, FileLoadableTableInterface): """Non-nullable columns with a server default, a scalar default and a callable default.""" __tablename__ = "demo_defaulted_record" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} demo_id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) demo_status: so.Mapped[str] = so.mapped_column(sa.String, nullable=False, server_default="open") @@ -225,7 +224,7 @@ class DemoTypedRecord(Base, FileLoadableTableInterface): """One column of each kind a typed (Parquet) input can fill.""" __tablename__ = "demo_typed_record" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} typed_id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) typed_small: so.Mapped[int | None] = so.mapped_column(sa.SmallInteger, nullable=True) @@ -234,7 +233,9 @@ class DemoTypedRecord(Base, FileLoadableTableInterface): typed_flag: so.Mapped[bool | None] = so.mapped_column(sa.Boolean, nullable=True) typed_date: so.Mapped[date | None] = so.mapped_column(sa.Date, nullable=True) typed_stamp: so.Mapped[datetime | None] = so.mapped_column(sa.DateTime, nullable=True) - typed_stamp_utc: so.Mapped[datetime | None] = so.mapped_column(sa.DateTime(timezone=True), nullable=True) + typed_stamp_utc: so.Mapped[datetime | None] = so.mapped_column( + sa.DateTime(timezone=True), nullable=True + ) typed_label: so.Mapped[str | None] = so.mapped_column(sa.String(10), nullable=True) @@ -242,25 +243,39 @@ class DemoDefaultKinds(Base, FileLoadableTableInterface): """One column for each kind of default a column can declare.""" __tablename__ = "demo_default_kinds" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} kind_id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) - kind_text: so.Mapped[str] = so.mapped_column(sa.String, nullable=False, server_default="it's text") + kind_text: so.Mapped[str] = so.mapped_column( + sa.String, nullable=False, server_default="it's text" + ) kind_int: so.Mapped[int] = so.mapped_column(sa.Integer, nullable=False, server_default="7") - kind_bool: so.Mapped[bool] = so.mapped_column(sa.Boolean, nullable=False, server_default=sa.true()) - kind_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False, server_default="2000-01-02") - kind_stamp: so.Mapped[datetime] = so.mapped_column(sa.DateTime, nullable=False, server_default="2000-01-02 10:30:00") - kind_expression: so.Mapped[str] = so.mapped_column(sa.String, nullable=False, server_default=sa.text("'expr'")) - kind_now: so.Mapped[datetime | None] = so.mapped_column(sa.DateTime, nullable=True, server_default=sa.func.current_timestamp()) + kind_bool: so.Mapped[bool] = so.mapped_column( + sa.Boolean, nullable=False, server_default=sa.true() + ) + kind_date: so.Mapped[date] = so.mapped_column( + sa.Date, nullable=False, server_default="2000-01-02" + ) + kind_stamp: so.Mapped[datetime] = so.mapped_column( + sa.DateTime, nullable=False, server_default="2000-01-02 10:30:00" + ) + kind_expression: so.Mapped[str] = so.mapped_column( + sa.String, nullable=False, server_default=sa.text("'expr'") + ) + kind_now: so.Mapped[datetime | None] = so.mapped_column( + sa.DateTime, nullable=True, server_default=sa.func.current_timestamp() + ) kind_py_int: so.Mapped[int | None] = so.mapped_column(sa.Integer, nullable=True, default=5) - kind_py_date: so.Mapped[date | None] = so.mapped_column(sa.Date, nullable=True, default=date(2001, 2, 3)) + kind_py_date: so.Mapped[date | None] = so.mapped_column( + sa.Date, nullable=True, default=date(2001, 2, 3) + ) class DemoBinaryRecord(Base, FileLoadableTableInterface): """A column type that the conversion policy has no handler for.""" __tablename__ = "demo_binary_record" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} demo_id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) demo_blob: so.Mapped[str | None] = so.mapped_column(sa.LargeBinary, nullable=True) @@ -270,7 +285,7 @@ class DemoKeyedRecord(Base, FileLoadableTableInterface): """A key the database assigns.""" __tablename__ = "demo_keyed_record" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} keyed_id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True, autoincrement=True) keyed_name: so.Mapped[str] = so.mapped_column(sa.String, nullable=False) @@ -280,7 +295,7 @@ class DemoBadDefaultRecord(Base, FileLoadableTableInterface): """A default that cannot be read as the type of its column.""" __tablename__ = "demo_bad_default_record" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} bad_id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) bad_count: so.Mapped[int | None] = so.mapped_column(sa.Integer, nullable=True, default="abc") diff --git a/tests/registry/test_model_descriptor.py b/tests/registry/test_model_descriptor.py index ac1e5ea..99730e4 100644 --- a/tests/registry/test_model_descriptor.py +++ b/tests/registry/test_model_descriptor.py @@ -1,7 +1,10 @@ +from typing import Any, ClassVar + +import pytest import sqlalchemy as sa import sqlalchemy.orm as so -import pytest from sqlalchemy.exc import NoInspectionAvailable + from orm_loader.registry.registry import ModelDescriptor from orm_loader.tables.orm_table import ORMTableBase @@ -10,7 +13,7 @@ def test_model_descriptor_from_model_basic(): class T(ORMTableBase, Base): - __tablename__ = "t" + __tablename__: ClassVar[str] = "t" id = sa.Column(sa.Integer, primary_key=True) a = sa.Column(sa.String) @@ -29,20 +32,19 @@ class Parent(ORMTableBase, Base): id = sa.Column(sa.Integer, primary_key=True) class Child(ORMTableBase, Base): - __tablename__ = "child" + __tablename__: ClassVar[str] = "child" id = sa.Column(sa.Integer, primary_key=True) parent_id = sa.Column(sa.Integer, sa.ForeignKey("parent.id")) desc = ModelDescriptor.from_model(Child) - assert desc.foreign_keys == { - "parent_id": ("parent", "id") - } + assert desc.foreign_keys == {"parent_id": ("parent", "id")} def test_model_descriptor_rejects_unmapped_class(): class NotMapped: pass + not_mapped: Any = NotMapped with pytest.raises(NoInspectionAvailable): - ModelDescriptor.from_model(NotMapped) # type: ignore + ModelDescriptor.from_model(not_mapped) diff --git a/tests/registry/test_registry.py b/tests/registry/test_registry.py index 81d4736..59479f7 100644 --- a/tests/registry/test_registry.py +++ b/tests/registry/test_registry.py @@ -1,20 +1,29 @@ import io -from orm_loader.registry.registry import ModelRegistry + import sqlalchemy as sa import sqlalchemy.orm as so -from orm_loader.tables.orm_table import ORMTableBase from orm_loader.registry.registry import ( - load_table_specs, - load_field_specs, - TableSpec, FieldSpec, + ModelRegistry, + TableSpec, + load_field_specs, + load_table_specs, ) +from orm_loader.tables.orm_table import ORMTableBase Base = so.declarative_base() + +def _table_spec(table_name: str, *, is_required: bool) -> TableSpec: + return TableSpec( + table_name=table_name, schema="cdm", is_required=is_required, description=table_name + ) + + class FakeResource: """Minimal stand-in for importlib.resources API.""" + def __init__(self, text: str): self._text = text @@ -54,7 +63,6 @@ def test_load_field_specs(): assert person_fields["gender_concept_id"].fk_table == "Concept" - def test_register_single_model(): class B(ORMTableBase, Base): __tablename__ = "b" @@ -88,9 +96,9 @@ def test_missing_required_tables(): reg = ModelRegistry(model_version="1.0") # Fake specs injected directly (unit-level test) - reg._table_specs = { # type: ignore - "person": type("S", (), {"is_required": True})(), - "visit": type("S", (), {"is_required": False})(), + reg._table_specs = { + "person": _table_spec("person", is_required=True), + "visit": _table_spec("visit", is_required=False), } assert reg.missing_required_tables() == {"person"} @@ -102,11 +110,11 @@ class R(ORMTableBase, Base): id = sa.Column(sa.Integer, primary_key=True) reg = ModelRegistry(model_version="1.0") - reg._table_specs = { # type: ignore - "r": type("S", (), {"is_required": True})(), - "x": type("S", (), {"is_required": True})(), + reg._table_specs = { + "r": _table_spec("r", is_required=True), + "x": _table_spec("x", is_required=True), } reg.register_model(R) assert reg.known_tables() == {"r", "x"} - assert reg.registered_tables() == {"r"} \ No newline at end of file + assert reg.registered_tables() == {"r"} diff --git a/tests/registry/test_validation_e2e.py b/tests/registry/test_validation_e2e.py index af5c003..86a9f46 100644 --- a/tests/registry/test_validation_e2e.py +++ b/tests/registry/test_validation_e2e.py @@ -1,14 +1,16 @@ import io + import sqlalchemy as sa import sqlalchemy.orm as so from orm_loader.registry.registry import ModelRegistry -from orm_loader.registry.validation_runner import ValidationRunner from orm_loader.registry.validation_presets import always_on_validators +from orm_loader.registry.validation_runner import ValidationRunner from orm_loader.tables.orm_table import ORMTableBase Base = so.declarative_base() + class FakeCSVResource: def __init__(self, text: str): self._text = text @@ -16,6 +18,7 @@ def __init__(self, text: str): def open(self, *_, **__): return io.StringIO(self._text) + def test_validation_runner_happy_path(): class Person(ORMTableBase, Base): __tablename__ = "person" @@ -105,10 +108,7 @@ class Concept(ORMTableBase, Base): runner = ValidationRunner(validators=always_on_validators()) report = runner.run(registry) - assert any( - i.message == "PRIMARY_KEY_COLUMN_NULLABLE" - for i in report.issues - ) + assert any(i.message == "PRIMARY_KEY_COLUMN_NULLABLE" for i in report.issues) def test_foreign_key_not_in_spec_warns(): @@ -128,8 +128,7 @@ class Child(ORMTableBase, Base): ) field_csv = FakeCSVResource( - - """cdmTableName,cdmFieldName,isRequired,cdmDatatype,isPrimaryKey,isForeignKey + """cdmTableName,cdmFieldName,isRequired,cdmDatatype,isPrimaryKey,isForeignKey Child,id,Yes,integer,Yes,No Child,parent_id,No,integer,No,No """ @@ -143,10 +142,8 @@ class Child(ORMTableBase, Base): runner = ValidationRunner(validators=always_on_validators()) report = runner.run(registry) - assert any( - i.message == "FOREIGN_KEY_NOT_IN_SPEC" - for i in report.issues - ) + assert any(i.message == "FOREIGN_KEY_NOT_IN_SPEC" for i in report.issues) + def test_fail_fast_stops_on_first_error(): class Drug(ORMTableBase, Base): diff --git a/tests/support.py b/tests/support.py index 0f38faf..975aee4 100644 --- a/tests/support.py +++ b/tests/support.py @@ -24,16 +24,22 @@ def interpret( arr = values if isinstance(values, (pa.Array, pa.ChunkedArray)) else pa.array(values) handler = policy.for_column(sa_type, table_name, column_name) out, invalid = handler.interpret(arr, sa_type, policy.nulls) - return pc.if_else(invalid, pa.scalar(None, out.type), out).to_pylist(), invalid.to_pylist() + return pc.if_else(invalid, pa.scalar(None, out.type), out).to_pylist(), [ + bool(flag) for flag in invalid.to_pylist() + ] -def read(raw: Any, sa_type: TypeEngine[Any], policy: ConversionPolicy | None = None, **column: Any) -> Any: +def read( + raw: Any, sa_type: TypeEngine[Any], policy: ConversionPolicy | None = None, **column: Any +) -> Any: """Return the value read from one raw value, failing if it is unreadable.""" values, invalid = interpret([raw], sa_type, policy, **column) assert not invalid[0], f"{raw!r} is unreadable as {sa_type!r}" return values[0] -def unreadable(raw: Any, sa_type: TypeEngine[Any], policy: ConversionPolicy | None = None, **column: Any) -> bool: +def unreadable( + raw: Any, sa_type: TypeEngine[Any], policy: ConversionPolicy | None = None, **column: Any +) -> bool: """Return True when one raw value cannot be read as the column type.""" return interpret([raw], sa_type, policy, **column)[1][0] diff --git a/tests/tables/test_id_allocator.py b/tests/tables/test_id_allocator.py index e6adaf8..669db68 100644 --- a/tests/tables/test_id_allocator.py +++ b/tests/tables/test_id_allocator.py @@ -1,10 +1,12 @@ from orm_loader.tables.allocators import IdAllocator + def test_allocator_sequence(): alloc = IdAllocator(start=5) assert alloc.next() == 6 assert alloc.next() == 7 + def test_allocator_reserve(): alloc = IdAllocator(start=0) r = alloc.reserve(3) diff --git a/tests/tables/test_serialisable_table.py b/tests/tables/test_serialisable_table.py index ce989e8..c9a1733 100644 --- a/tests/tables/test_serialisable_table.py +++ b/tests/tables/test_serialisable_table.py @@ -1,23 +1,25 @@ -from orm_loader.tables.serialisable_table import SerialisableTableInterface import sqlalchemy as sa import sqlalchemy.orm as so from sqlalchemy.orm import declarative_base +from orm_loader.tables.serialisable_table import SerialisableTableInterface + Base = declarative_base() + class ExampleTable(SerialisableTableInterface, Base): __tablename__ = "example" - id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) + id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) a: so.Mapped[int] = so.mapped_column(sa.Integer, nullable=False) b: so.Mapped[int | None] = so.mapped_column(sa.Integer, nullable=True) def test_to_dict_excludes_nulls(): obj = ExampleTable() - obj.id=1 - obj.a=10 - obj.b=None + obj.id = 1 + obj.a = 10 + obj.b = None out = obj.to_dict() @@ -30,32 +32,32 @@ def test_to_dict_excludes_nulls(): def test_to_dict_includes_nulls_when_requested(): obj = ExampleTable() - obj.id=1 - obj.a=10 - obj.b=None + obj.id = 1 + obj.a = 10 + obj.b = None out = obj.to_dict(include_nulls=True) assert out["b"] is None -def test_fingerprint_is_stable(): + +def test_fingerprint_is_stable(): obj = ExampleTable() - obj.id=1 - obj.a=10 - obj.b=20 + obj.id = 1 + obj.a = 10 + obj.b = 20 fp1 = obj.fingerprint() fp2 = obj.fingerprint() assert fp1 == fp2 + def test_to_dict_only_and_exclude(): obj = ExampleTable() - obj.id=1 - obj.a=10 - obj.b=20 + obj.id = 1 + obj.a = 10 + obj.b = 20 assert obj.to_dict(only={"a"}) == {"a": 10} assert obj.to_dict(exclude={"b"}) == {"id": 1, "a": 10} - - diff --git a/tests/tables/test_typing_protocols.py b/tests/tables/test_typing_protocols.py index 49a6a5f..afd397c 100644 --- a/tests/tables/test_typing_protocols.py +++ b/tests/tables/test_typing_protocols.py @@ -2,17 +2,19 @@ import sqlalchemy as sa import sqlalchemy.orm as so + +from orm_loader.tables.loadable_table import FileLoadableTableInterface from orm_loader.tables.orm_table import ORMTableBase from orm_loader.tables.serialisable_table import SerialisableTableInterface -from orm_loader.tables.loadable_table import FileLoadableTableInterface from orm_loader.tables.typing import ( - ORMTableProtocol, FileTableProtocol, + ORMTableProtocol, SerializedTableProtocol, ) Base = so.declarative_base() + def test_orm_table_protocol_conformance(): class Q(ORMTableBase, Base): __tablename__ = "f" @@ -24,12 +26,14 @@ class Q(ORMTableBase, Base): assert callable(Q.pk_names) assert callable(Q.model_columns) + def test_every_protocol_member_is_a_plain_classmethod(): for name in ("column_defaults", "sql_expression_defaults"): descriptor = vars(ORMTableProtocol)[name] assert isinstance(descriptor, classmethod) assert inspect.isfunction(descriptor.__func__) + def test_serialized_table_protocol_conformance(): class G(SerialisableTableInterface, ORMTableBase, Base): __tablename__ = "g" @@ -46,12 +50,13 @@ class G(SerialisableTableInterface, ORMTableBase, Base): assert callable(obj.to_json) assert callable(obj.fingerprint) + def test_file_table_protocol_conformance(): class P(FileLoadableTableInterface, ORMTableBase, Base): __tablename__ = "p" id = sa.Column(sa.Integer, primary_key=True) - assert isinstance(P, FileTableProtocol) + assert isinstance(P, FileTableProtocol) # Required class-level API assert callable(P.load_file) - assert callable(P.merge_from_staging) \ No newline at end of file + assert callable(P.merge_from_staging) diff --git a/tests/test_optional_deps.py b/tests/test_optional_deps.py index cd4c06c..bbc1968 100644 --- a/tests/test_optional_deps.py +++ b/tests/test_optional_deps.py @@ -1,5 +1,4 @@ -"""Tests that orm_loader stays importable without optional extras installed -""" +"""Tests that orm_loader stays importable without optional extras installed""" from __future__ import annotations @@ -7,7 +6,9 @@ import sys -def _run_blocked(import_statement: str, *, blocked_module: str = "duckdb_sqlalchemy") -> subprocess.CompletedProcess[str]: +def _run_blocked( + import_statement: str, *, blocked_module: str = "duckdb_sqlalchemy" +) -> subprocess.CompletedProcess[str]: code = f"import sys; sys.modules['{blocked_module}'] = None; {import_statement}" return subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=False) diff --git a/uv.lock b/uv.lock index 0811e6d..1a4b373 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -680,9 +680,9 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.3" }, { name = "python-dotenv", marker = "extra == 'dev'" }, { name = "requests", marker = "extra == 'dev'", specifier = ">=2.33.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.14.11" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.17.1" }, { name = "sqlalchemy", specifier = ">=2.0.45" }, - { name = "ty", marker = "extra == 'dev'", specifier = ">=0.0.59" }, + { name = "ty", marker = "extra == 'dev'", specifier = "==0.0.90" }, ] provides-extras = ["dev", "duckdb", "postgres"] @@ -1125,28 +1125,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.14.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d4/77/9a7fe084d268f8855d493e5031ea03fa0af8cc05887f638bf1c4e3363eb8/ruff-0.14.11.tar.gz", hash = "sha256:f6dc463bfa5c07a59b1ff2c3b9767373e541346ea105503b4c0369c520a66958", size = 5993417, upload-time = "2026-01-08T19:11:58.322Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/a6/a4c40a5aaa7e331f245d2dc1ac8ece306681f52b636b40ef87c88b9f7afd/ruff-0.14.11-py3-none-linux_armv6l.whl", hash = "sha256:f6ff2d95cbd335841a7217bdfd9c1d2e44eac2c584197ab1385579d55ff8830e", size = 12951208, upload-time = "2026-01-08T19:12:09.218Z" }, - { url = "https://files.pythonhosted.org/packages/5c/5c/360a35cb7204b328b685d3129c08aca24765ff92b5a7efedbdd6c150d555/ruff-0.14.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6f6eb5c1c8033680f4172ea9c8d3706c156223010b8b97b05e82c59bdc774ee6", size = 13330075, upload-time = "2026-01-08T19:12:02.549Z" }, - { url = "https://files.pythonhosted.org/packages/1b/9e/0cc2f1be7a7d33cae541824cf3f95b4ff40d03557b575912b5b70273c9ec/ruff-0.14.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f2fc34cc896f90080fca01259f96c566f74069a04b25b6205d55379d12a6855e", size = 12257809, upload-time = "2026-01-08T19:12:00.366Z" }, - { url = "https://files.pythonhosted.org/packages/a7/e5/5faab97c15bb75228d9f74637e775d26ac703cc2b4898564c01ab3637c02/ruff-0.14.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53386375001773ae812b43205d6064dae49ff0968774e6befe16a994fc233caa", size = 12678447, upload-time = "2026-01-08T19:12:13.899Z" }, - { url = "https://files.pythonhosted.org/packages/1b/33/e9767f60a2bef779fb5855cab0af76c488e0ce90f7bb7b8a45c8a2ba4178/ruff-0.14.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a697737dce1ca97a0a55b5ff0434ee7205943d4874d638fe3ae66166ff46edbe", size = 12758560, upload-time = "2026-01-08T19:11:42.55Z" }, - { url = "https://files.pythonhosted.org/packages/eb/84/4c6cf627a21462bb5102f7be2a320b084228ff26e105510cd2255ea868e5/ruff-0.14.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6845ca1da8ab81ab1dce755a32ad13f1db72e7fba27c486d5d90d65e04d17b8f", size = 13599296, upload-time = "2026-01-08T19:11:30.371Z" }, - { url = "https://files.pythonhosted.org/packages/88/e1/92b5ed7ea66d849f6157e695dc23d5d6d982bd6aa8d077895652c38a7cae/ruff-0.14.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:e36ce2fd31b54065ec6f76cb08d60159e1b32bdf08507862e32f47e6dde8bcbf", size = 15048981, upload-time = "2026-01-08T19:12:04.742Z" }, - { url = "https://files.pythonhosted.org/packages/61/df/c1bd30992615ac17c2fb64b8a7376ca22c04a70555b5d05b8f717163cf9f/ruff-0.14.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:590bcc0e2097ecf74e62a5c10a6b71f008ad82eb97b0a0079e85defe19fe74d9", size = 14633183, upload-time = "2026-01-08T19:11:40.069Z" }, - { url = "https://files.pythonhosted.org/packages/04/e9/fe552902f25013dd28a5428a42347d9ad20c4b534834a325a28305747d64/ruff-0.14.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53fe71125fc158210d57fe4da26e622c9c294022988d08d9347ec1cf782adafe", size = 14050453, upload-time = "2026-01-08T19:11:37.555Z" }, - { url = "https://files.pythonhosted.org/packages/ae/93/f36d89fa021543187f98991609ce6e47e24f35f008dfe1af01379d248a41/ruff-0.14.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a35c9da08562f1598ded8470fcfef2afb5cf881996e6c0a502ceb61f4bc9c8a3", size = 13757889, upload-time = "2026-01-08T19:12:07.094Z" }, - { url = "https://files.pythonhosted.org/packages/b7/9f/c7fb6ecf554f28709a6a1f2a7f74750d400979e8cd47ed29feeaa1bd4db8/ruff-0.14.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0f3727189a52179393ecf92ec7057c2210203e6af2676f08d92140d3e1ee72c1", size = 13955832, upload-time = "2026-01-08T19:11:55.064Z" }, - { url = "https://files.pythonhosted.org/packages/db/a0/153315310f250f76900a98278cf878c64dfb6d044e184491dd3289796734/ruff-0.14.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:eb09f849bd37147a789b85995ff734a6c4a095bed5fd1608c4f56afc3634cde2", size = 12586522, upload-time = "2026-01-08T19:11:35.356Z" }, - { url = "https://files.pythonhosted.org/packages/2f/2b/a73a2b6e6d2df1d74bf2b78098be1572191e54bec0e59e29382d13c3adc5/ruff-0.14.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c61782543c1231bf71041461c1f28c64b961d457d0f238ac388e2ab173d7ecb7", size = 12724637, upload-time = "2026-01-08T19:11:47.796Z" }, - { url = "https://files.pythonhosted.org/packages/f0/41/09100590320394401cd3c48fc718a8ba71c7ddb1ffd07e0ad6576b3a3df2/ruff-0.14.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:82ff352ea68fb6766140381748e1f67f83c39860b6446966cff48a315c3e2491", size = 13145837, upload-time = "2026-01-08T19:11:32.87Z" }, - { url = "https://files.pythonhosted.org/packages/3b/d8/e035db859d1d3edf909381eb8ff3e89a672d6572e9454093538fe6f164b0/ruff-0.14.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:728e56879df4ca5b62a9dde2dd0eb0edda2a55160c0ea28c4025f18c03f86984", size = 13850469, upload-time = "2026-01-08T19:12:11.694Z" }, - { url = "https://files.pythonhosted.org/packages/4e/02/bb3ff8b6e6d02ce9e3740f4c17dfbbfb55f34c789c139e9cd91985f356c7/ruff-0.14.11-py3-none-win32.whl", hash = "sha256:337c5dd11f16ee52ae217757d9b82a26400be7efac883e9e852646f1557ed841", size = 12851094, upload-time = "2026-01-08T19:11:45.163Z" }, - { url = "https://files.pythonhosted.org/packages/58/f1/90ddc533918d3a2ad628bc3044cdfc094949e6d4b929220c3f0eb8a1c998/ruff-0.14.11-py3-none-win_amd64.whl", hash = "sha256:f981cea63d08456b2c070e64b79cb62f951aa1305282974d4d5216e6e0178ae6", size = 14001379, upload-time = "2026-01-08T19:11:52.591Z" }, - { url = "https://files.pythonhosted.org/packages/c4/1c/1dbe51782c0e1e9cfce1d1004752672d2d4629ea46945d19d731ad772b3b/ruff-0.14.11-py3-none-win_arm64.whl", hash = "sha256:649fb6c9edd7f751db276ef42df1f3df41c38d67d199570ae2a7bd6cbc3590f0", size = 12938644, upload-time = "2026-01-08T19:11:50.027Z" }, +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e7/e6d5749983b020e6cb1055abec0a6f361a8e94f0184cdcf56d4760995f02/ruff-0.17.1.tar.gz", hash = "sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f", size = 5323349, upload-time = "2026-10-15T15:23:18.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/78/90c46ca6250709f7c37e397b60d06061adee8d0f7d13959ed6a8e49b14fd/ruff-0.17.1-py3-none-linux_armv6l.whl", hash = "sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a", size = 10147528, upload-time = "2026-10-15T15:22:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/a2e28bc35b083ae51fa07dfba5280a667c8b01ecaee7802fc8849dcd5c46/ruff-0.17.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb", size = 10225974, upload-time = "2026-10-15T15:22:36.328Z" }, + { url = "https://files.pythonhosted.org/packages/8e/71/7a9029a81c8206ee01a98f6cfeeb56e6c095af86857581a37045c46d7e94/ruff-0.17.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74", size = 9270361, upload-time = "2026-10-15T15:22:38.651Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/e075e9c917cb0575297880c96768304e881240171c870cbd8d9b7b54bec2/ruff-0.17.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978", size = 9938300, upload-time = "2026-10-15T15:22:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/87/27/0b4bb926a49de2383f161272b4d863571a4c7ecdbf595a6bc58813a13367/ruff-0.17.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640", size = 9948464, upload-time = "2026-10-15T15:22:43.681Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/518d57e0e02aecb8ea4acac6b57b51a53173d25fa734952bfdd7d531e88e/ruff-0.17.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1", size = 10638639, upload-time = "2026-10-15T15:22:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/01/82/d89a4e498ade59e6cee52d67f8da569019372212b87468c1fdeae884601c/ruff-0.17.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05", size = 11337238, upload-time = "2026-10-15T15:22:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7c/3b0fdef9352656317c977086fe1d802902406e4f2a0d8578da90c0e6d1c5/ruff-0.17.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785", size = 10896327, upload-time = "2026-10-15T15:22:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/5566290219708ce2928f3a29c4f843a1185b341ec6111c4631a33c5ceced/ruff-0.17.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189", size = 10350811, upload-time = "2026-10-15T15:22:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/ca8462dabb3c413990d49703ba306c30c862195a75448d080fa8fcce9775/ruff-0.17.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467", size = 10569994, upload-time = "2026-10-15T15:22:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ad/dd1864c1f1db17c8cfb297ce70b352388e6977620f66f7182c52855407df/ruff-0.17.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab", size = 10059672, upload-time = "2026-10-15T15:23:00.084Z" }, + { url = "https://files.pythonhosted.org/packages/65/45/3f56cf1d2168c2165cdd5ac6f9668aee66a4cfdd4caf2b2cc5ccb2b1d568/ruff-0.17.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca", size = 9947803, upload-time = "2026-10-15T15:23:03.132Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4b/5457cb2be4c1d1ea5e919558131a0f197a4a454a0530d9e30820e8206e4c/ruff-0.17.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4", size = 10351807, upload-time = "2026-10-15T15:23:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/49/fb/db15b4f8fa24a1e36b99e1401f42ef6df16ee299a48670d5d262b771e7d7/ruff-0.17.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a", size = 10765716, upload-time = "2026-10-15T15:23:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e8/1e186578076aff7a80735f625005fde678412ab609cd323ed182eb7d6b87/ruff-0.17.1-py3-none-win32.whl", hash = "sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a", size = 9895093, upload-time = "2026-10-15T15:23:10.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2e/da3d9589bae8e8b58f3a48bd5bc0b980232fb14a83bf02c3be7660f87c1b/ruff-0.17.1-py3-none-win_amd64.whl", hash = "sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7", size = 10097955, upload-time = "2026-10-15T15:23:13.309Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/0f2a4dea06a5e12c1191ee3d96b1e1233797d0d5e18f97a2b909a95a9836/ruff-0.17.1-py3-none-win_arm64.whl", hash = "sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4", size = 9853542, upload-time = "2026-10-15T15:23:16.05Z" }, ] [[package]] @@ -1231,27 +1230,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.61" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/63/6944925d0fe9a4bb9cc744e6c045a42bbd2ee4654c103190674577a36c3f/ty-0.0.61.tar.gz", hash = "sha256:acbf0d914cc7e2e57ccc440036af36114819e2a604a5ffb554e72e4ca7dd65a2", size = 6234957, upload-time = "2026-07-18T01:39:54.696Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/cf/044f31523e2768e3e64b0ca2ec32f70b3a731d4a2caa6ea110baf26e251c/ty-0.0.61-py3-none-linux_armv6l.whl", hash = "sha256:148779b8675eac93f40ec58bd70037fe67537117f20a23272264f8f136d41336", size = 11891448, upload-time = "2026-07-18T01:39:18.449Z" }, - { url = "https://files.pythonhosted.org/packages/d2/55/558cfe76b65d91d1854bbfac336020bd42fd887caa632d845d13c0c539eb/ty-0.0.61-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:08217382b3385808ee7288501ea3214b32631b08d1fd091ece6799b0c95264c5", size = 11602442, upload-time = "2026-07-18T01:39:20.914Z" }, - { url = "https://files.pythonhosted.org/packages/27/be/78c0ae6634cd606a68e5b46b338db427a48a1800c96a749b2d2f7a702e03/ty-0.0.61-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6d99c729011b47dec20e78a32ac9c8f6defd4cf62f7bb851bbccf70dde6cee50", size = 11125286, upload-time = "2026-07-18T01:39:22.893Z" }, - { url = "https://files.pythonhosted.org/packages/a4/18/a40793962f1b6337938ddb0bca7496b54e70879e23b4d2cc8dfd7e5d1af3/ty-0.0.61-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cda607978ae271b77e51c947663218bce635c3507e256865444b10c37cdb60d", size = 11663403, upload-time = "2026-07-18T01:39:25.017Z" }, - { url = "https://files.pythonhosted.org/packages/98/c1/7879244da5b30407dc368946d36be5024380073408b079f144ffe034030e/ty-0.0.61-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0d78f160a0f9434d570cdcdbc4dafba1f6aac3c47a32f9f63995b3cb55ffe4b6", size = 11715250, upload-time = "2026-07-18T01:39:27.045Z" }, - { url = "https://files.pythonhosted.org/packages/35/c4/8a4637cd58abd37f315dd515e24c582986cb1bfdf2edc4786882f5a4f69a/ty-0.0.61-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09aeab4800b36e93e4ce918699004da642d74988cac920b7592a6a2b9be6611c", size = 12393876, upload-time = "2026-07-18T01:39:29.197Z" }, - { url = "https://files.pythonhosted.org/packages/27/4b/27e7c640b1272743503229aa17ae2167a538040c4716a2fa1777c2b34fea/ty-0.0.61-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dccc8136df44142a109953a168be17b4915c99876b047d0b6672c31dae939bdf", size = 12958187, upload-time = "2026-07-18T01:39:31.308Z" }, - { url = "https://files.pythonhosted.org/packages/3a/f5/70eaaefb6081fb0a8115cff66fbfaa20dafac8c646df2477adad95a59de2/ty-0.0.61-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:220760c2d13a887d027ee1093172c24ac35b6e634805329c93a30908ae4d3f5c", size = 12560101, upload-time = "2026-07-18T01:39:33.35Z" }, - { url = "https://files.pythonhosted.org/packages/e3/5a/17bae3b6429b5c479dc6c1e344d34e1f79efbc27531f15f3ee5b5da63745/ty-0.0.61-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:effefbb89da7128d18059529d1c2ea390fe7f1f3882690d257ca2143d49a0c34", size = 12225389, upload-time = "2026-07-18T01:39:35.436Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0e/2ac380ba20d6395542c8df1d6fa4f00e2aead784c2e6aaefa1e02ed0610c/ty-0.0.61-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ba8b28a5ef811d5bb6461e37d76110c06fd20487474865c323d3d18b08b972b2", size = 12548403, upload-time = "2026-07-18T01:39:37.556Z" }, - { url = "https://files.pythonhosted.org/packages/b0/e5/7da4b73e825e1a9808c26d68b0156e9a37aede1846191210dfffb8c64042/ty-0.0.61-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:88ecd6d9b05e8174b1860dac9bd3e188d6cef5702b0d3239fd9f94f6ac73a29d", size = 11621813, upload-time = "2026-07-18T01:39:39.919Z" }, - { url = "https://files.pythonhosted.org/packages/9b/3c/5b58015e998cd0d89b17a463b6321421457d86d987574e8dac65ddfceba3/ty-0.0.61-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb0cdfe4c48542ffb9a1139825dfa3d4aae49e96e966682ef7da762ab97831ff", size = 11734101, upload-time = "2026-07-18T01:39:42.097Z" }, - { url = "https://files.pythonhosted.org/packages/a6/21/294f4cc819b7b12ed659fd860e5cdfbd592d4c768c8f23596685dbc43e6b/ty-0.0.61-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dff03873c0c3d0b44738f8b6d403b0756a31cf54c65136397df7624c6159b1f0", size = 11988401, upload-time = "2026-07-18T01:39:44.183Z" }, - { url = "https://files.pythonhosted.org/packages/2e/26/0f96f79fdac118521a9771e9eef3f9b3f447d647b2c77953e80a1715c7e8/ty-0.0.61-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a9210e80e3d41c1dfc751e9e8e0980272f475031fafd0fb0f48aee233c78da03", size = 12330624, upload-time = "2026-07-18T01:39:46.662Z" }, - { url = "https://files.pythonhosted.org/packages/e3/08/1e62d1bca5c0cebdc7a34db1f4b61557aab85961cedd56953dd2c32d3e66/ty-0.0.61-py3-none-win32.whl", hash = "sha256:e3e1fe06f49a5492a922a5df2739834aa5ee978c7dd10414119dc8755cc40c9c", size = 11313991, upload-time = "2026-07-18T01:39:48.761Z" }, - { url = "https://files.pythonhosted.org/packages/26/f1/d8e33b3aeb36b73d81ae34d10e46ec4abf506d68f4e0a1491a76a593dd42/ty-0.0.61-py3-none-win_amd64.whl", hash = "sha256:25f2291169e0298fcdbba1b1fea64f8207a6c1908dddef32346fd5e3e6ac9221", size = 12311717, upload-time = "2026-07-18T01:39:50.881Z" }, - { url = "https://files.pythonhosted.org/packages/e1/14/7caec26d93a943c0e7d15eb7374644508d08cbd387d112b722b12d14e044/ty-0.0.61-py3-none-win_arm64.whl", hash = "sha256:3e496f7698bc4b5bbb1eb66d8b5799ba87596d88d36604ca359083893fa2fc49", size = 11693485, upload-time = "2026-07-18T01:39:52.73Z" }, +version = "0.0.90" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/b1/613b5534624442a9118828d7a9e54d50b16f0f5c8b67c80eb6623f33b921/ty-0.0.90.tar.gz", hash = "sha256:25cef63ec07a0254c2f5d885a63b001d0bfa437b20caa0a408f4f52e66b133e7", size = 7996206, upload-time = "2026-10-20T14:25:19.477Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/33/0e3395afa2c31c0e8e91547629858d2a5de213c69af237e6e50f37545565/ty-0.0.90-py3-none-linux_armv6l.whl", hash = "sha256:0dedb54977b40bc86638fec2c90eefd593f3a6683003a86b7e6cb67c17bcc476", size = 14281945, upload-time = "2026-10-20T14:24:21.018Z" }, + { url = "https://files.pythonhosted.org/packages/d7/fd/5d632b2f88d5c18991a4936da8643c7f204cfbc4980f8c7cd9080af988ad/ty-0.0.90-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3d7c30ceaac76dfbfcf36b72d36272d21c3898e98fc4de296aa9cbae956dec9d", size = 13786355, upload-time = "2026-10-20T14:24:24.392Z" }, + { url = "https://files.pythonhosted.org/packages/79/08/0d2f37bb4f766ab9630f01b07f4d11f1f7a6a6188f920d5ca9fa8a4676b1/ty-0.0.90-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c0b7e4d9ae0939294f0a18652de9ad7d81da2d7f567cf93f1edda1263008c124", size = 12856535, upload-time = "2026-10-20T14:24:27.386Z" }, + { url = "https://files.pythonhosted.org/packages/e4/77/70658f6b80c345ae0c12f2192a5cc89b8ca13eccfe06616b0fac3806c261/ty-0.0.90-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76c4a095a86a132df374e652894fe08392aeb1601d6347e1eb4b58808094a302", size = 13696342, upload-time = "2026-10-20T14:24:30.413Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3e/693049f93653d2f6412bb0c2c0af1d01675e31643af29bef7177d3ff7e79/ty-0.0.90-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4e24a8aa07c05d9d14648289e79a88b7c7c2c522c58b53bb5f466a6ba4704e22", size = 14014180, upload-time = "2026-10-20T14:24:33.783Z" }, + { url = "https://files.pythonhosted.org/packages/27/08/19e373f97501eaadc8fc42e8c07cd54a64e375fbcf1470797be13590d5ba/ty-0.0.90-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40c3ea8fc1dfcf7502f71b93b12e80da1f7a8a5fd7e55ec9feabcf4ac39f72ea", size = 14935189, upload-time = "2026-10-20T14:24:37.459Z" }, + { url = "https://files.pythonhosted.org/packages/33/54/3331984f2c896a7383b6b18c86e968a63dd046baf503119a8e8e7682c752/ty-0.0.90-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1af75532bbe3e305089c30806fa5e804f8b01de231c1fa07918f7f3b335626da", size = 15311482, upload-time = "2026-10-20T14:24:40.87Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f3/179acf8d730c2c864767ffcc9081f5ea6fdb1b34ef711f1bd226200e9cff/ty-0.0.90-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f09449b34d837a044dab043aeb2b4b6ff2d2cef5eaf8116c7c19d34cc2a16a21", size = 15278844, upload-time = "2026-10-20T14:24:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/71/7e/661bc2fa20e0ac84c4d5f92e4c098ec667a0e920851a6d10c10a0251f0dc/ty-0.0.90-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc868ffa7dfcb3169eea3ba7d634e5a3283c7133fb932c6e47a2e16620d26ca5", size = 14558774, upload-time = "2026-10-20T14:24:47.814Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2e/3f65f61f4545f75255373be7646a606e004bb4cdc851e236bf17ca94bdc2/ty-0.0.90-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e2405206533dc896a2b711e781f2955d5883d85536f4df0ac262fca6517103c2", size = 14788158, upload-time = "2026-10-20T14:24:51.223Z" }, + { url = "https://files.pythonhosted.org/packages/1e/10/3c91a529591667b89d63c86806bfe2ac56d6989c7e0a09ee0964b7512deb/ty-0.0.90-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0dca4a29e8778285d2fc204af19254571486106fd14aed60769bfd272c795f6f", size = 13687177, upload-time = "2026-10-20T14:24:54.611Z" }, + { url = "https://files.pythonhosted.org/packages/21/b2/1b6d48606c67614bfb0c8e61a5d67a053ca1e3c5956ddb584f8569b8dc3d/ty-0.0.90-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d004b8e4e898fead929eceea669b73936444671748e0dd44e624cc109346c0dd", size = 14027780, upload-time = "2026-10-20T14:24:59.045Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c4/a0771debf725312524ca1434f8a146d37e573d45257ca4bebb9cc8e35bbc/ty-0.0.90-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6be182da4281b4581c4a1b27c6a0caebeed7212e4bdc98c6b8fe0d9bc36dea0", size = 14355890, upload-time = "2026-10-20T14:25:03.034Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4c/036724b6936ffc38e28ff5e20ab7972d609c65a3dd9645f3d10f57cfd16f/ty-0.0.90-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:0ba6fe0a24ff4cbc5034f3aa1c88b3ad1397315dd3369357e694fd157a11864e", size = 14692306, upload-time = "2026-10-20T14:25:07.461Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2a/c74fd5fc3fb5b046fe06a9067cabf5417b73e28249dc9af90ef72700f8fd/ty-0.0.90-py3-none-win32.whl", hash = "sha256:4c46a79caf84c4da0f7293e16bd769bd467c7cbc38015cac7608ab4e5ec8ba9c", size = 13235434, upload-time = "2026-10-20T14:25:10.545Z" }, + { url = "https://files.pythonhosted.org/packages/d2/54/be5cd6b17d8087a5e4c26892311c07e39d42b66c925b0f06cfc51a9ce83b/ty-0.0.90-py3-none-win_amd64.whl", hash = "sha256:8d77c16d20201b7a07ccaafb610eaf60488783dbba9865c4f9182e584f1426f8", size = 13855668, upload-time = "2026-10-20T14:25:13.648Z" }, + { url = "https://files.pythonhosted.org/packages/33/68/6dea089a897e8b314504eecc2633de3c8fe14f70e71a35606d14a2ba169e/ty-0.0.90-py3-none-win_arm64.whl", hash = "sha256:bfb63e74eeedb0a29d49cf58d611c540bfdfc0975dfce26efc6575bcf439208d", size = 13330853, upload-time = "2026-10-20T14:25:16.642Z" }, ] [[package]]