feat(server): sort and search guards for huge sources and a searched count memo (rows-first p37)

buckaroo/customizations/pd_stats_v2.py

@@ -101,6 +101,25 @@ def _type(is_bool: bool, is_numeric: bool, is_float: bool,
     return "obj"
 
 
+def with_type(flags: dict) -> dict:
+    """``flags``, the ``is_*`` keys a typing stat returns, plus the ``_type``
+    derived from them through the ``_type`` stat."""
+    return {**flags, '_type': _type(**{key.name: flags[key.name] for key in _type._stat_func.requires})}
+
+
+def schema_stats(ser: pd.Series) -> dict:
+    """The stats a column's dtype alone determines: ``dtype``, the ``is_*``
+    flags and ``_type``. ``typing_stats`` runs on a zero-row slice, so no value
+    is read, and ``memory_usage`` is left out because it measures the data.
+
+    An ``object`` column gives the typing nothing to go on, and pandas calls an
+    empty one a string, so every object column types as ``string`` here. Full
+    stats look at the values and may say ``obj``."""
+    flags = typing_stats(ser.iloc[:0])
+    del flags['memory_usage']
+    return with_type(flags)
+
+
 # ============================================================
 # Base Summary Stats (replaces DefaultSummaryStats ColAnalysis)
 # ============================================================

buckaroo/customizations/pl_stats_v2.py

@@ -21,7 +21,7 @@
 from buckaroo.pluggable_analysis_framework.column_filters import is_numeric_not_bool
 
 # Reused unchanged from pd_stats_v2 — operate on stat dict, not raw series
-from buckaroo.customizations.pd_stats_v2 import (_type, computed_default_summary_stats, histogram, cleaning_gen_ops, BaseSummaryResult, NumericStatsResult, HistogramSeriesResult)
+from buckaroo.customizations.pd_stats_v2 import (_type, computed_default_summary_stats, histogram, cleaning_gen_ops, with_type, BaseSummaryResult, NumericStatsResult, HistogramSeriesResult)
 
 
 # ============================================================
@@ -44,16 +44,27 @@ def pl_orig_col_name(ser: RawSeries) -> Any:
      'is_time': bool, 'is_decimal': bool, 'is_binary': bool, 'memory_usage': int})
 
 
-@stat()
-def pl_typing_stats(ser: RawSeries) -> PlTypingResult:
-    """Compute dtype and type flags for a polars column."""
-    dt = ser.dtype
+def pl_dtype_typing(dt: pl.DataType) -> dict:
+    """dtype string and type flags for a polars dtype. Needs a schema only,
+    which is what ``pl_schema_stats`` has to go on."""
     return {'dtype': str(dt), 'is_numeric': dt.is_numeric() and dt.base_type() is not pl.Decimal,
         'is_integer': dt.is_integer(), 'is_float': dt.is_float(), 'is_bool': dt == pl.Boolean,
         'is_datetime': dt.is_temporal() and dt not in (pl.Duration, pl.Time), 'is_timedelta': dt == pl.Duration,
         'is_string': dt in (pl.Utf8, pl.String), 'is_categorical': dt == pl.Categorical or isinstance(dt, pl.Enum),
         'is_period': False, 'is_interval': False, 'is_time': dt == pl.Time, 'is_decimal': dt.base_type() is pl.Decimal,
-        'is_binary': dt == pl.Binary, 'memory_usage': ser.estimated_size()}
+        'is_binary': dt == pl.Binary}
+
+
+@stat()
+def pl_typing_stats(ser: RawSeries) -> PlTypingResult:
+    """Compute dtype and type flags for a polars column."""
+    return {**pl_dtype_typing(ser.dtype), 'memory_usage': ser.estimated_size()}
+
+
+def pl_schema_stats(ser: pl.Series) -> dict:
+    """The stats a polars column's dtype alone determines: ``dtype``, the
+    ``is_*`` flags and ``_type``. Reads ``ser.dtype`` and no value."""
+    return with_type(pl_dtype_typing(ser.dtype))
 
 
 # ============================================================

buckaroo/customizations/xorq_stats_v2.py

@@ -138,6 +138,16 @@ def _type(is_bool: bool, is_integer: bool, is_float: bool, is_datetime: bool, is
     return "obj"
 
 
+def schema_stats(dtype: str) -> dict:
+    """The stats a column's dtype alone determines: ``dtype``, the ``is_*``
+    flags and ``_type``. This is what the xorq dataflow's schema tier
+    publishes, derived through the same stats the pipeline runs."""
+    flags = typing_stats(dtype)
+    return {"dtype": dtype, **flags, "_type": _type(
+        flags["is_bool"], flags["is_integer"], flags["is_float"], flags["is_datetime"],
+        flags["is_string"])}
+
+
 # ============================================================
 # Batched aggregates — one ibis.Expr each, folded into the batch query
 # ============================================================