From e99bdff3988d726fb9973d5aa5a373c9dc0f4ff4 Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Thu, 20 Aug 2026 10:56:54 -0600 Subject: [PATCH 1/4] Add depth-averaged and MAC flow solvers --- docs/architecture/0022-brinkman-flow.md | 78 ++++ docs/architecture/0023-mac-stokes.md | 81 ++++ docs/architecture/README.md | 2 + python/src/cellmodeller2/flow.py | 425 +++++++++++++++++++++ python/src/cellmodeller2/flow_reference.py | 116 ++++++ python/src/cellmodeller2/masks.py | 248 ++++++++++++ python/src/cellmodeller2/microfluidics.py | 364 ++++++++++++++++++ python/src/cellmodeller2/stokes.py | 408 ++++++++++++++++++++ python/tests/test_flow.py | 359 +++++++++++++++++ python/tests/test_stokes.py | 268 +++++++++++++ scripts/run_flow_benchmarks.py | 276 +++++++++++++ 11 files changed, 2625 insertions(+) create mode 100644 docs/architecture/0022-brinkman-flow.md create mode 100644 docs/architecture/0023-mac-stokes.md create mode 100644 python/src/cellmodeller2/flow.py create mode 100644 python/src/cellmodeller2/flow_reference.py create mode 100644 python/src/cellmodeller2/masks.py create mode 100644 python/src/cellmodeller2/microfluidics.py create mode 100644 python/src/cellmodeller2/stokes.py create mode 100644 python/tests/test_flow.py create mode 100644 python/tests/test_stokes.py create mode 100644 scripts/run_flow_benchmarks.py diff --git a/docs/architecture/0022-brinkman-flow.md b/docs/architecture/0022-brinkman-flow.md new file mode 100644 index 0000000..1fbd60e --- /dev/null +++ b/docs/architecture/0022-brinkman-flow.md @@ -0,0 +1,78 @@ +# ADR 0022: steady Hele-Shaw-Brinkman flow solve + +- Status: accepted +- Date: 2026-08-16 + +## Context + +Device flow fields are authored analytically, which is exact only for straight channels. A +junction, bend, pillar array, or partially blocking colony needs a numerical solve. At +microfluidic scale the Reynolds number is around `1e-4`, so the governing momentum balance is +inertia-free and linear, and for a fixed geometry the flow is steady: it can be computed once +in the authoring layer and handed to the engine as the existing face-staggered velocity +field, with no fluid solver in the simulation loop. + +## Decision + +`cellmodeller2.flow` solves the steady depth-averaged Darcy-Brinkman problem + +```text +div(m(x) grad p) = 0 v_face = -m_face * dp/dn +``` + +over the fluid voxels of a signal grid, where `m` is a per-voxel mobility field and `m_face` +is the harmonic mean of the two adjacent voxel mobilities, zero when either voxel is solid. +This is the Hele-Shaw closure: for shallow channels the depth-averaged Stokes equations +reduce exactly to this form with mobility proportional to the squared local gap height, and a +porous colony enters as additional drag, with resistances adding as +`1/m = 1/m_channel + 1/m_colony`. The uniform-mobility configuration is the Stokes limit of +the model and its validation gate. The in-plane viscous term is deliberately dropped: side +wall boundary layers, whose thickness is on the order of the gap height, are not resolved. A +full staggered-grid Stokes solve is the named refinement if a study needs them. + +Pressure is fixed on the fluid boundary faces of one axis - inlet one, outlet zero - and +every other exterior face carries no flux. The discrete operator is symmetric positive +definite and is solved matrix-free by Jacobi-preconditioned conjugate gradient in NumPy; no +new dependency is added. The face velocities are the discrete fluxes of the solved pressure, +so per-voxel mass conservation and zero velocity on closed faces hold by construction, and +the result passes the engine's velocity-field validation unchanged. Because the problem is +linear, the solved field is rescaled to a requested mean inlet speed, so callers never handle +pressure or viscosity units. A grid whose inlet is entirely blocked, or which declares +periodic boundaries, is an error. + +`colony_mobility` builds the Brinkman drag field from cell state: each cell's volume +accumulates into its center voxel, the resulting volume fraction sets a Kozeny-Carman style +drag `phi^2 / (1 - phi)^3` scaled by a model-chosen coefficient, and resistances add to the +base mobility. The closure coefficient is a modeling choice, not a measured constant, and is +documented as such. Binning a whole capsule into its center voxel is a nearest-voxel +rasterization: a cell longer than a voxel contributes entirely to one of the voxels it +spans, so the volume fraction, and the drag field with it, is noisier than the colony at +spacings comparable to a cell. + +For colony feedback the field must change mid-run, so the engine adds one mutation: +`Simulation.set_velocity_field` validates a replacement field against the full grid +specification and swaps it atomically; everything downstream - transport, drift, checkpoints - +uses whichever field is current. Model code chooses the re-solve cadence. + +## Validation sequence + +1. Uniform duct: solved field is uniform along the flow axis at exactly the requested mean + speed, transverse faces zero, per-voxel divergence at solver tolerance. +2. Parallel channels of unequal mobility split flux in the mobility ratio. +3. A blocking pillar routes flow around itself with equal flux through every cross section. +4. A half-blocked Brinkman region carries reduced flux consistent with added drag. +5. Fully blocked inlets and periodic boundaries are rejected. +6. A runtime field swap is validated, applied, and checkpointed. + +## Consequences + +- Arbitrary mask geometry, including CAD-derived layouts, gets a conservative flow field + from one build-time solve. +- Colony blockage feeds back on flow at a model-chosen cadence without any native fluid + solver. +- In-plane boundary layers are the stated accuracy limit of the closure. +- The solved field is a depth-averaged velocity: every voxel in a column carries the + column's mean. Advection of signals stays conservative, but a cell drifting near a floor + or ceiling moves at the mean rather than at the slower speed its true profile would give + it, and a rod sees no shear across the gap. A study that needs the profile within a + resolved gap belongs on the staggered MAC solve. diff --git a/docs/architecture/0023-mac-stokes.md b/docs/architecture/0023-mac-stokes.md new file mode 100644 index 0000000..24e1307 --- /dev/null +++ b/docs/architecture/0023-mac-stokes.md @@ -0,0 +1,81 @@ +# ADR 0023: staggered MAC Stokes-Brinkman solve and flow benchmarks + +- Status: accepted +- Date: 2026-08-18 + +## Context + +The Hele-Shaw solve (ADR 0022) depth-averages viscous drag into a mobility +field. That closure is the right cost point for device authoring and in-loop +colony feedback, but it cannot resolve viscous boundary layers on side walls +or the true cross-channel profile, and its accuracy claims need an anchor: a +solver whose only approximation is the mesh. + +## Decision + +`cellmodeller2.stokes` solves the inertia-free Stokes-Brinkman momentum +balance with incompressibility, + +```text +mu lap(v) - mu d(x) v - grad p = 0 div v = 0 +``` + +on the marker-and-cell staggering the engine already uses: velocities on +faces, pressure at cell centers, so the solved field is the engine's transport +input with no interpolation. Walls are the obstacle voxel boundaries and every +non-flow domain edge; wall planes sit half a spacing beyond the outermost site +centers, matching where the device helpers author floors and ceilings. Normal +velocities on fluid-solid faces are eliminated at zero and tangential +components see walls through reflected ghosts, the standard second-order +voxel-grid treatment. The flow-axis boundaries carry prescribed ghost +pressures (inlet one, outlet zero) with zero-gradient normal outflow, and the +linear solution is rescaled to a requested mean inlet speed, so viscosity +drops out; the Brinkman drag field is an inverse permeability +(`colony_drag` builds it from the colony's volume fraction). Collapsed axes +are invariant directions, matching engine transport semantics. + +The saddle-point system is solved through the pressure Schur complement +`S = D A^-1 D^T`, symmetric positive definite, by outer conjugate gradient +with three independent inner component-Laplacian conjugate gradient solves per +application - matrix-free NumPy throughout, no new dependency. The cost sits +well above the Hele-Shaw solve, which remains the default for device authoring +and the in-model re-solve cadence; the MAC solver is for resolved studies and +for anchoring the closure. + +## Validation + +`scripts/run_flow_benchmarks.py` runs both solvers against literature and +exact references and fails nonzero on any tolerance miss; `test_stokes.py` +enforces the same physics at test sizes. + +- Plane Poiseuille: exact parabola, observed convergence order 2. The duct + peak is interpolated to the centerline, since cell centers straddle the axis + of an evenly divided duct. +- Square duct: peak-to-mean velocity ratio 2.0962 (Shah & London 1978; + White, Viscous Fluid Flow), within 0.5% at 32 voxels per side. +- Two-layer Brinkman channel: exact ODE solution (Brinkman 1949) matched in + value and slope across the fluid-porous interface, with observed + second-order convergence. Both profiles are compared at unit mean, since the + solve rescales to the requested speed and amplitude carries no information. +- Cross-solver consistency: in a thin gap the depth-averaged MAC solution + reproduces the Hele-Shaw flux split around a pillar to under one percent - + each solver validates the other in the regime where both apply. +- Gap resolution: a channel one voxel across carries about two and a half + times the flux its parabolic profile would, converging toward the + lubrication limit as the gap resolves - within about ten percent at four + voxels and a few percent at eight. +- The zero-drag path is bit-identical to omitting the drag field, and solved + fields pass engine validation and discrete conservation checks unchanged. + +## Consequences + +- Resolved wall shear and cross-channel profiles are available where a study + needs them, at build-time cost. +- The Hele-Shaw closure's domain of validity is now measured, not asserted. +- Resolution bounds the MAC solve as the closure bounds the depth-averaged + one. Every solve reports `min_gap_voxels`, the fluid voxels across its + narrowest transverse channel, so a caller can tell which of the two solvers + is the better model of a given grid: below four voxels across a gap the + closure is, because it carries the gap-height physics analytically. +- Inlet and outlet impose fully developed flow; strongly developing flow at a + device inlet needs upstream padding voxels. diff --git a/docs/architecture/README.md b/docs/architecture/README.md index d28c0bf..bd74c2a 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -15,6 +15,8 @@ The [numerical contract](numerical-contract.md) is the best starting point for w - [Axis-aligned box constraints](0016-box-constraints.md) - [Axis-aligned cylinder constraints](0018-cylinder-constraints.md) - [Cell removal](0020-cell-removal.md) +- [Steady Hele-Shaw-Brinkman flow solve](0022-brinkman-flow.md) +- [Staggered MAC Stokes-Brinkman solve and flow benchmarks](0023-mac-stokes.md) ## Biological dynamics and signaling diff --git a/python/src/cellmodeller2/flow.py b/python/src/cellmodeller2/flow.py new file mode 100644 index 0000000..98addfd --- /dev/null +++ b/python/src/cellmodeller2/flow.py @@ -0,0 +1,425 @@ +"""Steady Hele-Shaw-Brinkman flow solve for device grids. + +The solver computes the depth-averaged Darcy-Brinkman pressure problem +``div(m grad p) = 0`` over the fluid voxels of a signal grid and returns the +face fluxes ``v = -m_face * dp/dn`` as a face-staggered velocity field. The +per-voxel mobility ``m`` carries the physics: uniform mobility is the Stokes +limit of the closure and resolves flow through arbitrary mask geometry, while +reduced mobility inside a colony (`colony_mobility`) adds Brinkman drag so a +packed trap diverts flow. Mobility is relative - the linear solution is +rescaled to a requested mean inlet speed - so callers never handle pressure or +viscosity units. Discrete conservation and zero velocity on closed faces hold +by construction, and the returned field passes the engine's grid validation +unchanged. + +Pressure is fixed on the fluid boundary faces of the flow axis (inlet one, +outlet zero) and every other exterior face carries no flux; the flow axis +boundaries must therefore be `FIXED` and no axis may be periodic. The discrete +operator is symmetric positive definite and is solved matrix-free with +Jacobi-preconditioned conjugate gradient. Side-wall boundary layers, whose +thickness is on the order of the gap height, are outside the closure. +""" + +from __future__ import annotations + +import math +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +from typing import Protocol + +import numpy as np +from numpy.typing import NDArray + +from ._core import GridBoundaryKind, SignalGridSpec, SignalGridVelocityField, Vec3 + +_FloatGrid = NDArray[np.float64] +_BoolGrid = NDArray[np.bool_] + +_AXES = {"x": 0, "y": 1, "z": 2} + + +class FlowError(ValueError): + """Raised when a flow problem is ill-posed or its solve fails.""" + + +@dataclass(frozen=True, slots=True) +class FlowSolveReport: + iterations: int + residual: float + mean_inlet_speed: float + max_speed: float + + +class _RodLike(Protocol): + @property + def position(self) -> Vec3: ... + @property + def length(self) -> float: ... + @property + def radius(self) -> float: ... + + +def _slices(axis: int, along: slice) -> tuple[slice, slice, slice]: + index: list[slice] = [slice(None), slice(None), slice(None)] + index[axis] = along + return index[0], index[1], index[2] + + +def _mobility_grid(spec: SignalGridSpec, mobility: Sequence[float] | None) -> _FloatGrid: + dims = (spec.shape.x, spec.shape.y, spec.shape.z) + sites = dims[0] * dims[1] * dims[2] + if mobility is None: + values = np.ones(dims, dtype=np.float64) + else: + if len(mobility) != sites: + raise FlowError("mobility must hold one value per grid site") + values = np.asarray(mobility, dtype=np.float64).reshape(dims) + if not bool(np.all(np.isfinite(values))) or bool(np.any(values < 0.0)): + raise FlowError("mobility values must be finite and non-negative") + obstacles = spec.obstacles + if obstacles: + if len(obstacles) != sites: + raise FlowError("obstacles must hold one flag per grid site") + solid = np.asarray(obstacles, dtype=np.uint8).reshape(dims) != 0 + values[solid] = 0.0 + return values + + +def _harmonic_faces(mobility: _FloatGrid, axis: int, spacing: float) -> _FloatGrid: + lower = mobility[_slices(axis, slice(None, -1))] + upper = mobility[_slices(axis, slice(1, None))] + total = lower + upper + product = 2.0 * lower * upper + return np.divide( + product, + total, + out=np.zeros_like(total), + where=total > 0.0, + ) / (spacing * spacing) + + +def _conjugate_gradient( + apply_operator: Callable[[_FloatGrid], _FloatGrid], + rhs: _FloatGrid, + diagonal: _FloatGrid, + tolerance: float, + max_iterations: int, + *, + mask: _BoolGrid | None = None, + label: str = "flow", +) -> tuple[_FloatGrid, int, float]: + """Solve a symmetric positive definite system by Jacobi-preconditioned CG. + + ``mask`` restricts the solve to the sites the operator acts on; entries + outside it stay at zero. + """ + + solution = np.zeros_like(rhs) + residual = rhs.copy() if mask is None else np.where(mask, rhs, 0.0) + rhs_norm = float(np.sqrt(np.sum(residual * residual))) + if rhs_norm == 0.0: + return solution, 0, 0.0 + scale = np.where(diagonal > 0.0, diagonal, 1.0) + preconditioned = residual / scale + direction = preconditioned.copy() + rho = float(np.sum(residual * preconditioned)) + relative = 1.0 + for iteration in range(1, max_iterations + 1): + transformed = apply_operator(direction) + curvature = float(np.sum(direction * transformed)) + if curvature <= 0.0: + break + alpha = rho / curvature + solution += alpha * direction + residual -= alpha * transformed + relative = float(np.sqrt(np.sum(residual * residual))) / rhs_norm + if relative <= tolerance: + return solution, iteration, relative + preconditioned = residual / scale + next_rho = float(np.sum(residual * preconditioned)) + direction = preconditioned + (next_rho / rho) * direction + rho = next_rho + raise FlowError(f"{label} solve did not converge: relative residual {relative:.3e}") + + +def _flow_axis_index(spec: SignalGridSpec, axis: str) -> int: + """Validate a flow problem's axis and boundary kinds, and return the axis.""" + + if axis not in _AXES: + raise FlowError("flow axis must be one of x, y, z") + flow_axis = _AXES[axis] + boundaries = ( + (spec.x_lower, spec.x_upper), + (spec.y_lower, spec.y_upper), + (spec.z_lower, spec.z_upper), + ) + for lower, upper in boundaries: + if lower.kind == GridBoundaryKind.PERIODIC or upper.kind == GridBoundaryKind.PERIODIC: + raise FlowError("the flow solver does not support periodic boundaries") + for boundary in boundaries[flow_axis]: + if boundary.kind != GridBoundaryKind.FIXED: + raise FlowError("the flow axis boundaries must be FIXED to act as inlet and outlet") + return flow_axis + + +def _kozeny_carman_drag(fraction: _FloatGrid, drag_coefficient: float) -> _FloatGrid: + """Kozeny-Carman style drag of a packed volume fraction.""" + + if not math.isfinite(drag_coefficient) or drag_coefficient < 0.0: + raise FlowError("drag coefficient must be finite and non-negative") + return drag_coefficient * fraction * fraction / (1.0 - fraction) ** 3 + + +def solve_flow_field( + spec: SignalGridSpec, + *, + mean_inlet_speed: float, + axis: str = "y", + mobility: Sequence[float] | None = None, + tolerance: float = 1.0e-10, + max_iterations: int = 50_000, +) -> tuple[SignalGridVelocityField, FlowSolveReport]: + """Solve the device flow and return the face-staggered velocity field. + + Flow runs from the lower to the upper boundary of ``axis``; a negative + ``mean_inlet_speed`` reverses it. The grid's shape, spacing, obstacles, + and boundary kinds are read from ``spec``; ``mobility`` optionally gives + one relative mobility per site (default uniform, the Stokes limit). + """ + + if not math.isfinite(mean_inlet_speed) or mean_inlet_speed == 0.0: + raise FlowError("mean inlet speed must be finite and nonzero") + flow_axis = _flow_axis_index(spec, axis) + + dims = (spec.shape.x, spec.shape.y, spec.shape.z) + spacing = (spec.spacing.x, spec.spacing.y, spec.spacing.z) + mobility_grid = _mobility_grid(spec, mobility) + + conductances = tuple( + _harmonic_faces(mobility_grid, index, spacing[index]) for index in range(3) + ) + step = spacing[flow_axis] + inlet_conductance = 2.0 * mobility_grid[_slices(flow_axis, slice(0, 1))] / (step * step) + outlet_conductance = 2.0 * mobility_grid[_slices(flow_axis, slice(-1, None))] / (step * step) + if not bool(np.any(inlet_conductance > 0.0)): + raise FlowError("the inlet boundary is entirely blocked") + + diagonal = np.zeros(dims, dtype=np.float64) + for index in range(3): + diagonal[_slices(index, slice(1, None))] += conductances[index] + diagonal[_slices(index, slice(None, -1))] += conductances[index] + diagonal[_slices(flow_axis, slice(0, 1))] += inlet_conductance + diagonal[_slices(flow_axis, slice(-1, None))] += outlet_conductance + + def apply_operator(pressure: _FloatGrid) -> _FloatGrid: + result = diagonal * pressure + for index in range(3): + faces = conductances[index] + result[_slices(index, slice(1, None))] -= ( + faces * pressure[_slices(index, slice(None, -1))] + ) + result[_slices(index, slice(None, -1))] -= ( + faces * pressure[_slices(index, slice(1, None))] + ) + return result + + rhs = np.zeros(dims, dtype=np.float64) + rhs[_slices(flow_axis, slice(0, 1))] += inlet_conductance + pressure, iterations, residual = _conjugate_gradient( + apply_operator, rhs, diagonal, tolerance, max_iterations + ) + + face_grids: list[_FloatGrid] = [] + for index in range(3): + face_dims = list(dims) + face_dims[index] += 1 + faces = np.zeros(tuple(face_dims), dtype=np.float64) + gradient = ( + pressure[_slices(index, slice(1, None))] - pressure[_slices(index, slice(None, -1))] + ) + faces[_slices(index, slice(1, -1))] = -conductances[index] * spacing[index] * gradient + face_grids.append(faces) + inlet_faces = -inlet_conductance * step * (pressure[_slices(flow_axis, slice(0, 1))] - 1.0) + outlet_faces = outlet_conductance * step * pressure[_slices(flow_axis, slice(-1, None))] + face_grids[flow_axis][_slices(flow_axis, slice(0, 1))] = inlet_faces + face_grids[flow_axis][_slices(flow_axis, slice(-1, None))] = outlet_faces + + open_inlet = inlet_conductance > 0.0 + solved_mean = float(np.mean(inlet_faces[open_inlet])) + # The inlet must carry a real share of whatever the solve moved anywhere, + # which makes the test independent of mobility, spacing, and grid size. + peak = max(float(np.max(np.abs(faces))) for faces in face_grids) + if peak == 0.0 or solved_mean <= 1.0e-9 * peak: + raise FlowError("the device carries no through-flow: the outlet is unreachable") + factor = mean_inlet_speed / solved_mean + scaled = [faces * factor for faces in face_grids] + + field = SignalGridVelocityField() + field.x_faces = [float(value) for value in scaled[0].ravel()] + field.y_faces = [float(value) for value in scaled[1].ravel()] + field.z_faces = [float(value) for value in scaled[2].ravel()] + max_speed = max(float(np.max(np.abs(faces))) for faces in scaled) + report = FlowSolveReport( + iterations=iterations, + residual=residual, + mean_inlet_speed=mean_inlet_speed, + max_speed=max_speed, + ) + return field, report + + +def gap_mobility(spec: SignalGridSpec) -> list[float]: + """Build the Hele-Shaw gap-height mobility field of a device grid. + + In the depth-averaged closure a channel's mobility scales with the square + of its gap height, so a shallow cavity resists through-flow far more than + the tall channel beside it. Each z column's gap is its fluid-voxel count + times the z spacing; every fluid voxel in the column gets the relative + mobility ``(gap / max_gap)^2`` and solid voxels get zero. + """ + + dims = (spec.shape.x, spec.shape.y, spec.shape.z) + obstacles = spec.obstacles + if obstacles: + if len(obstacles) != dims[0] * dims[1] * dims[2]: + raise FlowError("obstacles must hold one flag per grid site") + fluid = (np.asarray(obstacles, dtype=np.uint8).reshape(dims) == 0).astype(np.float64) + else: + fluid = np.ones(dims, dtype=np.float64) + gaps = fluid.sum(axis=2, keepdims=True) + max_gap = float(np.max(gaps)) + if max_gap == 0.0: + raise FlowError("the grid contains no fluid sites") + mobility = fluid * (gaps / max_gap) ** 2 + return [float(value) for value in mobility.ravel()] + + +def colony_volume_fraction( + spec: SignalGridSpec, + cells: Iterable[_RodLike], + *, + max_volume_fraction: float = 0.9, +) -> _FloatGrid: + """Rasterize the colony into a per-voxel volume fraction grid. + + Each cell's capsule volume accumulates into the voxel holding its center + (the grid origin is the center of site zero, so voxel ``i`` spans the + half-open interval centered on ``origin + i * spacing``); fractions are + capped at ``max_volume_fraction``. + """ + + if not 0.0 < max_volume_fraction < 1.0: + raise FlowError("maximum volume fraction must lie strictly between zero and one") + dims = (spec.shape.x, spec.shape.y, spec.shape.z) + origin = (spec.origin.x, spec.origin.y, spec.origin.z) + spacing = (spec.spacing.x, spec.spacing.y, spec.spacing.z) + volume = np.zeros(dims, dtype=np.float64) + for cell in cells: + position = (cell.position.x, cell.position.y, cell.position.z) + indices: list[int] = [] + inside = True + for component in range(3): + index = math.floor( + (position[component] - origin[component]) / spacing[component] + 0.5 + ) + if not 0 <= index < dims[component]: + inside = False + break + indices.append(index) + if not inside: + continue + radius = cell.radius + capsule = math.pi * radius * radius * cell.length + (4.0 / 3.0) * math.pi * radius**3 + volume[indices[0], indices[1], indices[2]] += capsule + return np.minimum(volume / spec.voxel_volume, max_volume_fraction) + + + +class _SpeciesRodLike(Protocol): + @property + def position(self) -> Vec3: ... + @property + def species(self) -> list[float]: ... + + +def colony_species_density( + spec: SignalGridSpec, + cells: Iterable[_SpeciesRodLike], + *, + species: int, +) -> list[float]: + """Rasterize one intracellular species into a per-voxel density. + + Each cell's level accumulates into the voxel holding its center, on the + same nearest-voxel convention as `colony_volume_fraction`, and the total is + divided by the voxel volume. A rate written per cell and per unit of that + species becomes a rate per unit volume of field, which is what an affine + grid reaction carries. + """ + + if species < 0: + raise FlowError("species index must be non-negative") + dims = (spec.shape.x, spec.shape.y, spec.shape.z) + origin = (spec.origin.x, spec.origin.y, spec.origin.z) + spacing = (spec.spacing.x, spec.spacing.y, spec.spacing.z) + totals = np.zeros(dims, dtype=np.float64) + for cell in cells: + levels = cell.species + if species >= len(levels): + raise FlowError("species index is outside the cell's species") + position = (cell.position.x, cell.position.y, cell.position.z) + indices: list[int] = [] + for component in range(3): + index = math.floor( + (position[component] - origin[component]) / spacing[component] + 0.5 + ) + if not 0 <= index < dims[component]: + break + indices.append(index) + if len(indices) != 3: + continue + totals[indices[0], indices[1], indices[2]] += max(0.0, levels[species]) + return [float(value) for value in (totals / spec.voxel_volume).ravel()] + + +def colony_mobility( + spec: SignalGridSpec, + cells: Iterable[_RodLike], + *, + base: float | Sequence[float] = 1.0, + drag_coefficient: float = 100.0, + max_volume_fraction: float = 0.9, +) -> list[float]: + """Build the Brinkman mobility field from the current colony. + + Each cell's capsule volume accumulates into the voxel holding its center + (the grid origin is the center of site zero, so voxel ``i`` spans the + half-open interval centered on ``origin + i * spacing``); the resulting + volume fraction ``phi`` adds Kozeny-Carman style drag + ``drag_coefficient * phi^2 / (1 - phi)^3`` to the base resistance, so + ``1/m = 1/base + drag``. ``base`` is a uniform value or a per-site field + such as `gap_mobility`. The drag coefficient is a modeling choice: it + sets how strongly a packed colony resists through-flow relative to the + open channel. Solid voxels stay at zero mobility. + """ + + dims = (spec.shape.x, spec.shape.y, spec.shape.z) + if isinstance(base, (int, float)): + if not math.isfinite(base) or base <= 0.0: + raise FlowError("base mobility must be finite and positive") + base_grid = np.full(dims, float(base), dtype=np.float64) + else: + if len(base) != dims[0] * dims[1] * dims[2]: + raise FlowError("base mobility must hold one value per grid site") + base_grid = np.asarray(base, dtype=np.float64).reshape(dims) + if not bool(np.all(np.isfinite(base_grid))) or bool(np.any(base_grid < 0.0)): + raise FlowError("base mobility values must be finite and non-negative") + fraction = colony_volume_fraction(spec, cells, max_volume_fraction=max_volume_fraction) + drag = _kozeny_carman_drag(fraction, drag_coefficient) + # m = b / (1 + b * drag) is 1 / (1/b + drag) extended continuously to b = 0. + mobility = base_grid / (1.0 + base_grid * drag) + obstacles = spec.obstacles + if obstacles: + solid = np.asarray(obstacles, dtype=np.uint8).reshape(dims) != 0 + mobility[solid] = 0.0 + return [float(value) for value in mobility.ravel()] diff --git a/python/src/cellmodeller2/flow_reference.py b/python/src/cellmodeller2/flow_reference.py new file mode 100644 index 0000000..293aa2a --- /dev/null +++ b/python/src/cellmodeller2/flow_reference.py @@ -0,0 +1,116 @@ +"""Exact and tabulated reference solutions for validating flow solves. + +These are the analytic answers the flow solvers are measured against, kept in +one place so a benchmark run and a test suite check the same physics. They +also give a model author a way to validate a device grid: build the reference +geometry with `duct_grid`, solve it, and compare. + +References: +- Plane Poiseuille and rectangular duct series: F. M. White, "Viscous Fluid + Flow" (3rd ed., ch. 3); the duct peak-to-mean ratio is tabulated in R. K. + Shah and A. L. London, "Laminar Flow Forced Convection in Ducts" (1978). +- Two-layer Brinkman channel: the exact solution of the Brinkman equation + (H. C. Brinkman, Appl. Sci. Res. A1, 1949) matched in value and slope across + the fluid-porous interface. +- Hele-Shaw closure: depth-averaged Stokes flow in a thin gap obeys Darcy's + law with mobility proportional to the squared gap height (H. S. Hele-Shaw, + 1898). +""" + +from __future__ import annotations + +import math + +import numpy as np +from numpy.typing import NDArray + +from ._core import GridBoundaryKind, GridShape, SignalGridSpec, Vec3 + +_Profile = NDArray[np.float64] + +SQUARE_DUCT_PEAK_TO_MEAN = 2.0962 +"""Peak-to-mean axial velocity of fully developed flow in a square duct.""" + + +def duct_grid( + nx: int, ny: int, nz: int, spacing: tuple[float, float, float] +) -> SignalGridSpec: + """A duct grid flowing along y between fixed inlet and outlet boundaries.""" + + shape = GridShape() + shape.x, shape.y, shape.z = nx, ny, nz + spec = SignalGridSpec() + spec.signal_count = 1 + spec.shape = shape + spec.spacing = Vec3(*spacing) + spec.diffusion = [1.0] + spec.advection = [Vec3()] + for name in ("y_lower", "y_upper"): + boundary = getattr(spec, name) + boundary.kind = GridBoundaryKind.FIXED + boundary.values = [0.0] + setattr(spec, name, boundary) + return spec + + +def site_index(spec: SignalGridSpec, x: int, y: int, z: int) -> int: + """The flat site index of a lattice position in the grid's site order.""" + + return (x * spec.shape.y + y) * spec.shape.z + z + + +def plane_poiseuille(positions: _Profile) -> _Profile: + """The unit-mean parabolic profile across a channel of unit width.""" + + return 6.0 * positions * (1.0 - positions) + + +def two_layer_brinkman(drag: float, positions: _Profile) -> _Profile: + """The exact profile of a channel half open and half porous. + + A unit pressure gradient drives flow across a unit-width channel with + no-slip walls at both edges. The lower half is open Stokes flow and the + upper half carries Brinkman drag ``drag``, an inverse permeability. The + two branches match in value and in slope at the interface, and the + solution is unscaled: its amplitude is part of the reference. + """ + + if drag <= 0.0: + raise ValueError("drag must be positive") + root = math.sqrt(drag) + matrix = np.array( + [ + [0.0, math.cosh(root), math.sinh(root)], + [0.5, -math.cosh(root * 0.5), -math.sinh(root * 0.5)], + [1.0, -root * math.sinh(root * 0.5), -root * math.cosh(root * 0.5)], + ] + ) + rhs = np.array([-1.0 / drag, 1.0 / drag + 0.125, 0.5]) + linear, cosh_c, sinh_c = (float(value) for value in np.linalg.solve(matrix, rhs)) + profile: _Profile = np.where( + positions < 0.5, + -positions * positions / 2.0 + linear * positions, + 1.0 / drag + cosh_c * np.cosh(root * positions) + sinh_c * np.sinh(root * positions), + ) + return profile + + +def centerline_value(profile: _Profile) -> float: + """Interpolate a cell-centered symmetric profile to its center. + + Cell centers straddle the axis of a duct with an even cell count, so the + largest sampled value understates the peak. The symmetric four-point + combination ``(9 * inner - outer) / 8`` per axis recovers the center value + exactly for a parabola and to fourth order for a smooth even profile. + """ + + result = profile + for _ in range(profile.ndim): + count = result.shape[0] + if count < 4 or count % 2 != 0: + raise ValueError("centerline interpolation needs at least four cells per axis") + middle = count // 2 + inner = 0.5 * (result[middle - 1] + result[middle]) + outer = 0.5 * (result[middle - 2] + result[middle + 1]) + result = (9.0 * inner - outer) / 8.0 + return float(result) diff --git a/python/src/cellmodeller2/masks.py b/python/src/cellmodeller2/masks.py new file mode 100644 index 0000000..011f139 --- /dev/null +++ b/python/src/cellmodeller2/masks.py @@ -0,0 +1,248 @@ +"""Extraction of device geometry from photomask CAD. + +A mask DXF is authoring input, like every other predicate in the microfluidics +helpers: geometry is read once into plain data and the runtime never touches +CAD. The reader is deliberately minimal and closed: it parses axis-aligned +closed `LWPOLYLINE` outlines from the model-space `ENTITIES` section of an +ASCII DXF, ignoring block definitions (orphaned array remnants in mask files), +paper space, and every other entity kind. Files are size-bounded and nothing is +executed. + +Mask drawings conventionally use one drawing unit per millimeter; pass +``unit_scale=1000.0`` to obtain micrometer coordinates. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +MAX_MASK_BYTES = 64 << 20 + + +class MaskError(ValueError): + """Raised when a mask file cannot be safely read or interpreted.""" + + +@dataclass(frozen=True, slots=True) +class MaskPolyline: + layer: str + closed: bool + vertices: tuple[tuple[float, float], ...] + block: str | None = None + + +@dataclass(frozen=True, slots=True) +class MaskRectangle: + layer: str + center: tuple[float, float] + width: float + height: float + + +def load_mask_polylines( + path: str | os.PathLike[str], + *, + max_bytes: int = MAX_MASK_BYTES, + include_blocks: bool = False, +) -> tuple[MaskPolyline, ...]: + """Read the polylines of an ASCII DXF mask drawing. + + Model-space entities always load. With ``include_blocks``, polylines inside + block definitions load as well, tagged with their block name; mask files + from array-based CAD workflows often keep real geometry only in otherwise + orphaned blocks, in world coordinates. + """ + + source = Path(path) + try: + with source.open("rb") as stream: + encoded = stream.read(max_bytes + 1) + except OSError as error: + raise MaskError(f"could not read mask {source}") from error + if not encoded: + raise MaskError("mask file is empty") + if len(encoded) > max_bytes: + raise MaskError(f"mask exceeds the {max_bytes}-byte limit") + try: + text = encoded.decode("ascii", errors="replace") + except UnicodeDecodeError as error: # pragma: no cover - replace never raises + raise MaskError("mask is not ASCII DXF") from error + + lines = text.splitlines() + if len(lines) < 2: + raise MaskError("mask is not a group-coded DXF document") + + polylines: list[MaskPolyline] = [] + in_entities = False + in_blocks = False + block_name: str | None = None + pending_block_name = False + layer = "" + closed = False + xs: list[float] = [] + ys: list[float] = [] + collecting = False + + def finish() -> None: + nonlocal collecting + if collecting and len(xs) == len(ys) and len(xs) >= 2: + polylines.append( + MaskPolyline( + layer=layer, + closed=closed, + vertices=tuple(zip(xs, ys, strict=True)), + block=block_name, + ) + ) + collecting = False + + index = 0 + while index + 1 < len(lines): + code = lines[index].strip() + value = lines[index + 1].strip() + index += 2 + if code != "0": + if pending_block_name and code == "2": + block_name = value + pending_block_name = False + elif collecting: + try: + if code == "8": + layer = value + elif code == "70": + closed = bool(int(value) & 1) + elif code == "10": + xs.append(float(value)) + elif code == "20": + ys.append(float(value)) + except ValueError as error: + raise MaskError(f"mask contains a malformed {code} group") from error + continue + finish() + pending_block_name = False + if value == "SECTION": + # A section names itself in the group pair that follows: code 2, + # then the name. Anything else leaves the section unnamed rather + # than silently reading the next value as its name. + named = index + 1 < len(lines) and lines[index].strip() == "2" + section = lines[index + 1].strip() if named else "" + in_entities = section == "ENTITIES" + in_blocks = section == "BLOCKS" + elif value == "ENDSEC": + in_entities = False + in_blocks = False + elif value == "BLOCK": + pending_block_name = True + elif value == "ENDBLK": + block_name = None + elif value == "LWPOLYLINE" and ( + in_entities or (include_blocks and in_blocks and block_name is not None) + ): + collecting = True + layer = "" + closed = False + xs = [] + ys = [] + finish() + if not polylines: + raise MaskError("mask contains no model-space polylines") + return tuple(polylines) + + +def extract_rectangles( + polylines: tuple[MaskPolyline, ...], + *, + layer: str | None = None, + unit_scale: float = 1.0, + alignment_tolerance: float = 1.0e-9, +) -> tuple[MaskRectangle, ...]: + """Return the closed axis-aligned rectangles among the polylines. + + A rectangle is a closed outline of four or five vertices (the fifth may + repeat the first) whose distinct vertices are exactly the four corners of + its bounding box. Coordinates and sizes are multiplied by ``unit_scale``. + """ + + if unit_scale <= 0.0: + raise MaskError("unit scale must be positive") + rectangles: list[MaskRectangle] = [] + for polyline in polylines: + if layer is not None and polyline.layer != layer: + continue + if not polyline.closed or not (4 <= len(polyline.vertices) <= 5): + continue + xs = [vertex[0] for vertex in polyline.vertices] + ys = [vertex[1] for vertex in polyline.vertices] + low_x, high_x = min(xs), max(xs) + low_y, high_y = min(ys), max(ys) + if high_x - low_x <= 0.0 or high_y - low_y <= 0.0: + continue + corners = { + (low_x, low_y), + (low_x, high_y), + (high_x, low_y), + (high_x, high_y), + } + matched: set[tuple[float, float]] = set() + aligned = True + for x, y in polyline.vertices: + corner = next( + ( + candidate + for candidate in corners + if abs(x - candidate[0]) <= alignment_tolerance + and abs(y - candidate[1]) <= alignment_tolerance + ), + None, + ) + if corner is None: + aligned = False + break + matched.add(corner) + if not aligned or matched != corners: + continue + rectangles.append( + MaskRectangle( + layer=polyline.layer, + center=( + (low_x + high_x) * 0.5 * unit_scale, + (low_y + high_y) * 0.5 * unit_scale, + ), + width=(high_x - low_x) * unit_scale, + height=(high_y - low_y) * unit_scale, + ) + ) + return tuple(rectangles) + + +def match_rectangles( + rectangles: tuple[MaskRectangle, ...], + width: float, + height: float, + *, + tolerance: float = 1.0, + allow_rotated: bool = True, +) -> tuple[MaskRectangle, ...]: + """Return the rectangles whose size matches ``width x height``. + + Sizes compare within ``tolerance`` in the rectangles' own units; with + ``allow_rotated`` the swapped orientation also matches. + """ + + if width <= 0.0 or height <= 0.0 or tolerance < 0.0: + raise MaskError("match dimensions must be positive and tolerance non-negative") + + def matches(rectangle: MaskRectangle) -> bool: + direct = ( + abs(rectangle.width - width) <= tolerance + and abs(rectangle.height - height) <= tolerance + ) + rotated = ( + abs(rectangle.width - height) <= tolerance + and abs(rectangle.height - width) <= tolerance + ) + return direct or (allow_rotated and rotated) + + return tuple(rectangle for rectangle in rectangles if matches(rectangle)) diff --git a/python/src/cellmodeller2/microfluidics.py b/python/src/cellmodeller2/microfluidics.py new file mode 100644 index 0000000..a30b142 --- /dev/null +++ b/python/src/cellmodeller2/microfluidics.py @@ -0,0 +1,364 @@ +"""Authoring helpers for microfluidic device models. + +A device is described once in physical coordinates and then projected into the +engine's typed inputs: box wall constraints for mechanics, a solid mask for the +signal grid, and a numerically solved face-staggered flow field for advection +(the steady Hele-Shaw solve of `cellmodeller2.flow`, so mass is conserved per +voxel through any mask geometry). The runtime receives only materialized data; +every predicate here is authoring-time. + +Voxelization is conservative: a lattice site is solid only when its entire +voxel lies inside a wall, so the mechanics walls enclose the solid mask and a +cell pressed against a wall always has a fluid site to sample. The mask's +fluid region therefore reaches up to half a voxel into each wall, which is the +staircase accuracy of any mask at the grid resolution. +""" + +from __future__ import annotations + +import os +from collections import Counter +from dataclasses import dataclass + +from ._core import ( # pyright: ignore[reportMissingModuleSource] + BoxConstraintInit, + ConstraintRegion, + GridBoundaryKind, + PlaneConstraintInit, + SignalGridSpec, + Simulation, + Vec3, +) +from .flow import gap_mobility, solve_flow_field +from .masks import MaskError, extract_rectangles, load_mask_polylines + +# A voxel edge that lands on a wall plane belongs to the wall, so the voxel +# tests admit a rounding margin: without it a wall drawn exactly on a lattice +# face resolves as solid or fluid according to float rounding. +_EDGE_TOLERANCE = 1.0e-6 + +# Wall blocks as (low corner, high corner) pairs in device coordinates. +_Blocks = tuple[tuple[tuple[float, float, float], tuple[float, float, float]], ...] + + +def _reaches(edge: float, wall: float, half: float) -> bool: + """Whether a voxel's lower edge has reached a wall lying above it.""" + + return edge >= wall - _EDGE_TOLERANCE * half + + +def _recedes(edge: float, wall: float, half: float) -> bool: + """Whether a voxel's upper edge has reached a wall lying below it.""" + + return edge <= wall + _EDGE_TOLERANCE * half + + + +class _ChannelDevice: + """A trap fed by a straight channel along y, projected onto engine inputs. + + Subclasses describe their geometry with `_solid`, a predicate over a + voxel's center and half extents, and inherit the projection of that + geometry into a solid mask, a solved flow field, and inlet and outlet + boundaries. + """ + + __slots__ = () + + mean_flow_speed: float + + def _solid(self, px: float, py: float, pz: float, half: tuple[float, float, float]) -> bool: + """Whether a voxel lies entirely inside a wall.""" + + raise NotImplementedError + + def _wall_boxes( + self, simulation: Simulation, blocks: _Blocks, region: ConstraintRegion + ) -> None: + for low, high in blocks: + box = BoxConstraintInit() + box.center = Vec3( + *((left + right) * 0.5 for left, right in zip(low, high, strict=True)) + ) + box.half_extents = Vec3( + *((right - left) * 0.5 for left, right in zip(low, high, strict=True)) + ) + box.coefficient = 1.0 + box.allowed_region = region + simulation.add_box_constraint(box) + + def apply_to_grid( + self, + spec: SignalGridSpec, + inlet_values: list[float], + outlet_values: list[float], + ) -> None: + """Materialize the device's solid mask, solved flow field, and y inlet and outlet.""" + + shape = spec.shape + origin = spec.origin + spacing = spec.spacing + # A site is solid only when its whole voxel lies inside a wall, so the + # mask's solid region is enclosed by the mechanics walls. The voxel + # holding any position a cell can reach is then fluid, which guarantees + # every sampling stencil has at least one fluid corner. + half = (spacing.x * 0.5, spacing.y * 0.5, spacing.z * 0.5) + obstacles = [0] * (shape.x * shape.y * shape.z) + for x in range(shape.x): + px = origin.x + spacing.x * x + for y in range(shape.y): + py = origin.y + spacing.y * y + for z in range(shape.z): + pz = origin.z + spacing.z * z + if self._solid(px, py, pz, half): + obstacles[x * shape.y * shape.z + y * shape.z + z] = 1 + spec.obstacles = obstacles + + spec.advection = [Vec3() for _ in range(spec.signal_count)] + spec.y_lower.kind = GridBoundaryKind.FIXED + spec.y_lower.values = list(inlet_values) + spec.y_upper.kind = GridBoundaryKind.FIXED + spec.y_upper.values = list(outlet_values) + + if self.mean_flow_speed != 0.0: + field, _ = solve_flow_field( + spec, + mean_inlet_speed=self.mean_flow_speed, + axis="y", + mobility=gap_mobility(spec), + ) + spec.velocity_field = field + + +@dataclass(frozen=True, slots=True) +class TrapChannelDevice(_ChannelDevice): + """An open-sided cell trap fed by a straight flow channel. + + The channel runs along the y axis between ``channel_far_x`` and + ``trap_open_x``. The trap cavity spans ``trap_open_x`` to ``trap_back_x`` + in x and ``-trap_half_y`` to ``trap_half_y`` in y, open toward the channel + and sealed on its other three sides by solid blocks. Media flows along +y + through the channel with the solved steady device flow; the dead-end trap + interior carries only the weak circulation at its mouth and exchanges with + the channel by diffusion through the open face. + """ + + trap_open_x: float = -60.0 + trap_back_x: float = 60.0 + trap_half_y: float = 15.0 + trap_half_z: float = 3.0 + channel_far_x: float = -100.0 + channel_half_length: float = 120.0 + wall_thickness: float = 2.0 + mean_flow_speed: float = 0.0 + + def add_constraints(self, simulation: Simulation) -> None: + """Add the device's wall constraints to a simulation.""" + + # The walls reach past the chamber in z. A box pushes a cell that has + # entered it toward its nearest face, so a wall ending level with the + # ceiling offers a cell pressed against that ceiling an escape of zero + # length in z, which the chamber then blocks: the cell never leaves in + # y and a crowded trap drives it further in. Standing the walls proud + # keeps the way out of a wall the way the cell came in. + wall_top = self.trap_half_z + self.wall_thickness + blocks = ( + ( + (self.trap_open_x, self.trap_half_y, -wall_top), + ( + self.trap_back_x + self.wall_thickness, + self.channel_half_length, + wall_top, + ), + ), + ( + (self.trap_open_x, -self.channel_half_length, -wall_top), + ( + self.trap_back_x + self.wall_thickness, + -self.trap_half_y, + wall_top, + ), + ), + ( + ( + self.trap_back_x, + -self.trap_half_y - self.wall_thickness, + -wall_top, + ), + ( + self.trap_back_x + self.wall_thickness, + self.trap_half_y + self.wall_thickness, + wall_top, + ), + ), + ) + self._wall_boxes(simulation, blocks, ConstraintRegion.OUTSIDE) + + chamber = BoxConstraintInit() + chamber.center = Vec3( + (self.channel_far_x + self.trap_back_x + self.wall_thickness) * 0.5, + 0.0, + 0.0, + ) + chamber.half_extents = Vec3( + (self.trap_back_x + self.wall_thickness - self.channel_far_x) * 0.5, + self.channel_half_length, + self.trap_half_z, + ) + chamber.coefficient = 1.0 + chamber.allowed_region = ConstraintRegion.INSIDE + simulation.add_box_constraint(chamber) + + def _solid(self, px: float, py: float, pz: float, half: tuple[float, float, float]) -> bool: + hx, hy, hz = half + if _recedes(px + hx, self.channel_far_x, hx): + return True + if _reaches(px - hx, self.trap_back_x + self.wall_thickness, hx): + return True + if _reaches(abs(pz) - hz, self.trap_half_z, hz): + return True + if _reaches(px - hx, self.trap_open_x, hx) and _reaches( + abs(py) - hy, self.trap_half_y, hy + ): + return True + return _reaches(px - hx, self.trap_back_x, hx) + +@dataclass(frozen=True, slots=True) +class BiopixelTrapDevice(_ChannelDevice): + """One trap of a biopixel array: a shallow monolayer cavity beside a tall channel. + + The cavity spans ``0`` to ``trap_depth`` in x with its open face at ``x = 0`` + toward the channel, ``trap_width`` in y, and only ``trap_height`` in z, so + the colony grows as a monolayer under the cavity ceiling. The flow channel + runs along y between ``-channel_width`` and ``0`` at the full + ``channel_height``. The device floor is ``z = 0``. An array device repeats + this trap along its channels; every trap sees the same fresh-media flow, so + one simulated trap is representative of each biopixel in the array when + inter-trap coupling is not modeled. + """ + + trap_depth: float = 95.0 + trap_width: float = 100.0 + trap_height: float = 1.65 + channel_width: float = 100.0 + channel_height: float = 10.0 + channel_half_length: float = 150.0 + wall_thickness: float = 10.0 + mean_flow_speed: float = 0.0 + + @classmethod + def from_mask( + cls, + path: str | os.PathLike[str], + *, + layer: str = "Layer-2", + wall_inset: float = 5.0, + unit_scale: float = 1000.0, + mean_flow_speed: float = 0.0, + ) -> BiopixelTrapDevice: + """Derive the trap footprint from a photomask drawing. + + The mask draws each trap's outer wall outline. The cavity is the + outline minus ``wall_inset`` per wall: two side walls across the long + dimension and one back wall across the short dimension, whose remaining + side is the open face toward the channel. The drawing must contain one + uniform trap population on the layer. + """ + + polylines = load_mask_polylines(path) + rectangles = extract_rectangles(polylines, layer=layer, unit_scale=unit_scale) + if not rectangles: + raise MaskError(f"mask layer {layer!r} contains no rectangles") + sizes = Counter( + (round(max(r.width, r.height), 3), round(min(r.width, r.height), 3)) + for r in rectangles + ) + (long_side, short_side), count = sizes.most_common(1)[0] + if count < 2: + raise MaskError(f"mask layer {layer!r} has no repeated trap outline") + width = long_side - 2.0 * wall_inset + depth = short_side - wall_inset + if width <= 0.0 or depth <= 0.0: + raise MaskError("wall inset leaves no cavity") + return cls(trap_width=width, trap_depth=depth, mean_flow_speed=mean_flow_speed) + + def add_constraints(self, simulation: Simulation) -> None: + """Add the device's wall constraints to a simulation.""" + + half_y = self.trap_width * 0.5 + thickness = self.wall_thickness + + floor = PlaneConstraintInit() + floor.point = Vec3(0.0, 0.0, 0.0) + floor.inward_normal = Vec3(0.0, 0.0, 1.0) + simulation.add_plane_constraint(floor) + + ceiling = PlaneConstraintInit() + ceiling.point = Vec3(0.0, 0.0, self.channel_height) + ceiling.inward_normal = Vec3(0.0, 0.0, -1.0) + simulation.add_plane_constraint(ceiling) + + boxes = ( + ( + (0.0, -half_y - thickness, self.trap_height), + ( + self.trap_depth + thickness, + half_y + thickness, + self.channel_height + thickness, + ), + ), + ( + (0.0, half_y, -thickness), + ( + self.trap_depth + thickness, + self.channel_half_length, + self.channel_height + thickness, + ), + ), + ( + (0.0, -self.channel_half_length, -thickness), + ( + self.trap_depth + thickness, + -half_y, + self.channel_height + thickness, + ), + ), + ( + (self.trap_depth, -half_y - thickness, -thickness), + ( + self.trap_depth + thickness, + half_y + thickness, + self.channel_height + thickness, + ), + ), + ) + self._wall_boxes(simulation, boxes, ConstraintRegion.OUTSIDE) + + chamber = BoxConstraintInit() + chamber.center = Vec3( + (self.trap_depth + self.wall_thickness - self.channel_width) * 0.5, + 0.0, + self.channel_height * 0.5, + ) + chamber.half_extents = Vec3( + (self.channel_width + self.trap_depth + self.wall_thickness) * 0.5, + self.channel_half_length, + self.channel_height * 0.5, + ) + chamber.coefficient = 1.0 + chamber.allowed_region = ConstraintRegion.INSIDE + simulation.add_box_constraint(chamber) + + def _solid(self, px: float, py: float, pz: float, half: tuple[float, float, float]) -> bool: + hx, hy, hz = half + if _recedes(px + hx, -self.channel_width, hx): + return True + if _reaches(px - hx, self.trap_depth + self.wall_thickness, hx): + return True + if _recedes(pz + hz, 0.0, hz) or _reaches(pz - hz, self.channel_height, hz): + return True + if _reaches(px - hx, 0.0, hx) and _reaches(abs(py) - hy, self.trap_width * 0.5, hy): + return True + if _reaches(px - hx, self.trap_depth, hx): + return True + return _reaches(px - hx, 0.0, hx) and _reaches(pz - hz, self.trap_height, hz) diff --git a/python/src/cellmodeller2/stokes.py b/python/src/cellmodeller2/stokes.py new file mode 100644 index 0000000..92adae7 --- /dev/null +++ b/python/src/cellmodeller2/stokes.py @@ -0,0 +1,408 @@ +"""Staggered-grid Stokes-Brinkman flow solve for device grids. + +This is the high-fidelity companion to the Hele-Shaw solver in +`cellmodeller2.flow`: it resolves the full velocity field, including viscous +boundary layers on every wall, instead of depth-averaging them into a mobility +closure. The momentum balance is inertia-free Stokes with an optional Brinkman +drag, + +```text +mu * lap(v) - mu * d(x) * v - grad(p) = 0 div(v) = 0 +``` + +discretized on the marker-and-cell staggering the engine already uses: +velocity components on faces, pressure at cell centers. No-slip walls are the +obstacle voxel boundaries and every non-flow domain edge; wall planes sit half +a spacing beyond the outermost site centers, which is exactly where the device +helpers author their floors and ceilings. Normal velocities on fluid-solid +faces are eliminated at zero, and tangential components see the wall through +reflected ghosts. Pressure is fixed beyond the fluid boundary faces of the +flow axis (inlet one, outlet zero) with a zero-gradient outflow condition on +the normal velocity, so a fully developed channel reproduces its exact +profile shape. Because the problem is linear, the solution is rescaled to a +requested mean inlet speed and viscosity drops out; the drag field ``d`` is an +inverse permeability with units of one over length squared. + +The saddle-point system is solved by the pressure Schur complement: an outer +conjugate gradient on `S = D A^-1 D^T` (symmetric positive definite), with +each application solving three independent component Laplacians by inner +conjugate gradient. Everything is matrix-free NumPy. This costs far more than +the Hele-Shaw solve - it is the build-time and benchmark solver, not the +per-hundred-steps re-solve inside a running model. +""" + +# pyright: reportPrivateUsage=false + +from __future__ import annotations + +import math +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from typing import TypeVar + +import numpy as np + +from ._core import SignalGridSpec, SignalGridVelocityField +from .flow import ( + FlowError, + _BoolGrid, + _conjugate_gradient, + _FloatGrid, + _flow_axis_index, + _kozeny_carman_drag, + _RodLike, + colony_volume_fraction, +) + +# Either a float field or a mask: shifting and reflecting treat them alike. +_Grid = TypeVar("_Grid", _FloatGrid, _BoolGrid) + + +@dataclass(frozen=True, slots=True) +class StokesSolveReport: + outer_iterations: int + inner_iterations: int + divergence_rms: float + mean_inlet_speed: float + max_speed: float + min_gap_voxels: int + """Fluid voxels across the narrowest channel, transverse to the flow. + + A no-slip profile needs several voxels to resolve, so this number bounds + the solve's accuracy: a channel one voxel across carries roughly two and a + half times the flux its true parabolic profile would, four voxels bring + that within about ten percent, and eight within a few percent. Below four, + the depth-averaged Hele-Shaw closure of `cellmodeller2.flow` is the more + accurate model of a shallow channel. + """ + + +def colony_drag( + spec: SignalGridSpec, + cells: Iterable[_RodLike], + *, + drag_coefficient: float, + max_volume_fraction: float = 0.9, +) -> list[float]: + """Build the Brinkman drag field (inverse permeability) from the colony. + + The colony's per-voxel volume fraction ``phi`` sets a Kozeny-Carman style + drag ``drag_coefficient * phi^2 / (1 - phi)^3``. The coefficient carries + units of one over length squared and is a modeling choice. Solid voxels + stay at zero (they are walls, not porous media). + """ + + fraction = colony_volume_fraction(spec, cells, max_volume_fraction=max_volume_fraction) + drag = _kozeny_carman_drag(fraction, drag_coefficient) + obstacles = spec.obstacles + if obstacles: + dims = (spec.shape.x, spec.shape.y, spec.shape.z) + solid = np.asarray(obstacles, dtype=np.uint8).reshape(dims) != 0 + drag[solid] = 0.0 + return [float(value) for value in drag.ravel()] + + +def _minimum_gap_voxels(fluid: _BoolGrid, dims: tuple[int, int, int], flow_axis: int) -> int: + """The shortest run of fluid voxels across any axis transverse to the flow.""" + + shortest = 0 + for axis in range(3): + if axis == flow_axis or dims[axis] <= 1: + continue + lines = np.moveaxis(fluid, axis, -1) + padded = np.zeros((*lines.shape[:-1], lines.shape[-1] + 2), dtype=np.int8) + padded[..., 1:-1] = lines + edges = np.diff(padded, axis=-1) + starts = np.argwhere(edges == 1) + ends = np.argwhere(edges == -1) + if starts.size == 0: + continue + runs = ends[:, -1] - starts[:, -1] + axis_shortest = int(runs.min()) + shortest = axis_shortest if shortest == 0 else min(shortest, axis_shortest) + return shortest + + +class _StokesOperator: + """The masked component Laplacians, divergence, and gradient of one problem.""" + + def __init__( + self, + dims: tuple[int, int, int], + spacing: tuple[float, float, float], + fluid: _BoolGrid, + drag: _FloatGrid, + flow_axis: int, + ) -> None: + self.dims = dims + self.spacing = spacing + self.fluid = fluid + # Collapsed axes (a single site) are invariant directions, matching + # the engine's transport semantics: no wall reflection across them. + self.live_axes = tuple(a for a in range(3) if dims[a] > 1) + + self.active: list[_BoolGrid] = [] + self.exists: list[_BoolGrid] = [] + self.face_drag: list[_FloatGrid] = [] + for c in range(3): + lower = self._cell_beside(fluid, c, -1) + upper = self._cell_beside(fluid, c, +1) + active = lower & upper + exists = lower | upper + if c == flow_axis: + # The flow-axis boundary faces carry prescribed ghost pressures + # rather than a wall, so they stay active beside one fluid cell. + active[self._edge_slice(c, 0)] = fluid[self._edge_slice(c, 0)] + active[self._edge_slice(c, -1)] = fluid[self._edge_slice(c, -1)] + self.active.append(active) + self.exists.append(exists) + drag_low = self._cell_beside_values(drag, c, -1) + drag_high = self._cell_beside_values(drag, c, +1) + counts = lower.astype(np.float64) + upper.astype(np.float64) + face_drag = np.divide( + drag_low + drag_high, + counts, + out=np.zeros_like(drag_low), + where=counts > 0.0, + ) + self.face_drag.append(face_drag) + + def _face_dims(self, c: int) -> tuple[int, int, int]: + dims = list(self.dims) + dims[c] += 1 + return dims[0], dims[1], dims[2] + + def _edge_slice(self, axis: int, edge: int) -> tuple[slice, slice, slice]: + index: list[slice] = [slice(None)] * 3 + index[axis] = slice(0, 1) if edge == 0 else slice(-1, None) + return index[0], index[1], index[2] + + def _cell_beside(self, cells: _BoolGrid, c: int, side: int) -> _BoolGrid: + """Whether the cell on ``side`` of each c-face exists and is fluid.""" + + face_dims = self._face_dims(c) + result = np.zeros(face_dims, dtype=bool) + target: list[slice] = [slice(None)] * 3 + target[c] = slice(1, None) if side < 0 else slice(0, -1) + result[target[0], target[1], target[2]] = cells + return result + + def _cell_beside_values(self, values: _FloatGrid, c: int, side: int) -> _FloatGrid: + face_dims = self._face_dims(c) + result = np.zeros(face_dims, dtype=np.float64) + target: list[slice] = [slice(None)] * 3 + target[c] = slice(1, None) if side < 0 else slice(0, -1) + result[target[0], target[1], target[2]] = values + return result + + def _shift(self, field: _Grid, axis: int, offset: int) -> _Grid: + """The field sampled at ``index + offset`` along ``axis``, zero beyond. + + Values past the array edge read as zero, and a mask shifted this way + reads as false, which is the wall the reflection tests look for. + """ + + result = np.zeros_like(field) + source: list[slice] = [slice(None)] * 3 + target: list[slice] = [slice(None)] * 3 + if offset > 0: + source[axis] = slice(1, None) + target[axis] = slice(0, -1) + else: + source[axis] = slice(0, -1) + target[axis] = slice(1, None) + result[target[0], target[1], target[2]] = field[source[0], source[1], source[2]] + return result + + def apply_momentum(self, c: int, u: _FloatGrid) -> _FloatGrid: + """``A u = -lap(u) + d u`` on active c-faces, zero elsewhere.""" + + active = self.active[c] + result = self.face_drag[c] * u + for a in self.live_axes: + inv_h2 = 1.0 / (self.spacing[a] * self.spacing[a]) + for offset in (-1, +1): + neighbor = self._shift(u, a, offset) + if a == c: + # Along the component axis neighbor faces hold genuine + # velocities (zero on walls). Active faces on the array + # edge - the flow-axis inlet and outlet - use a + # zero-gradient ghost equal to the face value. + edge = self._edge_slice(a, 0 if offset < 0 else -1) + ghost = np.zeros_like(u) + ghost[edge] = u[edge] + neighbor = neighbor + ghost + else: + # Across the component axis a neighbor location with no + # adjacent fluid cell lies inside a wall whose plane sits + # half a spacing away: reflect for no-slip. + reflect = ~self._shift(self.exists[c], a, offset) + neighbor = np.where(reflect, -u, neighbor) + result -= (neighbor - u) * inv_h2 + result[~active] = 0.0 + return result + + def divergence(self, velocity: list[_FloatGrid]) -> _FloatGrid: + result = np.zeros(self.dims, dtype=np.float64) + for c in range(3): + faces = velocity[c] + inv_h = 1.0 / self.spacing[c] + upper: list[slice] = [slice(None)] * 3 + lower: list[slice] = [slice(None)] * 3 + upper[c] = slice(1, None) + lower[c] = slice(0, -1) + result += ( + faces[upper[0], upper[1], upper[2]] - faces[lower[0], lower[1], lower[2]] + ) * inv_h + result[~self.fluid] = 0.0 + return result + + def gradient(self, pressure: _FloatGrid) -> list[_FloatGrid]: + """``-D^T p`` per component: the pressure gradient on active faces.""" + + fields: list[_FloatGrid] = [] + clean = np.where(self.fluid, pressure, 0.0) + for c in range(3): + face = np.zeros(self._face_dims(c), dtype=np.float64) + face += self._cell_beside_values(clean, c, +1) + face -= self._cell_beside_values(clean, c, -1) + face *= 1.0 / self.spacing[c] + face[~self.active[c]] = 0.0 + fields.append(face) + return fields + + +def solve_stokes_field( + spec: SignalGridSpec, + *, + mean_inlet_speed: float, + axis: str = "y", + drag: Sequence[float] | None = None, + tolerance: float = 1.0e-8, + max_outer_iterations: int = 500, + inner_tolerance: float = 1.0e-10, + max_inner_iterations: int = 50_000, +) -> tuple[SignalGridVelocityField, StokesSolveReport]: + """Solve the staggered Stokes-Brinkman flow and return the velocity field. + + Flow runs from the lower to the upper boundary of ``axis``; a negative + ``mean_inlet_speed`` reverses it. ``drag`` optionally gives one Brinkman + drag value (inverse permeability, units 1/length^2) per site; omitted or + zero drag is pure Stokes. + """ + + if not math.isfinite(mean_inlet_speed) or mean_inlet_speed == 0.0: + raise FlowError("mean inlet speed must be finite and nonzero") + flow_axis = _flow_axis_index(spec, axis) + + dims = (spec.shape.x, spec.shape.y, spec.shape.z) + spacing = (spec.spacing.x, spec.spacing.y, spec.spacing.z) + sites = dims[0] * dims[1] * dims[2] + obstacles = spec.obstacles + if obstacles: + if len(obstacles) != sites: + raise FlowError("obstacles must hold one flag per grid site") + fluid = np.asarray(obstacles, dtype=np.uint8).reshape(dims) == 0 + else: + fluid = np.ones(dims, dtype=bool) + if drag is None: + drag_grid = np.zeros(dims, dtype=np.float64) + else: + if len(drag) != sites: + raise FlowError("drag must hold one value per grid site") + drag_grid = np.asarray(drag, dtype=np.float64).reshape(dims) + if not bool(np.all(np.isfinite(drag_grid))) or bool(np.any(drag_grid < 0.0)): + raise FlowError("drag values must be finite and non-negative") + drag_grid = np.where(fluid, drag_grid, 0.0) + + operator = _StokesOperator(dims, spacing, fluid, drag_grid, flow_axis) + if not bool(np.any(operator.active[flow_axis][operator._edge_slice(flow_axis, 0)])): + raise FlowError("the inlet boundary is entirely blocked") + + # Momentum right-hand side from the prescribed inlet and outlet ghost + # pressures (one and zero). + force: list[_FloatGrid] = [ + np.zeros(operator._face_dims(c), dtype=np.float64) for c in range(3) + ] + inlet_slice = operator._edge_slice(flow_axis, 0) + inlet_active = operator.active[flow_axis][inlet_slice] + force[flow_axis][inlet_slice] = np.where( + inlet_active, 1.0 / spacing[flow_axis], 0.0 + ) + + # Momentum diagonals for the inner Jacobi preconditioner. + diagonals: list[_FloatGrid] = [] + for c in range(3): + diagonal = operator.face_drag[c].copy() + for a in operator.live_axes: + diagonal += 2.0 / (spacing[a] * spacing[a]) + diagonals.append(diagonal) + + inner_total = 0 + + def solve_momentum(rhs: list[_FloatGrid]) -> list[_FloatGrid]: + nonlocal inner_total + solution: list[_FloatGrid] = [] + for c in range(3): + component, iterations, _ = _conjugate_gradient( + lambda u, c=c: operator.apply_momentum(c, u), + rhs[c], + diagonals[c], + inner_tolerance, + max_inner_iterations, + mask=operator.active[c], + label="stokes momentum", + ) + inner_total += iterations + solution.append(component) + return solution + + particular = solve_momentum(force) + schur_rhs = -operator.divergence(particular) + + def apply_schur(q: _FloatGrid) -> _FloatGrid: + # gradient() is -D^T, so negating the divergence gives S = D A^-1 D^T. + return -operator.divergence(solve_momentum(operator.gradient(q))) + + pressure, outer_iterations, _ = _conjugate_gradient( + apply_schur, + schur_rhs, + np.ones(dims, dtype=np.float64), + tolerance, + max_outer_iterations, + mask=fluid, + label="stokes pressure", + ) + + correction = solve_momentum(operator.gradient(pressure)) + velocity = [particular[c] - correction[c] for c in range(3)] + divergence = operator.divergence(velocity) + divergence_rms = float(np.sqrt(np.mean(divergence[fluid] ** 2))) if bool( + np.any(fluid) + ) else 0.0 + + inlet_values = velocity[flow_axis][inlet_slice] + open_inlet = operator.active[flow_axis][inlet_slice] + solved_mean = float(np.mean(inlet_values[open_inlet])) + # The inlet must carry a real share of whatever the solve moved anywhere, + # which makes the test independent of drag, spacing, and grid size. + peak = max(float(np.max(np.abs(component))) for component in velocity) + if peak == 0.0 or solved_mean <= 1.0e-9 * peak: + raise FlowError("the device carries no through-flow: the outlet is unreachable") + factor = mean_inlet_speed / solved_mean + scaled = [component * factor for component in velocity] + + field = SignalGridVelocityField() + field.x_faces = [float(value) for value in scaled[0].ravel()] + field.y_faces = [float(value) for value in scaled[1].ravel()] + field.z_faces = [float(value) for value in scaled[2].ravel()] + max_speed = max(float(np.max(np.abs(component))) for component in scaled) + report = StokesSolveReport( + outer_iterations=outer_iterations, + inner_iterations=inner_total, + divergence_rms=divergence_rms * abs(factor), + mean_inlet_speed=mean_inlet_speed, + max_speed=max_speed, + min_gap_voxels=_minimum_gap_voxels(fluid, dims, flow_axis), + ) + return field, report diff --git a/python/tests/test_flow.py b/python/tests/test_flow.py new file mode 100644 index 0000000..7e603c5 --- /dev/null +++ b/python/tests/test_flow.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass + +import pytest +from cellmodeller2 import ( + GridBoundaryKind, + GridShape, + SignalGridSpec, + SignalGridVelocityField, + SignalIntegrationKind, + Simulation, + Vec3, +) +from cellmodeller2.flow import FlowError, colony_mobility, gap_mobility, solve_flow_field +from cellmodeller2.microfluidics import TrapChannelDevice + + +def _duct(nx: int = 4, ny: int = 8, nz: int = 3) -> SignalGridSpec: + shape = GridShape() + shape.x, shape.y, shape.z = nx, ny, nz + spec = SignalGridSpec() + spec.signal_count = 1 + spec.shape = shape + spec.spacing = Vec3(1.0, 1.0, 1.0) + spec.diffusion = [1.0] + spec.advection = [Vec3()] + for name in ("y_lower", "y_upper"): + boundary = getattr(spec, name) + boundary.kind = GridBoundaryKind.FIXED + boundary.values = [0.0] + setattr(spec, name, boundary) + return spec + + +def _site(spec: SignalGridSpec, x: int, y: int, z: int) -> int: + return (x * spec.shape.y + y) * spec.shape.z + z + + +def _y_face(spec: SignalGridSpec, x: int, fy: int, z: int) -> int: + return (x * (spec.shape.y + 1) + fy) * spec.shape.z + z + + +def _cross_section_fluxes(spec: SignalGridSpec, field: SignalGridVelocityField) -> list[float]: + return [ + sum( + field.y_faces[_y_face(spec, x, fy, z)] + for x in range(spec.shape.x) + for z in range(spec.shape.z) + ) + for fy in range(spec.shape.y + 1) + ] + + +def test_uniform_duct_is_exact_plug_flow() -> None: + spec = _duct() + field, report = solve_flow_field(spec, mean_inlet_speed=5.0) + assert all(math.isclose(value, 5.0, abs_tol=1.0e-8) for value in field.y_faces) + assert all(abs(value) < 1.0e-8 for value in field.x_faces) + assert all(abs(value) < 1.0e-8 for value in field.z_faces) + assert math.isclose(report.max_speed, 5.0, rel_tol=1.0e-9) + spec.velocity_field = field + spec.validate() + + +def test_parallel_channels_split_flux_in_the_mobility_ratio() -> None: + spec = _duct(nx=2, ny=6, nz=1) + mobility = [1.0 if x == 0 else 3.0 for x in range(2) for _ in range(6)] + field, _ = solve_flow_field(spec, mean_inlet_speed=4.0, mobility=mobility) + slow = field.y_faces[_y_face(spec, 0, 3, 0)] + fast = field.y_faces[_y_face(spec, 1, 3, 0)] + assert math.isclose(fast / slow, 3.0, rel_tol=1.0e-6) + assert math.isclose((slow + fast) / 2.0, 4.0, rel_tol=1.0e-9) + assert all(abs(value) < 1.0e-8 for value in field.x_faces) + + +def test_a_pillar_routes_flow_around_itself_conservatively() -> None: + spec = _duct(nx=5, ny=7, nz=1) + obstacles = [0] * (5 * 7) + for y in (2, 3, 4): + obstacles[_site(spec, 2, y, 0)] = 1 + spec.obstacles = obstacles + field, _ = solve_flow_field(spec, mean_inlet_speed=6.0) + + fluxes = _cross_section_fluxes(spec, field) + for flux in fluxes[1:]: + assert math.isclose(flux, fluxes[0], rel_tol=1.0e-6) + # Faces of the pillar carry no flow; its flanks carry more than the inlet mean. + assert field.y_faces[_y_face(spec, 2, 3, 0)] == 0.0 + assert field.y_faces[_y_face(spec, 1, 3, 0)] > 6.0 + assert any(value != 0.0 for value in field.x_faces) + spec.velocity_field = field + spec.validate() + + +def test_brinkman_drag_diverts_flux_from_a_porous_region() -> None: + spec = _duct(nx=2, ny=6, nz=1) + mobility = [1.0] * (2 * 6) + for y in (2, 3): + mobility[_site(spec, 1, y, 0)] = 0.05 + field, _ = solve_flow_field(spec, mean_inlet_speed=4.0, mobility=mobility) + open_flux = field.y_faces[_y_face(spec, 0, 3, 0)] + porous_flux = field.y_faces[_y_face(spec, 1, 3, 0)] + assert porous_flux > 0.0 + assert open_flux > 4.0 > porous_flux + fluxes = _cross_section_fluxes(spec, field) + for flux in fluxes[1:]: + assert math.isclose(flux, fluxes[0], rel_tol=1.0e-6) + + +def test_ill_posed_problems_are_rejected() -> None: + spec = _duct() + with pytest.raises(FlowError, match="one of x, y, z"): + solve_flow_field(spec, mean_inlet_speed=1.0, axis="w") + with pytest.raises(FlowError, match="finite and nonzero"): + solve_flow_field(spec, mean_inlet_speed=0.0) + with pytest.raises(FlowError, match="must be FIXED"): + solve_flow_field(spec, mean_inlet_speed=1.0, axis="x") + with pytest.raises(FlowError, match="one value per grid site"): + solve_flow_field(spec, mean_inlet_speed=1.0, mobility=[1.0]) + + periodic = _duct() + boundary = periodic.x_lower + boundary.kind = GridBoundaryKind.PERIODIC + periodic.x_lower = boundary + boundary = periodic.x_upper + boundary.kind = GridBoundaryKind.PERIODIC + periodic.x_upper = boundary + with pytest.raises(FlowError, match="periodic"): + solve_flow_field(periodic, mean_inlet_speed=1.0) + + blocked_inlet = _duct(nx=3, ny=4, nz=1) + obstacles = [0] * (3 * 4) + for x in range(3): + obstacles[_site(blocked_inlet, x, 0, 0)] = 1 + blocked_inlet.obstacles = obstacles + with pytest.raises(FlowError, match="entirely blocked"): + solve_flow_field(blocked_inlet, mean_inlet_speed=1.0) + + dead_end = _duct(nx=3, ny=4, nz=1) + obstacles = [0] * (3 * 4) + for x in range(3): + obstacles[_site(dead_end, x, 2, 0)] = 1 + dead_end.obstacles = obstacles + with pytest.raises(FlowError, match="no through-flow"): + solve_flow_field(dead_end, mean_inlet_speed=1.0) + + +@dataclass(frozen=True) +class _Rod: + position: Vec3 + length: float = 3.0 + radius: float = 0.5 + + +def test_colony_mobility_adds_drag_where_cells_pack() -> None: + spec = _duct(nx=3, ny=3, nz=1) + spec.spacing = Vec3(4.0, 4.0, 4.0) + obstacles = [0] * 9 + obstacles[_site(spec, 2, 2, 0)] = 1 + spec.obstacles = obstacles + # Site centers sit at multiples of the spacing: voxel (0,0,0) is centered + # on the origin and voxel (1,1,0) on (4, 4, 0). + crowd = [_Rod(Vec3(0.0, 0.0, 0.0)) for _ in range(40)] + lone = [_Rod(Vec3(4.0, 4.0, 0.0))] + outside = [_Rod(Vec3(-10.0, 0.0, 0.0))] + mobility = colony_mobility(spec, crowd + lone + outside, base=1.0, drag_coefficient=100.0) + packed = mobility[_site(spec, 0, 0, 0)] + sparse = mobility[_site(spec, 1, 1, 0)] + empty = mobility[_site(spec, 0, 2, 0)] + assert mobility[_site(spec, 2, 2, 0)] == 0.0 + assert packed < sparse < empty == 1.0 + # Packed voxels hit the volume-fraction cap rather than shrinking without bound. + capped = 1.0 / (1.0 + 100.0 * 0.9**2 / (1.0 - 0.9) ** 3) + assert math.isclose(packed, capped, rel_tol=1.0e-9) + + # A per-site base composes with the colony drag; zero-drag recovery is exact. + layered = colony_mobility(spec, [], base=[0.5] * 9, drag_coefficient=100.0) + assert layered[_site(spec, 1, 1, 0)] == 0.5 + assert layered[_site(spec, 2, 2, 0)] == 0.0 + + with pytest.raises(FlowError, match="finite and positive"): + colony_mobility(spec, [], base=0.0) + with pytest.raises(FlowError, match="one value per grid site"): + colony_mobility(spec, [], base=[1.0]) + with pytest.raises(FlowError, match="strictly between"): + colony_mobility(spec, [], max_volume_fraction=1.0) + + +def test_gap_mobility_scales_with_the_squared_gap_height() -> None: + spec = _duct(nx=2, ny=4, nz=4) + obstacles = [0] * (2 * 4 * 4) + for y in range(4): + for z in range(1, 4): + obstacles[_site(spec, 1, y, z)] = 1 + spec.obstacles = obstacles + mobility = gap_mobility(spec) + assert mobility[_site(spec, 0, 0, 0)] == 1.0 + assert math.isclose(mobility[_site(spec, 1, 0, 0)], 0.0625) + assert mobility[_site(spec, 1, 0, 2)] == 0.0 + + blocked = _duct(nx=1, ny=1, nz=1) + blocked.obstacles = [1] + with pytest.raises(FlowError, match="no fluid sites"): + gap_mobility(blocked) + + +def test_simulation_swaps_the_solved_field_at_runtime() -> None: + spec = _duct(nx=2, ny=4, nz=1) + field, _ = solve_flow_field(spec, mean_inlet_speed=3.0) + simulation = Simulation() + simulation.configure_signal_grid(spec, [0.0] * spec.site_count) + simulation.set_velocity_field(field) + + invalid = SignalGridVelocityField() + invalid.x_faces = [0.0] + invalid.y_faces = [0.0] + invalid.z_faces = [0.0] + with pytest.raises(ValueError, match="every lattice face"): + simulation.set_velocity_field(invalid) + simulation.set_velocity_field(None) + + +def test_a_solved_field_advects_signals_through_the_engine() -> None: + def _mid_level(with_flow: bool) -> float: + spec = _duct(nx=1, ny=12, nz=1) + spec.diffusion = [0.01] + spec.integration = SignalIntegrationKind.CRANK_NICOLSON + boundary = spec.y_lower + boundary.values = [10.0] + spec.y_lower = boundary + if with_flow: + field, _ = solve_flow_field(spec, mean_inlet_speed=2.0) + spec.velocity_field = field + simulation = Simulation() + simulation.configure_signal_grid(spec, [0.0] * spec.site_count) + for _ in range(10): + simulation.step(0.5) + return simulation.sample_signals(Vec3(0.5, 6.0, 0.5))[0] + + advected = _mid_level(with_flow=True) + diffused = _mid_level(with_flow=False) + assert advected > 5.0 + assert advected > 10.0 * diffused + + +def test_trap_channel_device_supports_a_numerical_field() -> None: + device = TrapChannelDevice(mean_flow_speed=20.0) + shape = GridShape() + shape.x, shape.y, shape.z = 64, 72, 4 + spec = SignalGridSpec() + spec.signal_count = 1 + spec.shape = shape + spec.origin = Vec3(-140.0, -144.0, -8.0) + spec.spacing = Vec3(4.0, 4.0, 4.0) + spec.diffusion = [40.0] + spec.advection = [Vec3()] + device.apply_to_grid(spec, inlet_values=[10.0], outlet_values=[0.0]) + + field, report = solve_flow_field(spec, mean_inlet_speed=20.0) + spec.velocity_field = field + spec.validate() + + # The straight channel carries plug flow near the requested mean; the + # dead-end trap sees only the weak recirculation at its mouth. + mid_face = shape.y // 2 + channel_speed = max( + field.y_faces[_y_face(spec, x, mid_face, z)] + for x in range(shape.x) + for z in range(shape.z) + ) + trap_column = int((0.0 - spec.origin.x) / spec.spacing.x) + trap_speed = abs(field.y_faces[_y_face(spec, trap_column, mid_face, 1)]) + assert channel_speed > 15.0 + assert trap_speed < channel_speed * 0.05 + assert report.max_speed >= channel_speed + + +def test_anisotropic_spacing_scales_the_solved_speeds() -> None: + """A duct's mean speed is set by the request, not by the lattice shape. + + The conductances divide by the squared spacing and the face fluxes + multiply it back, so a grid whose voxels are not cubes must still carry + exactly the requested speed, and a stretched cross-section must split + flux evenly across it. + """ + + shape = GridShape() + shape.x, shape.y, shape.z = 4, 6, 3 + spec = SignalGridSpec() + spec.signal_count = 1 + spec.shape = shape + spec.spacing = Vec3(5.0, 0.4, 1.65) + spec.diffusion = [1.0] + spec.advection = [Vec3()] + for name in ("y_lower", "y_upper"): + boundary = getattr(spec, name) + boundary.kind = GridBoundaryKind.FIXED + boundary.values = [0.0] + setattr(spec, name, boundary) + field, _ = solve_flow_field(spec, mean_inlet_speed=7.0) + spec.velocity_field = field + spec.validate() + assert all(math.isclose(value, 7.0, rel_tol=1.0e-8) for value in field.y_faces) + + +def test_reversed_and_transverse_flow_axes_solve() -> None: + spec = _duct() + field, _ = solve_flow_field(spec, mean_inlet_speed=-3.0) + spec.velocity_field = field + spec.validate() + assert all(math.isclose(value, -3.0, abs_tol=1.0e-8) for value in field.y_faces) + + across = _duct() + for name in ("y_lower", "y_upper"): + boundary = getattr(across, name) + boundary.kind = GridBoundaryKind.NO_FLUX + boundary.values = [] + setattr(across, name, boundary) + for name in ("x_lower", "x_upper"): + boundary = getattr(across, name) + boundary.kind = GridBoundaryKind.FIXED + boundary.values = [0.0] + setattr(across, name, boundary) + sideways, _ = solve_flow_field(across, mean_inlet_speed=2.0, axis="x") + across.velocity_field = sideways + across.validate() + assert all(math.isclose(value, 2.0, abs_tol=1.0e-8) for value in sideways.x_faces) + + +def test_partly_blocked_inlets_and_walled_off_pockets_solve() -> None: + spec = _duct(nx=4, ny=6, nz=1) + obstacles = [0] * 24 + for y in range(6): + obstacles[_site(spec, 0, y, 0)] = 1 + spec.obstacles = obstacles + field, _ = solve_flow_field(spec, mean_inlet_speed=2.0) + spec.velocity_field = field + spec.validate() + inlet = [field.y_faces[_y_face(spec, x, 0, 0)] for x in range(4)] + assert inlet[0] == 0.0 + assert all(math.isclose(value, 2.0, rel_tol=1.0e-8) for value in inlet[1:]) + fluxes = _cross_section_fluxes(spec, field) + assert all(math.isclose(flux, fluxes[0], rel_tol=1.0e-8) for flux in fluxes) + + pocket = _duct(nx=5, ny=6, nz=1) + sealed = [0] * 30 + for y in (2, 4): + for x in (3, 4): + sealed[_site(pocket, x, y, 0)] = 1 + sealed[_site(pocket, 2, 3, 0)] = 1 + pocket.obstacles = sealed + sealed_field, _ = solve_flow_field(pocket, mean_inlet_speed=1.0) + pocket.velocity_field = sealed_field + pocket.validate() + # The pocket is cut off from the flow, so it carries none. + assert sealed_field.y_faces[_y_face(pocket, 3, 3, 0)] == 0.0 + assert sealed_field.y_faces[_y_face(pocket, 4, 3, 0)] == 0.0 diff --git a/python/tests/test_stokes.py b/python/tests/test_stokes.py new file mode 100644 index 0000000..1216921 --- /dev/null +++ b/python/tests/test_stokes.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import math + +import numpy as np +import pytest +from cellmodeller2 import GridBoundaryKind, Vec3 +from cellmodeller2.flow import FlowError, gap_mobility, solve_flow_field +from cellmodeller2.flow_reference import ( + SQUARE_DUCT_PEAK_TO_MEAN, + centerline_value, + duct_grid, + plane_poiseuille, + site_index, + two_layer_brinkman, +) +from cellmodeller2.stokes import colony_drag, solve_stokes_field + + +def _plane_poiseuille_error(nx: int) -> float: + spec = duct_grid(nx, 6, 1, (1.0 / nx, 0.25, 1.0)) + field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-10) + profile = np.asarray(field.y_faces).reshape(nx, 7, 1)[:, 3, 0] + positions = (np.arange(nx) + 0.5) / nx + exact = plane_poiseuille(positions) + return float(np.max(np.abs(profile - exact)) / np.max(exact)) + + +def test_plane_poiseuille_profile_converges_at_second_order() -> None: + coarse = _plane_poiseuille_error(8) + fine = _plane_poiseuille_error(16) + assert coarse < 0.02 + assert fine < 0.005 + assert 3.0 < coarse / fine < 5.0 + + +def test_square_duct_peak_to_mean_matches_shah_and_london() -> None: + # u_max / u_mean = 2.0962 for a square duct (Shah & London 1978). + n = 16 + spec = duct_grid(n, 6, n, (1.0 / n, 0.25, 1.0 / n)) + field, report = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + cross = np.asarray(field.y_faces).reshape(n, 7, n)[:, 3, :] + ratio = centerline_value(cross) / float(cross.mean()) + assert abs(ratio - SQUARE_DUCT_PEAK_TO_MEAN) / SQUARE_DUCT_PEAK_TO_MEAN < 0.015 + assert report.divergence_rms < 1.0e-6 + + +def test_two_layer_brinkman_channel_matches_the_exact_solution() -> None: + drag_value = 200.0 + nz = 32 + spec = duct_grid(1, 6, nz, (1.0, 0.25, 1.0 / nz)) + drag = [ + 0.0 if (z + 0.5) / nz < 0.5 else drag_value + for _ in range(1) + for _ in range(6) + for z in range(nz) + ] + field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, drag=drag, tolerance=1.0e-9) + profile = np.asarray(field.y_faces).reshape(1, 7, nz)[0, 3, :] + positions = (np.arange(nz) + 0.5) / nz + # The solve rescales to the requested mean speed, so both profiles are + # compared at unit mean. + exact = two_layer_brinkman(drag_value, positions) + exact = exact / exact.mean() + error = float(np.max(np.abs(profile / profile.mean() - exact)) / np.max(np.abs(exact))) + assert error < 0.01 + # The open layer carries several times the porous layer's flux. + open_flux = float(profile[positions < 0.5].sum()) + porous_flux = float(profile[positions >= 0.5].sum()) + assert open_flux / porous_flux > 3.0 + + +def test_zero_drag_recovers_pure_stokes() -> None: + spec = duct_grid(8, 6, 1, (0.125, 0.25, 1.0)) + plain, _ = solve_stokes_field(spec, mean_inlet_speed=2.0) + dragged, _ = solve_stokes_field(spec, mean_inlet_speed=2.0, drag=[0.0] * (8 * 6)) + assert plain.y_faces == dragged.y_faces + + +def test_stokes_field_is_engine_valid_and_conservative_around_a_pillar() -> None: + spec = duct_grid(9, 12, 1, (1.0, 1.0, 1.0)) + obstacles = [0] * (9 * 12) + for y in (5, 6): + for x in (4, 5): + obstacles[site_index(spec, x, y, 0)] = 1 + spec.obstacles = obstacles + field, report = solve_stokes_field(spec, mean_inlet_speed=6.0, tolerance=1.0e-9) + spec.velocity_field = field + spec.validate() + assert report.divergence_rms < 1.0e-6 + + def y_face(x: int, fy: int) -> float: + return field.y_faces[x * 13 + fy] + + fluxes = [sum(y_face(x, fy) for x in range(9)) for fy in range(13)] + for flux in fluxes[1:]: + assert math.isclose(flux, fluxes[0], rel_tol=1.0e-5) + assert y_face(4, 6) == 0.0 + assert y_face(1, 6) > 6.0 + + +def test_thin_gap_stokes_depth_averages_to_the_hele_shaw_solution() -> None: + # A shallow channel with a pillar: depth-averaging the resolved MAC field + # must reproduce the Hele-Shaw flux split around the pillar. + nx, ny, nz = 6, 10, 6 + spec = duct_grid(nx, ny, nz, (1.0, 1.0, 0.05)) + obstacles = [0] * (nx * ny * nz) + for y in (4, 5): + for x in (1, 2): + for z in range(nz): + obstacles[site_index(spec, x, y, z)] = 1 + spec.obstacles = obstacles + + stokes_field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + hele_shaw_field, _ = solve_flow_field( + spec, mean_inlet_speed=1.0, mobility=gap_mobility(spec) + ) + + def column_flux(field_values: list[float], x: int, fy: int) -> float: + return sum(field_values[(x * (ny + 1) + fy) * nz + z] for z in range(nz)) + + mid = ny // 2 + stokes_split = [column_flux(stokes_field.y_faces, x, mid) for x in range(nx)] + hele_shaw_split = [column_flux(hele_shaw_field.y_faces, x, mid) for x in range(nx)] + stokes_total = sum(stokes_split) + hele_shaw_total = sum(hele_shaw_split) + for x in range(nx): + assert math.isclose( + stokes_split[x] / stokes_total, + hele_shaw_split[x] / hele_shaw_total, + abs_tol=0.01, + ) + + +def test_ill_posed_stokes_problems_are_rejected() -> None: + spec = duct_grid(4, 6, 1, (1.0, 1.0, 1.0)) + with pytest.raises(FlowError, match="one of x, y, z"): + solve_stokes_field(spec, mean_inlet_speed=1.0, axis="w") + with pytest.raises(FlowError, match="finite and nonzero"): + solve_stokes_field(spec, mean_inlet_speed=0.0) + with pytest.raises(FlowError, match="must be FIXED"): + solve_stokes_field(spec, mean_inlet_speed=1.0, axis="x") + with pytest.raises(FlowError, match="one value per grid site"): + solve_stokes_field(spec, mean_inlet_speed=1.0, drag=[1.0]) + + blocked = duct_grid(3, 4, 1, (1.0, 1.0, 1.0)) + obstacles = [0] * 12 + for x in range(3): + obstacles[site_index(blocked, x, 2, 0)] = 1 + blocked.obstacles = obstacles + with pytest.raises(FlowError, match="no through-flow"): + solve_stokes_field(blocked, mean_inlet_speed=1.0) + + +def test_colony_drag_rasterizes_the_colony() -> None: + spec = duct_grid(3, 3, 1, (4.0, 4.0, 4.0)) + obstacles = [0] * 9 + obstacles[site_index(spec, 2, 2, 0)] = 1 + spec.obstacles = obstacles + + class _Rod: + def __init__(self, x: float, y: float) -> None: + self.position = Vec3(x, y, 0.0) + self.length = 3.0 + self.radius = 0.5 + + crowd = [_Rod(0.0, 0.0) for _ in range(40)] + drag = colony_drag(spec, crowd, drag_coefficient=50.0) + packed = drag[site_index(spec, 0, 0, 0)] + empty = drag[site_index(spec, 1, 1, 0)] + solid = drag[site_index(spec, 2, 2, 0)] + assert packed > 0.0 + assert empty == 0.0 + assert solid == 0.0 + assert math.isclose(packed, 50.0 * 0.9**2 / (1.0 - 0.9) ** 3, rel_tol=1.0e-9) + with pytest.raises(FlowError, match="finite and non-negative"): + colony_drag(spec, [], drag_coefficient=-1.0) + + +def test_thin_gaps_over_predict_flux_until_they_are_resolved() -> None: + """The MAC solve needs several voxels across a channel to resolve no-slip. + + Two stacked channels of gap ratio four carry flux in the ratio of their + cubed gaps, so their mean velocities differ by the squared ratio. A gap + one voxel across cannot hold a parabola and carries far too much; the + ratio approaches the lubrication limit as the gap resolves, and the report + names the resolution so a caller can judge the error. + """ + + lubrication = 1.0 / 16.0 + errors: list[float] = [] + for thin in (1, 2, 4, 8): + nz = thin + 1 + 4 * thin + spec = duct_grid(1, 8, nz, (1.0, 1.0, 1.0)) + obstacles = [0] * (8 * nz) + for y in range(8): + obstacles[site_index(spec, 0, y, thin)] = 1 + spec.obstacles = obstacles + field, report = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + profile = np.asarray(field.y_faces).reshape(1, 9, nz)[0, 4, :] + ratio = float(profile[:thin].mean() / profile[thin + 1 :].mean()) + errors.append(ratio / lubrication) + assert report.min_gap_voxels == thin + assert errors[0] > 2.0 + assert errors[1] < errors[0] + assert errors[2] < 1.2 + assert errors[3] < 1.05 + + +def test_reversed_and_transverse_flow_axes_solve() -> None: + forward = duct_grid(8, 6, 1, (0.125, 0.25, 1.0)) + field, report = solve_stokes_field(forward, mean_inlet_speed=-2.0) + forward.velocity_field = field + forward.validate() + inlet = np.asarray(field.y_faces).reshape(8, 7, 1)[:, 0, 0] + assert math.isclose(float(inlet.mean()), -2.0, rel_tol=1.0e-6) + assert report.max_speed > 0.0 + + # The same channel across x reproduces the same profile. + across = duct_grid(6, 8, 1, (0.25, 0.125, 1.0)) + across.y_lower.kind = GridBoundaryKind.NO_FLUX + across.y_lower.values = [] + across.y_upper.kind = GridBoundaryKind.NO_FLUX + across.y_upper.values = [] + for name in ("x_lower", "x_upper"): + boundary = getattr(across, name) + boundary.kind = GridBoundaryKind.FIXED + boundary.values = [0.0] + setattr(across, name, boundary) + sideways, _ = solve_stokes_field(across, mean_inlet_speed=1.0, axis="x") + across.velocity_field = sideways + across.validate() + profile = np.asarray(sideways.x_faces).reshape(7, 8, 1)[3, :, 0] + positions = (np.arange(8) + 0.5) / 8 + error = float(np.max(np.abs(profile - plane_poiseuille(positions))) / 1.5) + assert error < 0.02 + + +def test_partly_blocked_inlets_and_walled_off_pockets_solve() -> None: + spec = duct_grid(4, 6, 1, (1.0, 1.0, 1.0)) + obstacles = [0] * 24 + for y in range(6): + obstacles[site_index(spec, 0, y, 0)] = 1 + spec.obstacles = obstacles + field, report = solve_stokes_field(spec, mean_inlet_speed=2.0, tolerance=1.0e-9) + spec.velocity_field = field + spec.validate() + inlet = np.asarray(field.y_faces).reshape(4, 7, 1)[:, 0, 0] + # The mean is taken over open inlet faces, and the blocked column is still. + assert math.isclose(float(inlet[1:].mean()), 2.0, rel_tol=1.0e-6) + assert inlet[0] == 0.0 + assert report.divergence_rms < 1.0e-6 + + # A fluid site sealed off from the flow leaves the solve well posed. + pocket = duct_grid(5, 6, 1, (1.0, 1.0, 1.0)) + sealed = [0] * 30 + for y in (2, 4): + for x in (3, 4): + sealed[site_index(pocket, x, y, 0)] = 1 + for x in (3, 4): + sealed[site_index(pocket, x, 3, 0)] = 0 + sealed[site_index(pocket, 2, 3, 0)] = 1 + pocket.obstacles = sealed + sealed_field, sealed_report = solve_stokes_field(pocket, mean_inlet_speed=1.0) + pocket.velocity_field = sealed_field + pocket.validate() + assert sealed_report.divergence_rms < 1.0e-6 + assert sealed_field.y_faces[(3 * 7) + 3] == 0.0 diff --git a/scripts/run_flow_benchmarks.py b/scripts/run_flow_benchmarks.py new file mode 100644 index 0000000..a3bb122 --- /dev/null +++ b/scripts/run_flow_benchmarks.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Validate the flow solvers against literature and exact solutions. + +Runs the benchmark suite for both flow solvers - the Hele-Shaw closure +(`cellmodeller2.flow`) and the staggered MAC Stokes-Brinkman solver +(`cellmodeller2.stokes`) - and prints a table of computed values against their +references. Exits nonzero if any benchmark exceeds its tolerance, so the +script doubles as a CI gate. + +The reference solutions and their citations live in +`cellmodeller2.flow_reference`, so this script and the test suite measure the +same physics. + +Usage: uv run python scripts/run_flow_benchmarks.py [--fine] +`--fine` doubles every benchmark's resolution to demonstrate mesh convergence. +""" + +from __future__ import annotations + +import argparse +import math +import sys +import time +from dataclasses import dataclass + +import numpy as np +from cellmodeller2.flow import gap_mobility, solve_flow_field +from cellmodeller2.flow_reference import ( + SQUARE_DUCT_PEAK_TO_MEAN, + centerline_value, + duct_grid, + plane_poiseuille, + site_index, + two_layer_brinkman, +) +from cellmodeller2.stokes import solve_stokes_field + + +@dataclass(frozen=True) +class Result: + solver: str + benchmark: str + metric: str + computed: float + reference: float + tolerance: float + seconds: float + + @property + def error(self) -> float: + # A zero reference marks a benchmark whose computed value is itself an + # error measure; the tolerance then bounds it absolutely. + if self.reference == 0.0: + return abs(self.computed) + return abs(self.computed - self.reference) / abs(self.reference) + + @property + def passed(self) -> bool: + return self.error <= self.tolerance + + +def bench_plane_poiseuille_order(coarse: int) -> list[Result]: + results: list[Result] = [] + errors: list[float] = [] + for n in (coarse, coarse * 2): + start = time.perf_counter() + spec = duct_grid(n, 6, 1, (1.0 / n, 0.25, 1.0)) + field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-10) + profile = np.asarray(field.y_faces).reshape(n, 7, 1)[:, 3, 0] + positions = (np.arange(n) + 0.5) / n + exact = plane_poiseuille(positions) + error = float(np.max(np.abs(profile - exact)) / np.max(exact)) + errors.append(error) + results.append( + Result( + "stokes", + f"plane Poiseuille n={n}", + "max relative profile error", + error, + 0.0, + 0.03 if n == coarse else 0.008, + time.perf_counter() - start, + ) + ) + order = math.log2(errors[0] / errors[1]) + results.append( + Result( + "stokes", + "plane Poiseuille refinement", + "observed convergence order", + order, + 2.0, + 0.25, + 0.0, + ) + ) + return results + + +def bench_square_duct(n: int) -> Result: + start = time.perf_counter() + spec = duct_grid(n, 6, n, (1.0 / n, 0.25, 1.0 / n)) + field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + cross = np.asarray(field.y_faces).reshape(n, 7, n)[:, 3, :] + # Cell centers straddle the duct axis, so the peak is interpolated rather + # than taken from the largest sample, which would understate it. + ratio = centerline_value(cross) / float(cross.mean()) + return Result( + "stokes", + f"square duct (n={n})", + f"u_max / u_mean (Shah & London: {SQUARE_DUCT_PEAK_TO_MEAN})", + ratio, + SQUARE_DUCT_PEAK_TO_MEAN, + 0.015 * (16.0 / n) ** 2, + time.perf_counter() - start, + ) + + +def bench_two_layer_brinkman(coarse: int) -> list[Result]: + drag_value = 200.0 + results: list[Result] = [] + errors: list[float] = [] + for nz in (coarse, coarse * 2): + start = time.perf_counter() + spec = duct_grid(1, 6, nz, (1.0, 0.25, 1.0 / nz)) + drag = [ + 0.0 if (z + 0.5) / nz < 0.5 else drag_value + for _ in range(6) + for z in range(nz) + ] + field, _ = solve_stokes_field( + spec, mean_inlet_speed=1.0, drag=drag, tolerance=1.0e-9 + ) + profile = np.asarray(field.y_faces).reshape(1, 7, nz)[0, 3, :] + positions = (np.arange(nz) + 0.5) / nz + # The solve rescales to the requested mean speed, so amplitude carries + # no information: both profiles are compared at unit mean. + exact = two_layer_brinkman(drag_value, positions) + exact = exact / exact.mean() + error = float(np.max(np.abs(profile / profile.mean() - exact)) / np.max(np.abs(exact))) + errors.append(error) + results.append( + Result( + "stokes", + f"two-layer Brinkman (n={nz})", + "max relative profile error vs exact ODE", + error, + 0.0, + 0.01 if nz == coarse else 0.003, + time.perf_counter() - start, + ) + ) + results.append( + Result( + "stokes", + "two-layer Brinkman refinement", + "observed convergence order", + math.log2(errors[0] / errors[1]), + 2.0, + 0.3, + 0.0, + ) + ) + return results + + +def bench_hele_shaw_duct(scale: int) -> Result: + start = time.perf_counter() + spec = duct_grid(4 * scale, 8 * scale, 3 * scale, (1.0, 1.0, 1.0)) + field, _ = solve_flow_field(spec, mean_inlet_speed=5.0) + error = float(max(abs(v - 5.0) for v in field.y_faces)) + return Result( + "hele-shaw", + "uniform duct", + "max |u - mean| (exact plug flow)", + error, + 0.0, + 1.0e-6, + time.perf_counter() - start, + ) + + +def bench_hele_shaw_mobility_split(scale: int) -> Result: + start = time.perf_counter() + columns, rows = 2 * scale, 6 * scale + spec = duct_grid(columns, rows, 1, (1.0, 1.0, 1.0)) + mobility = [ + 1.0 if x < columns // 2 else 3.0 for x in range(columns) for _ in range(rows) + ] + field, _ = solve_flow_field(spec, mean_inlet_speed=4.0, mobility=mobility) + middle = rows // 2 + slow = field.y_faces[0 * (rows + 1) + middle] + fast = field.y_faces[(columns - 1) * (rows + 1) + middle] + return Result( + "hele-shaw", + "parallel channels", + "flux ratio at mobility ratio 3 (exact 3)", + fast / slow, + 3.0, + 1.0e-5, + time.perf_counter() - start, + ) + + +def bench_cross_solver_consistency(scale: int) -> Result: + start = time.perf_counter() + nx, ny, nz = 6 * scale, 10 * scale, 6 * scale + spec = duct_grid(nx, ny, nz, (1.0 / scale, 1.0 / scale, 0.05 / scale)) + obstacles = [0] * (nx * ny * nz) + for y in range(4 * scale, 6 * scale): + for x in range(scale, 3 * scale): + for z in range(nz): + obstacles[site_index(spec, x, y, z)] = 1 + spec.obstacles = obstacles + stokes_field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + hele_shaw_field, _ = solve_flow_field( + spec, mean_inlet_speed=1.0, mobility=gap_mobility(spec) + ) + + def column_flux(values: list[float], x: int, fy: int) -> float: + return sum(values[(x * (ny + 1) + fy) * nz + z] for z in range(nz)) + + mid = ny // 2 + stokes_split = np.array([column_flux(stokes_field.y_faces, x, mid) for x in range(nx)]) + hele_shaw_split = np.array( + [column_flux(hele_shaw_field.y_faces, x, mid) for x in range(nx)] + ) + deviation = float( + np.max(np.abs(stokes_split / stokes_split.sum() - hele_shaw_split / hele_shaw_split.sum())) + ) + return Result( + "cross-check", + "thin-gap pillar", + "max |flux-share difference| MAC vs Hele-Shaw", + deviation, + 0.0, + 0.01, + time.perf_counter() - start, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fine", action="store_true", help="double the benchmark resolutions" + ) + arguments = parser.parse_args() + scale = 2 if arguments.fine else 1 + + results: list[Result] = [] + results.extend(bench_plane_poiseuille_order(8 * scale)) + results.append(bench_square_duct(16 * scale)) + results.extend(bench_two_layer_brinkman(32 * scale)) + results.append(bench_hele_shaw_duct(scale)) + results.append(bench_hele_shaw_mobility_split(scale)) + results.append(bench_cross_solver_consistency(scale)) + + width = max(len(r.benchmark) for r in results) + print(f"{'solver':<11} {'benchmark':<{width}} {'computed':>10} {'reference':>10} " + f"{'error':>9} {'tol':>7} {'time':>7} status") + failures = 0 + for r in results: + status = "pass" if r.passed else "FAIL" + if not r.passed: + failures += 1 + print( + f"{r.solver:<11} {r.benchmark:<{width}} {r.computed:>10.5f} " + f"{r.reference:>10.5f} {r.error:>9.5f} {r.tolerance:>7.3g} " + f"{r.seconds:>6.2f}s {status} [{r.metric}]" + ) + print(f"\n{len(results) - failures}/{len(results)} benchmarks passed") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 1ac63e04a632dc8d8872dcfd874591a15bb0a00f Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Tue, 18 Aug 2026 15:08:07 -0600 Subject: [PATCH 2/4] Let a running simulation swap its signal grid velocity field --- cpp/core/signals.cpp | 7 ++++++ cpp/core/simulation.cpp | 7 ++++++ cpp/include/cm/signals.hpp | 1 + cpp/include/cm/simulation.hpp | 1 + cpp/python/bindings.cpp | 1 + python/src/cellmodeller2/_core.pyi | 1 + tests/cpp/signal_grid_test.cpp | 36 ++++++++++++++++++++++++++++++ 7 files changed, 54 insertions(+) diff --git a/cpp/core/signals.cpp b/cpp/core/signals.cpp index afc4de8..c96c014 100644 --- a/cpp/core/signals.cpp +++ b/cpp/core/signals.cpp @@ -597,6 +597,13 @@ void SignalGrid::replace_levels(std::vector levels) { levels_ = std::move(levels); } +void SignalGrid::set_velocity_field(std::optional field) { + SignalGridCheckpoint candidate{.spec = spec_, .levels = levels_}; + candidate.spec.velocity_field = std::move(field); + candidate.validate(); + spec_ = std::move(candidate.spec); +} + void SignalGrid::validate_step(float dt) const { if (!std::isfinite(dt) || dt < 0.0F) { throw std::invalid_argument("time step must be finite and non-negative"); diff --git a/cpp/core/simulation.cpp b/cpp/core/simulation.cpp index 85113f5..589e790 100644 --- a/cpp/core/simulation.cpp +++ b/cpp/core/simulation.cpp @@ -171,6 +171,13 @@ void Simulation::set_signal_levels(std::span levels) { signal_grid_->set_levels(levels); } +void Simulation::set_velocity_field(std::optional field) { + if (!signal_grid_.has_value()) { + throw std::logic_error("simulation does not have a signal grid"); + } + signal_grid_->set_velocity_field(std::move(field)); +} + std::pair Simulation::divide(CellId parent_id, float first_fraction) { return state_.divide(parent_id, first_fraction); } diff --git a/cpp/include/cm/signals.hpp b/cpp/include/cm/signals.hpp index 9b6b89e..f08d802 100644 --- a/cpp/include/cm/signals.hpp +++ b/cpp/include/cm/signals.hpp @@ -124,6 +124,7 @@ class SignalGrid { [[nodiscard]] SignalGridCheckpoint checkpoint() const; void set_levels(std::span levels); void replace_levels(std::vector levels); + void set_velocity_field(std::optional field); void validate_step(float dt) const; void validate() const; diff --git a/cpp/include/cm/simulation.hpp b/cpp/include/cm/simulation.hpp index eead8e6..81eb6bb 100644 --- a/cpp/include/cm/simulation.hpp +++ b/cpp/include/cm/simulation.hpp @@ -44,6 +44,7 @@ class Simulation { void clear_coupled_rate_plan() noexcept; void configure_signal_grid(const SignalGridSpec& spec, std::vector levels = {}); void set_signal_levels(std::span levels); + void set_velocity_field(std::optional field); std::pair divide(CellId parent_id, float first_fraction); std::pair divide_equal(CellId parent_id); void step(float dt); diff --git a/cpp/python/bindings.cpp b/cpp/python/bindings.cpp index 930c35d..6713823 100644 --- a/cpp/python/bindings.cpp +++ b/cpp/python/bindings.cpp @@ -487,6 +487,7 @@ NB_MODULE(_core, module) { simulation.set_signal_levels(levels); }, "levels"_a) + .def("set_velocity_field", &cm::Simulation::set_velocity_field, "field"_a.none()) .def("divide", &cm::Simulation::divide, "parent_id"_a, "first_fraction"_a) .def("divide_equal", &cm::Simulation::divide_equal, "parent_id"_a) .def("step", &cm::Simulation::step, "dt"_a) diff --git a/python/src/cellmodeller2/_core.pyi b/python/src/cellmodeller2/_core.pyi index 2f5ff1c..8fb910c 100644 --- a/python/src/cellmodeller2/_core.pyi +++ b/python/src/cellmodeller2/_core.pyi @@ -529,6 +529,7 @@ class Simulation: def clear_coupled_rate_plan(self) -> None: ... def configure_signal_grid(self, spec: SignalGridSpec, levels: list[float] = ...) -> None: ... def set_signal_levels(self, levels: list[float]) -> None: ... + def set_velocity_field(self, field: SignalGridVelocityField | None) -> None: ... def divide(self, parent_id: int, first_fraction: float) -> tuple[int, int]: ... def divide_equal(self, parent_id: int) -> tuple[int, int]: ... def step(self, dt: float) -> None: ... diff --git a/tests/cpp/signal_grid_test.cpp b/tests/cpp/signal_grid_test.cpp index 89ea1c8..61b3a38 100644 --- a/tests/cpp/signal_grid_test.cpp +++ b/tests/cpp/signal_grid_test.cpp @@ -367,4 +367,40 @@ int main() { invalid.advection = {{0.5F, 0.0F, 0.0F}}; assert_throws([&] { invalid.validate(); }); } + + { + auto spec = line_spec(3); + spec.diffusion = {0.0F}; + spec.x_lower.kind = cm::GridBoundaryKind::fixed; + spec.x_lower.values = {0.0F}; + spec.x_upper.kind = cm::GridBoundaryKind::fixed; + spec.x_upper.values = {0.0F}; + + cm::SignalGrid grid(spec); + grid.set_velocity_field(cm::SignalGridVelocityField{ + .x_faces = std::vector(4, 2.0F), + .y_faces = std::vector(6, 0.0F), + .z_faces = std::vector(6, 0.0F), + }); + assert_throws([&grid] { + grid.set_velocity_field(cm::SignalGridVelocityField{ + .x_faces = std::vector(5, 0.0F), + .y_faces = std::vector(6, 0.0F), + .z_faces = std::vector(6, 0.0F), + }); + }); + grid.set_velocity_field(std::nullopt); + + cm::Simulation simulation; + simulation.configure_signal_grid(spec); + simulation.set_velocity_field(cm::SignalGridVelocityField{ + .x_faces = std::vector(4, 2.0F), + .y_faces = std::vector(6, 0.0F), + .z_faces = std::vector(6, 0.0F), + }); + simulation.set_velocity_field(std::nullopt); + + cm::Simulation bare; + assert_throws([&bare] { bare.set_velocity_field(std::nullopt); }); + } } From 5fa1cc6eb21e4642a985b512487cfd1bf3a6c747 Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Thu, 20 Aug 2026 15:56:10 -0600 Subject: [PATCH 3/4] Separate mask layouts from device dimensions --- python/src/cellmodeller2/masks.py | 14 +-- python/src/cellmodeller2/microfluidics.py | 53 ++-------- python/tests/test_masks.py | 76 ++++++++++++++ python/tests/test_microfluidics.py | 118 ++++++++++++++++++++++ 4 files changed, 210 insertions(+), 51 deletions(-) create mode 100644 python/tests/test_masks.py create mode 100644 python/tests/test_microfluidics.py diff --git a/python/src/cellmodeller2/masks.py b/python/src/cellmodeller2/masks.py index 011f139..b6c6d0a 100644 --- a/python/src/cellmodeller2/masks.py +++ b/python/src/cellmodeller2/masks.py @@ -6,10 +6,9 @@ closed `LWPOLYLINE` outlines from the model-space `ENTITIES` section of an ASCII DXF, ignoring block definitions (orphaned array remnants in mask files), paper space, and every other entity kind. Files are size-bounded and nothing is -executed. - -Mask drawings conventionally use one drawing unit per millimeter; pass -``unit_scale=1000.0`` to obtain micrometer coordinates. +executed. Coordinates remain in drawing units unless the caller supplies a +source-specific ``unit_scale`` to ``extract_rectangles``; the reader does not +infer physical units from a DXF header or file convention. """ from __future__ import annotations @@ -50,9 +49,10 @@ def load_mask_polylines( """Read the polylines of an ASCII DXF mask drawing. Model-space entities always load. With ``include_blocks``, polylines inside - block definitions load as well, tagged with their block name; mask files - from array-based CAD workflows often keep real geometry only in otherwise - orphaned blocks, in world coordinates. + block definitions load as authored and are tagged with their block name. + INSERT transforms are not applied, so callers may interpret block geometry + only when the relevant definitions are known to use the desired coordinate + system directly. """ source = Path(path) diff --git a/python/src/cellmodeller2/microfluidics.py b/python/src/cellmodeller2/microfluidics.py index a30b142..a6a3e23 100644 --- a/python/src/cellmodeller2/microfluidics.py +++ b/python/src/cellmodeller2/microfluidics.py @@ -16,8 +16,6 @@ from __future__ import annotations -import os -from collections import Counter from dataclasses import dataclass from ._core import ( # pyright: ignore[reportMissingModuleSource] @@ -30,7 +28,6 @@ Vec3, ) from .flow import gap_mobility, solve_flow_field -from .masks import MaskError, extract_rectangles, load_mask_polylines # A voxel edge that lands on a wall plane belongs to the wall, so the voxel # tests admit a rounding margin: without it a wall drawn exactly on a lattice @@ -231,13 +228,17 @@ class BiopixelTrapDevice(_ChannelDevice): toward the channel, ``trap_width`` in y, and only ``trap_height`` in z, so the colony grows as a monolayer under the cavity ceiling. The flow channel runs along y between ``-channel_width`` and ``0`` at the full - ``channel_height``. The device floor is ``z = 0``. An array device repeats - this trap along its channels; every trap sees the same fresh-media flow, so - one simulated trap is representative of each biopixel in the array when - inter-trap coupling is not modeled. + ``channel_height``. The device floor is ``z = 0``. + + The default trap dimensions are the 100 by 85 by 1.65 micrometer trapping + region reported by Prindle et al. (Nature 481, 39-44, 2012; + doi:10.1038/nature10722). Channel dimensions, numerical wall thickness, and + flow speed are modeling inputs rather than measurements from that study. A + single instance makes no claim that every trap in an array has identical + local flow or concentration boundary conditions. """ - trap_depth: float = 95.0 + trap_depth: float = 85.0 trap_width: float = 100.0 trap_height: float = 1.65 channel_width: float = 100.0 @@ -246,42 +247,6 @@ class BiopixelTrapDevice(_ChannelDevice): wall_thickness: float = 10.0 mean_flow_speed: float = 0.0 - @classmethod - def from_mask( - cls, - path: str | os.PathLike[str], - *, - layer: str = "Layer-2", - wall_inset: float = 5.0, - unit_scale: float = 1000.0, - mean_flow_speed: float = 0.0, - ) -> BiopixelTrapDevice: - """Derive the trap footprint from a photomask drawing. - - The mask draws each trap's outer wall outline. The cavity is the - outline minus ``wall_inset`` per wall: two side walls across the long - dimension and one back wall across the short dimension, whose remaining - side is the open face toward the channel. The drawing must contain one - uniform trap population on the layer. - """ - - polylines = load_mask_polylines(path) - rectangles = extract_rectangles(polylines, layer=layer, unit_scale=unit_scale) - if not rectangles: - raise MaskError(f"mask layer {layer!r} contains no rectangles") - sizes = Counter( - (round(max(r.width, r.height), 3), round(min(r.width, r.height), 3)) - for r in rectangles - ) - (long_side, short_side), count = sizes.most_common(1)[0] - if count < 2: - raise MaskError(f"mask layer {layer!r} has no repeated trap outline") - width = long_side - 2.0 * wall_inset - depth = short_side - wall_inset - if width <= 0.0 or depth <= 0.0: - raise MaskError("wall inset leaves no cavity") - return cls(trap_width=width, trap_depth=depth, mean_flow_speed=mean_flow_speed) - def add_constraints(self, simulation: Simulation) -> None: """Add the device's wall constraints to a simulation.""" diff --git a/python/tests/test_masks.py b/python/tests/test_masks.py new file mode 100644 index 0000000..a0c85e7 --- /dev/null +++ b/python/tests/test_masks.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import math +from pathlib import Path + +import pytest +from cellmodeller2.masks import ( + MaskError, + MaskPolyline, + extract_rectangles, + load_mask_polylines, + match_rectangles, +) + + +def test_rectangle_extraction_is_selective_and_explicitly_scaled() -> None: + polylines = ( + MaskPolyline("A", True, ((0.0, 0.0), (2.0, 0.0), (2.0, 1.0), (0.0, 1.0))), + MaskPolyline("A", False, ((0.0, 0.0), (2.0, 0.0), (2.0, 1.0), (0.0, 1.0))), + MaskPolyline("A", True, ((0.0, 0.0), (2.0, 0.5), (2.0, 1.0), (0.0, 1.0))), + MaskPolyline("B", True, ((0.0, 0.0), (1.0, 0.0), (1.0, 2.0), (0.0, 2.0), (0.0, 0.0))), + ) + rectangles = extract_rectangles(polylines, unit_scale=10.0) + assert len(rectangles) == 2 + assert math.isclose(rectangles[0].width, 20.0) + assert rectangles[0].center == (10.0, 5.0) + + layered = extract_rectangles(polylines, layer="B", unit_scale=10.0) + assert len(layered) == 1 + assert math.isclose(layered[0].height, 20.0) + + rotated = match_rectangles(rectangles, 10.0, 20.0, tolerance=0.01) + assert len(rotated) == 2 + strict = match_rectangles(rectangles, 10.0, 20.0, tolerance=0.01, allow_rotated=False) + assert len(strict) == 1 + + +def test_mask_reader_rejects_unusable_input(tmp_path: Path) -> None: + empty = tmp_path / "empty.dxf" + empty.write_text("") + with pytest.raises(MaskError, match="empty"): + load_mask_polylines(empty) + + no_entities = tmp_path / "no-entities.dxf" + no_entities.write_text(" 0\nSECTION\n 2\nHEADER\n 0\nENDSEC\n 0\nEOF\n") + with pytest.raises(MaskError, match="no model-space polylines"): + load_mask_polylines(no_entities) + with pytest.raises(MaskError, match="byte limit"): + load_mask_polylines(no_entities, max_bytes=8) + + +def test_block_definitions_are_opt_in_and_retain_their_name(tmp_path: Path) -> None: + source = tmp_path / "blocks.dxf" + source.write_text( + " 0\nSECTION\n 2\nBLOCKS\n" + " 0\nBLOCK\n 2\nFEATURE\n" + " 0\nLWPOLYLINE\n 8\nBLOCK-LAYER\n 70\n1\n" + " 10\n10\n 20\n20\n 10\n11\n 20\n20\n" + " 10\n11\n 20\n21\n 10\n10\n 20\n21\n" + " 0\nENDBLK\n 0\nENDSEC\n" + " 0\nSECTION\n 2\nENTITIES\n" + " 0\nLWPOLYLINE\n 8\nMODEL-LAYER\n 70\n1\n" + " 10\n0\n 20\n0\n 10\n1\n 20\n0\n" + " 10\n1\n 20\n1\n 10\n0\n 20\n1\n" + " 0\nENDSEC\n 0\nEOF\n" + ) + + model_space = load_mask_polylines(source) + assert len(model_space) == 1 + assert model_space[0].block is None + + with_blocks = load_mask_polylines(source, include_blocks=True) + assert len(with_blocks) == 2 + block = next(polyline for polyline in with_blocks if polyline.block is not None) + assert block.block == "FEATURE" + assert block.vertices[0] == (10.0, 20.0) diff --git a/python/tests/test_microfluidics.py b/python/tests/test_microfluidics.py new file mode 100644 index 0000000..3cb6a23 --- /dev/null +++ b/python/tests/test_microfluidics.py @@ -0,0 +1,118 @@ +# pyright: reportPrivateUsage=false + +from __future__ import annotations + +import math + +from cellmodeller2 import GridShape, SignalGridSpec, Vec3 +from cellmodeller2.microfluidics import BiopixelTrapDevice, TrapChannelDevice + + +def _grid() -> SignalGridSpec: + shape = GridShape() + shape.x, shape.y, shape.z = 64, 72, 4 + grid = SignalGridSpec() + grid.signal_count = 1 + grid.shape = shape + grid.origin = Vec3(-140.0, -144.0, -8.0) + grid.spacing = Vec3(4.0, 4.0, 4.0) + grid.diffusion = [40.0] + grid.advection = [Vec3()] + return grid + + +def test_device_grid_projection_is_engine_valid() -> None: + device = TrapChannelDevice(mean_flow_speed=20.0) + grid = _grid() + device.apply_to_grid(grid, inlet_values=[10.0], outlet_values=[0.0]) + grid.validate() + + assert grid.velocity_field is not None + assert any(value != 0.0 for value in grid.velocity_field.y_faces) + assert grid.y_lower.values == [10.0] + + solid = sum(grid.obstacles) + assert 0 < solid < len(grid.obstacles) + + +def test_device_flow_runs_through_the_channel_and_rests_in_the_trap() -> None: + device = TrapChannelDevice(mean_flow_speed=20.0) + grid = _grid() + device.apply_to_grid(grid, inlet_values=[10.0], outlet_values=[0.0]) + assert grid.velocity_field is not None + + def y_face(x: int, fy: int, z: int) -> float: + assert grid.velocity_field is not None + return grid.velocity_field.y_faces[ + x * (grid.shape.y + 1) * grid.shape.z + fy * grid.shape.z + z + ] + + center_x = (device.channel_far_x + device.trap_open_x) * 0.5 + channel_column = int((center_x - grid.origin.x) / grid.spacing.x + 0.5) + trap_column = int((0.0 - grid.origin.x) / grid.spacing.x + 0.5) + mid_face = grid.shape.y // 2 + fluid_z = int((0.0 - grid.origin.z) / grid.spacing.z + 0.5) + channel_speed = y_face(channel_column, mid_face, fluid_z) + trap_speed = abs(y_face(trap_column, mid_face, fluid_z)) + assert channel_speed > 15.0 + assert trap_speed < channel_speed * 0.05 + + half = (grid.spacing.x * 0.5, grid.spacing.y * 0.5, grid.spacing.z * 0.5) + assert device._solid(device.trap_back_x + grid.spacing.x, 0.0, 0.0, half) + assert not device._solid(device.trap_back_x, 0.0, 0.0, half) + assert not device._solid(center_x, 0.0, 0.0, half) + + +def test_biopixel_defaults_separate_reported_trap_from_model_channel() -> None: + device = BiopixelTrapDevice() + + assert device.trap_width == 100.0 + assert device.trap_depth == 85.0 + assert device.trap_height == 1.65 + assert device.channel_width == 100.0 + assert device.channel_height == 10.0 + assert device.wall_thickness == 10.0 + + +def test_wall_surfaces_stay_inside_the_fluid_mask() -> None: + cases = ( + ( + TrapChannelDevice(), + (4.0, 4.0, 4.0), + ( + (60.0, 0.0, 0.0), + (-60.0, 15.0, 0.0), + (0.0, -15.0, 0.0), + (-100.0, 0.0, 0.0), + (0.0, 0.0, 3.0), + (0.0, 0.0, -3.0), + ), + ), + ( + BiopixelTrapDevice(), + (5.0, 5.0, 1.65), + ( + (85.0, 0.0, 0.8), + (0.0, 50.0, 0.8), + (0.0, -50.0, 0.8), + (-100.0, 0.0, 5.0), + (50.0, 0.0, 1.65), + (-50.0, 0.0, 10.0), + ), + ), + ) + + for device, spacing, surfaces in cases: + half = (spacing[0] * 0.5, spacing[1] * 0.5, spacing[2] * 0.5) + for surface in surfaces: + assert not device._solid(surface[0], surface[1], surface[2], half) + + +def test_biopixel_cavity_is_shallow_beside_the_model_channel() -> None: + device = BiopixelTrapDevice() + half = (2.5, 2.5, 0.825) + + assert not device._solid(42.5, 0.0, 0.825, half) + assert device._solid(42.5, 0.0, 2.475, half) + assert not device._solid(-50.0, 0.0, 9.075, half) + assert math.isclose(device.trap_height / device.channel_height, 0.165) From 1c7a72568b2394778cc95027367cf98b38aa3251 Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Sat, 29 Aug 2026 09:21:42 -0600 Subject: [PATCH 4/4] Move flow solvers into native backends --- CMakeLists.txt | 32 +- README.md | 2 +- cpp/core/flow_system.hpp | 608 ++++++++++++++++++ cpp/core/simulation.cpp | 18 + cpp/cpu/cpu_backend.cpp | 16 +- cpp/cpu/cpu_flow.cpp | 270 ++++++++ cpp/cuda/cuda_backend.cu | 19 +- cpp/cuda/cuda_flow.cu | 388 +++++++++++ cpp/cuda/cuda_flow.cuh | 20 + cpp/cuda/kernels/flow.cu | 379 +++++++++++ cpp/cuda/kernels/flow.cuh | 56 ++ cpp/include/cm/backend.hpp | 7 + cpp/include/cm/flow.hpp | 72 +++ cpp/include/cm/simulation.hpp | 6 + cpp/include/cm/types.hpp | 2 + cpp/metal/kernels/flow.metal | 316 +++++++++ cpp/metal/metal_backend.mm | 19 +- cpp/metal/metal_flow.hpp | 31 + cpp/metal/metal_flow.mm | 579 +++++++++++++++++ cpp/python/bindings.cpp | 68 +- docs/architecture/0022-brinkman-flow.md | 40 +- docs/architecture/0023-mac-stokes.md | 20 +- docs/development/validation.md | 3 +- python/src/cellmodeller2/__init__.py | 14 + python/src/cellmodeller2/_core.pyi | 77 +++ python/src/cellmodeller2/flow.py | 213 ++---- python/src/cellmodeller2/microfluidics.py | 14 +- python/src/cellmodeller2/stokes.py | 358 ++--------- python/tests/test_flow.py | 39 +- python/tests/test_stokes.py | 47 +- scripts/run_flow_benchmarks.py | 76 ++- tests/conformance/README.md | 6 +- .../backend_contract_conformance_test.cpp | 2 + tests/conformance/flow_conformance_test.cpp | 124 ++++ tests/cpp/flow_test.cpp | 108 ++++ 35 files changed, 3467 insertions(+), 582 deletions(-) create mode 100644 cpp/core/flow_system.hpp create mode 100644 cpp/cpu/cpu_flow.cpp create mode 100644 cpp/cuda/cuda_flow.cu create mode 100644 cpp/cuda/cuda_flow.cuh create mode 100644 cpp/cuda/kernels/flow.cu create mode 100644 cpp/cuda/kernels/flow.cuh create mode 100644 cpp/include/cm/flow.hpp create mode 100644 cpp/metal/kernels/flow.metal create mode 100644 cpp/metal/metal_flow.hpp create mode 100644 cpp/metal/metal_flow.mm create mode 100644 tests/conformance/flow_conformance_test.cpp create mode 100644 tests/cpp/flow_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e96ad7f..0d6e6e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,13 +31,14 @@ add_library(cm_core STATIC cpp/cpu/cpu_contacts.cpp cpp/cpu/cpu_constraints.cpp cpp/cpu/cpu_coupled.cpp + cpp/cpu/cpu_flow.cpp cpp/cpu/cpu_mechanics.cpp cpp/cpu/cpu_species.cpp ) add_library(cm::core ALIAS cm_core) target_compile_features(cm_core PUBLIC cxx_std_23) -target_include_directories(cm_core PUBLIC cpp/include) +target_include_directories(cm_core PUBLIC cpp/include PRIVATE cpp) set_target_properties(cm_core PROPERTIES POSITION_INDEPENDENT_CODE ON) if(CM_ENABLE_METAL) @@ -51,6 +52,7 @@ if(CM_ENABLE_METAL) set(CM_GENERATED_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated") set(CM_METAL_GROWTH_HEADER "${CM_GENERATED_DIR}/cm/metal/growth_source.hpp") + set(CM_METAL_FLOW_HEADER "${CM_GENERATED_DIR}/cm/metal/flow_source.hpp") set(CM_METAL_CONTACTS_HEADER "${CM_GENERATED_DIR}/cm/metal/contacts_source.hpp") set(CM_METAL_COUPLED_RATES_HEADER "${CM_GENERATED_DIR}/cm/metal/coupled_rates_source.hpp") set(CM_METAL_MECHANICS_HEADER "${CM_GENERATED_DIR}/cm/metal/mechanics_source.hpp") @@ -68,6 +70,18 @@ if(CM_ENABLE_METAL) cmake/EmbedMetalSource.cmake VERBATIM ) + add_custom_command( + OUTPUT "${CM_METAL_FLOW_HEADER}" + COMMAND "${CMAKE_COMMAND}" + "-DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/cpp/metal/kernels/flow.metal" + "-DOUTPUT=${CM_METAL_FLOW_HEADER}" + "-DSYMBOL=flow_source" + -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/EmbedMetalSource.cmake" + DEPENDS + cpp/metal/kernels/flow.metal + cmake/EmbedMetalSource.cmake + VERBATIM + ) add_custom_command( OUTPUT "${CM_METAL_CONTACTS_HEADER}" COMMAND "${CMAKE_COMMAND}" @@ -131,9 +145,11 @@ if(CM_ENABLE_METAL) target_sources(cm_core PRIVATE cpp/metal/metal_backend.mm + cpp/metal/metal_flow.mm "${CM_METAL_CONTACTS_HEADER}" "${CM_METAL_COUPLED_RATES_HEADER}" "${CM_METAL_GROWTH_HEADER}" + "${CM_METAL_FLOW_HEADER}" "${CM_METAL_MECHANICS_HEADER}" "${CM_METAL_SIGNALS_HEADER}" "${CM_METAL_SPECIES_HEADER}" @@ -158,9 +174,11 @@ if(CM_ENABLE_CUDA) target_sources(cm_core PRIVATE cpp/cuda/cuda_backend.cu + cpp/cuda/cuda_flow.cu cpp/cuda/kernels/contacts.cu cpp/cuda/kernels/coupled_rates.cu cpp/cuda/kernels/growth.cu + cpp/cuda/kernels/flow.cu cpp/cuda/kernels/mechanics.cu cpp/cuda/kernels/signals.cu cpp/cuda/kernels/species.cu @@ -225,6 +243,10 @@ if(CM_BUILD_TESTS) target_link_libraries(cm_coupled_rates_test PRIVATE cm::core) add_test(NAME coupled_rates COMMAND cm_coupled_rates_test) + add_executable(cm_flow_test tests/cpp/flow_test.cpp) + target_link_libraries(cm_flow_test PRIVATE cm::core) + add_test(NAME flow COMMAND cm_flow_test) + add_executable(cm_checkpoint_test tests/cpp/checkpoint_test.cpp) target_link_libraries(cm_checkpoint_test PRIVATE cm::core) target_include_directories(cm_checkpoint_test PRIVATE tests/conformance) @@ -264,6 +286,14 @@ if(CM_BUILD_TESTS) add_test(NAME coupled_rates_conformance COMMAND cm_coupled_rates_conformance_test) set_tests_properties(coupled_rates_conformance PROPERTIES LABELS conformance) + add_executable(cm_flow_conformance_test + tests/conformance/flow_conformance_test.cpp + ) + target_link_libraries(cm_flow_conformance_test PRIVATE cm::core) + target_include_directories(cm_flow_conformance_test PRIVATE tests/conformance) + add_test(NAME flow_conformance COMMAND cm_flow_conformance_test) + set_tests_properties(flow_conformance PROPERTIES LABELS conformance) + add_executable(cm_lifecycle_conformance_test tests/conformance/lifecycle_conformance_test.cpp ) diff --git a/README.md b/README.md index 8567bee..a577de4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ CellModeller2 is an accelerator-native successor to [CellModeller](https://github.com/cellmodeller/CellModeller) for individual-based multicellular modeling. It combines a Python modeling interface with a C++23 engine and independent CPU, Apple Metal, and NVIDIA CUDA implementations. -Models can combine rod-shaped growth and division, lineage, contact mechanics and constraints, intracellular dynamics, and cell-grid signaling. Versioned checkpoints, batch manifests, data-only scenes, and Parquet/Zarr exports support reproducible research workflows. +Models can combine rod-shaped growth and division, lineage, contact mechanics and constraints, intracellular dynamics, cell-grid signaling, and steady fluid flow through voxelized microfluidic device geometries. Both depth-averaged Darcy-Brinkman flow and resolved staggered-grid Stokes-Brinkman flow execute through the selected CPU, Metal, or CUDA backend. Versioned checkpoints, batch manifests, data-only scenes, and Parquet/Zarr exports support reproducible research workflows. ## Backend status diff --git a/cpp/core/flow_system.hpp b/cpp/core/flow_system.hpp new file mode 100644 index 0000000..4aad3b3 --- /dev/null +++ b/cpp/core/flow_system.hpp @@ -0,0 +1,608 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cm/flow.hpp" + +namespace cm::detail { + +class FlowGridLayout { + public: + FlowGridLayout(const SignalGridSpec& spec, FlowAxis flow_axis) + : dimensions_{spec.shape.x, spec.shape.y, spec.shape.z}, + spacing_{spec.spacing.x, spec.spacing.y, spec.spacing.z}, + flow_axis_(static_cast(flow_axis)) { + face_counts_[0] = spec.x_face_count(); + face_counts_[1] = spec.y_face_count(); + face_counts_[2] = spec.z_face_count(); + face_offsets_[1] = face_counts_[0]; + face_offsets_[2] = face_counts_[0] + face_counts_[1]; + total_face_count_ = face_offsets_[2] + face_counts_[2]; + } + + [[nodiscard]] const std::array& dimensions() const noexcept { + return dimensions_; + } + + [[nodiscard]] const std::array& spacing() const noexcept { return spacing_; } + + [[nodiscard]] std::size_t flow_axis() const noexcept { return flow_axis_; } + + [[nodiscard]] std::size_t site_count() const noexcept { + return static_cast(dimensions_[0]) * dimensions_[1] * dimensions_[2]; + } + + [[nodiscard]] std::size_t face_count(std::size_t component) const noexcept { + return face_counts_[component]; + } + + [[nodiscard]] const std::array& face_counts() const noexcept { + return face_counts_; + } + + [[nodiscard]] const std::array& face_offsets() const noexcept { + return face_offsets_; + } + + [[nodiscard]] std::size_t total_face_count() const noexcept { return total_face_count_; } + + [[nodiscard]] std::size_t site_index(std::uint32_t x, std::uint32_t y, + std::uint32_t z) const noexcept { + return (static_cast(x) * dimensions_[1] + y) * dimensions_[2] + z; + } + + [[nodiscard]] std::array site_coordinates(std::size_t index) const noexcept { + const auto z = static_cast(index % dimensions_[2]); + index /= dimensions_[2]; + const auto y = static_cast(index % dimensions_[1]); + return {static_cast(index / dimensions_[1]), y, z}; + } + + [[nodiscard]] std::array face_dimensions(std::size_t component) const noexcept { + auto result = dimensions_; + ++result[component]; + return result; + } + + [[nodiscard]] std::size_t local_face_index(std::size_t component, std::uint32_t x, + std::uint32_t y, std::uint32_t z) const noexcept { + if (component == 0) { + return (static_cast(x) * dimensions_[1] + y) * dimensions_[2] + z; + } + if (component == 1) { + return (static_cast(x) * (dimensions_[1] + 1) + y) * dimensions_[2] + z; + } + return (static_cast(x) * dimensions_[1] + y) * (dimensions_[2] + 1) + z; + } + + [[nodiscard]] std::size_t face_index(std::size_t component, std::uint32_t x, std::uint32_t y, + std::uint32_t z) const noexcept { + return face_offsets_[component] + local_face_index(component, x, y, z); + } + + [[nodiscard]] std::pair> face_coordinates( + std::size_t index) const noexcept { + std::size_t component = 0; + while (component < 2 && index >= face_offsets_[component] + face_counts_[component]) { + ++component; + } + auto local = index - face_offsets_[component]; + const auto face_dims = face_dimensions(component); + const auto z = static_cast(local % face_dims[2]); + local /= face_dims[2]; + const auto y = static_cast(local % face_dims[1]); + return {component, {static_cast(local / face_dims[1]), y, z}}; + } + + [[nodiscard]] std::optional adjacent_site(std::size_t component, + const std::array& face, + int side) const noexcept { + auto site = face; + if (side < 0) { + if (face[component] == 0) { + return std::nullopt; + } + --site[component]; + } else if (face[component] >= dimensions_[component]) { + return std::nullopt; + } + return site_index(site[0], site[1], site[2]); + } + + [[nodiscard]] std::optional neighbor_site(std::size_t index, std::size_t axis, + int offset) const noexcept { + auto site = site_coordinates(index); + if (offset < 0) { + if (site[axis] == 0) { + return std::nullopt; + } + --site[axis]; + } else { + if (site[axis] + 1 >= dimensions_[axis]) { + return std::nullopt; + } + ++site[axis]; + } + return site_index(site[0], site[1], site[2]); + } + + [[nodiscard]] std::optional neighbor_face(std::size_t index, std::size_t axis, + int offset) const noexcept { + const auto [component, original] = face_coordinates(index); + auto face = original; + const auto face_dims = face_dimensions(component); + if (offset < 0) { + if (face[axis] == 0) { + return std::nullopt; + } + --face[axis]; + } else { + if (face[axis] + 1 >= face_dims[axis]) { + return std::nullopt; + } + ++face[axis]; + } + return face_index(component, face[0], face[1], face[2]); + } + + private: + std::array dimensions_{}; + std::array spacing_{}; + std::size_t flow_axis_{0}; + std::array face_counts_{}; + std::array face_offsets_{}; + std::size_t total_face_count_{0}; +}; + +[[nodiscard]] inline float harmonic_mean(float first, float second) noexcept { + const auto sum = first + second; + return sum > 0.0F ? 2.0F * first * second / sum : 0.0F; +} + +class DepthAveragedFlowSystem { + public: + DepthAveragedFlowSystem(const SignalGridSpec& spec, std::span mobility, + FlowAxis axis) + : spec_(spec), + layout_(spec, axis), + mobility_(layout_.site_count(), 1.0F), + diagonal_(layout_.site_count()), + right_hand_side_(layout_.site_count()) { + validate_flow_grid(spec_, axis); + if (!mobility.empty()) { + if (mobility.size() != layout_.site_count()) { + throw std::invalid_argument("flow mobility must hold one value per grid site"); + } + std::copy(mobility.begin(), mobility.end(), mobility_.begin()); + } + for (std::size_t site = 0; site < mobility_.size(); ++site) { + if (!std::isfinite(mobility_[site]) || mobility_[site] < 0.0F) { + throw std::invalid_argument("flow mobility values must be finite and non-negative"); + } + if (spec_.solid_site(site)) { + mobility_[site] = 0.0F; + } + } + + bool open_inlet = false; + for (std::size_t site = 0; site < layout_.site_count(); ++site) { + const auto value = mobility_[site]; + if (value == 0.0F) { + continue; + } + const auto coordinates = layout_.site_coordinates(site); + for (std::size_t component = 0; component < 3; ++component) { + const auto inverse_square = + 1.0F / (layout_.spacing()[component] * layout_.spacing()[component]); + for (const auto offset : {-1, 1}) { + const auto neighbor = layout_.neighbor_site(site, component, offset); + if (neighbor.has_value()) { + diagonal_[site] += harmonic_mean(value, mobility_[*neighbor]) * inverse_square; + } + } + } + if (coordinates[layout_.flow_axis()] == 0) { + const auto boundary = + 2.0F * value / + (layout_.spacing()[layout_.flow_axis()] * layout_.spacing()[layout_.flow_axis()]); + diagonal_[site] += boundary; + right_hand_side_[site] = boundary; + open_inlet = true; + } + if (coordinates[layout_.flow_axis()] + 1 == layout_.dimensions()[layout_.flow_axis()]) { + diagonal_[site] += + 2.0F * value / + (layout_.spacing()[layout_.flow_axis()] * layout_.spacing()[layout_.flow_axis()]); + } + } + if (!open_inlet) { + throw std::invalid_argument("the flow inlet boundary is entirely blocked"); + } + } + + [[nodiscard]] const FlowGridLayout& layout() const noexcept { return layout_; } + [[nodiscard]] const std::vector& mobility() const noexcept { return mobility_; } + [[nodiscard]] const std::vector& diagonal() const noexcept { return diagonal_; } + [[nodiscard]] const std::vector& right_hand_side() const noexcept { + return right_hand_side_; + } + + void apply(std::span input, std::vector& output) const { + if (input.size() != layout_.site_count()) { + throw std::invalid_argument("depth-averaged flow vector has the wrong size"); + } + output.assign(input.size(), 0.0); + for (std::size_t site = 0; site < input.size(); ++site) { + if (diagonal_[site] == 0.0F) { + continue; + } + auto result = static_cast(diagonal_[site]) * input[site]; + for (std::size_t component = 0; component < 3; ++component) { + const auto inverse_square = 1.0 / (static_cast(layout_.spacing()[component]) * + layout_.spacing()[component]); + for (const auto offset : {-1, 1}) { + const auto neighbor = layout_.neighbor_site(site, component, offset); + if (neighbor.has_value()) { + const auto conductance = + static_cast(harmonic_mean(mobility_[site], mobility_[*neighbor])) * + inverse_square; + result -= conductance * input[*neighbor]; + } + } + } + output[site] = result; + } + } + + [[nodiscard]] std::vector velocity(std::span pressure) const { + if (pressure.size() != layout_.site_count()) { + throw std::invalid_argument("depth-averaged pressure vector has the wrong size"); + } + std::vector result(layout_.total_face_count(), 0.0F); + for (std::size_t face_index = 0; face_index < result.size(); ++face_index) { + const auto [component, face] = layout_.face_coordinates(face_index); + const auto lower = layout_.adjacent_site(component, face, -1); + const auto upper = layout_.adjacent_site(component, face, 1); + const auto spacing = static_cast(layout_.spacing()[component]); + double value = 0.0; + if (lower.has_value() && upper.has_value()) { + const auto face_mobility = harmonic_mean(mobility_[*lower], mobility_[*upper]); + value = + -static_cast(face_mobility) * (pressure[*upper] - pressure[*lower]) / spacing; + } else if (component == layout_.flow_axis() && upper.has_value()) { + value = 2.0 * static_cast(mobility_[*upper]) * (1.0 - pressure[*upper]) / spacing; + } else if (component == layout_.flow_axis() && lower.has_value()) { + value = 2.0 * static_cast(mobility_[*lower]) * pressure[*lower] / spacing; + } + result[face_index] = static_cast(value); + } + return result; + } + + [[nodiscard]] std::vector open_inlet_faces() const { + std::vector result(layout_.total_face_count(), 0); + const auto component = layout_.flow_axis(); + const auto face_dims = layout_.face_dimensions(component); + for (std::uint32_t x = 0; x < face_dims[0]; ++x) { + for (std::uint32_t y = 0; y < face_dims[1]; ++y) { + for (std::uint32_t z = 0; z < face_dims[2]; ++z) { + std::array face{x, y, z}; + if (face[component] != 0) { + continue; + } + const auto upper = layout_.adjacent_site(component, face, 1); + if (upper.has_value() && mobility_[*upper] > 0.0F) { + result[layout_.face_index(component, x, y, z)] = 1; + } + } + } + } + return result; + } + + private: + const SignalGridSpec& spec_; + FlowGridLayout layout_; + std::vector mobility_; + std::vector diagonal_; + std::vector right_hand_side_; +}; + +class ResolvedFlowSystem { + public: + ResolvedFlowSystem(const SignalGridSpec& spec, std::span drag, FlowAxis axis) + : spec_(spec), + layout_(spec, axis), + fluid_(layout_.site_count(), 1), + drag_(layout_.site_count()), + active_(layout_.total_face_count()), + exists_(layout_.total_face_count()), + face_drag_(layout_.total_face_count()), + diagonal_(layout_.total_face_count()), + force_(layout_.total_face_count()) { + validate_flow_grid(spec_, axis); + if (!drag.empty()) { + if (drag.size() != layout_.site_count()) { + throw std::invalid_argument("resolved-flow drag must hold one value per grid site"); + } + std::copy(drag.begin(), drag.end(), drag_.begin()); + } + for (std::size_t site = 0; site < layout_.site_count(); ++site) { + fluid_[site] = spec_.solid_site(site) ? 0 : 1; + if (!std::isfinite(drag_[site]) || drag_[site] < 0.0F) { + throw std::invalid_argument("resolved-flow drag values must be finite and non-negative"); + } + if (fluid_[site] == 0) { + drag_[site] = 0.0F; + } + } + + bool open_inlet = false; + for (std::size_t index = 0; index < layout_.total_face_count(); ++index) { + const auto [component, face] = layout_.face_coordinates(index); + const auto lower = layout_.adjacent_site(component, face, -1); + const auto upper = layout_.adjacent_site(component, face, 1); + const auto lower_fluid = lower.has_value() && fluid_[*lower] != 0; + const auto upper_fluid = upper.has_value() && fluid_[*upper] != 0; + exists_[index] = lower_fluid || upper_fluid ? 1 : 0; + auto active = lower_fluid && upper_fluid; + if (component == layout_.flow_axis() && + (face[component] == 0 || face[component] == layout_.dimensions()[component])) { + active = lower_fluid || upper_fluid; + } + active_[index] = active ? 1 : 0; + if (!active) { + continue; + } + float sum = 0.0F; + float count = 0.0F; + if (lower_fluid) { + sum += drag_[*lower]; + count += 1.0F; + } + if (upper_fluid) { + sum += drag_[*upper]; + count += 1.0F; + } + face_drag_[index] = sum / count; + diagonal_[index] = face_drag_[index]; + for (std::size_t axis_index = 0; axis_index < 3; ++axis_index) { + if (layout_.dimensions()[axis_index] > 1) { + diagonal_[index] += + 2.0F / (layout_.spacing()[axis_index] * layout_.spacing()[axis_index]); + } + } + if (component == layout_.flow_axis() && face[component] == 0) { + force_[index] = 1.0F / layout_.spacing()[component]; + open_inlet = true; + } + } + if (!open_inlet) { + throw std::invalid_argument("the resolved-flow inlet boundary is entirely blocked"); + } + } + + [[nodiscard]] const FlowGridLayout& layout() const noexcept { return layout_; } + [[nodiscard]] const std::vector& fluid() const noexcept { return fluid_; } + [[nodiscard]] const std::vector& active() const noexcept { return active_; } + [[nodiscard]] const std::vector& exists() const noexcept { return exists_; } + [[nodiscard]] const std::vector& face_drag() const noexcept { return face_drag_; } + [[nodiscard]] const std::vector& diagonal() const noexcept { return diagonal_; } + [[nodiscard]] const std::vector& force() const noexcept { return force_; } + + void apply_momentum(std::span input, std::vector& output) const { + if (input.size() != layout_.total_face_count()) { + throw std::invalid_argument("resolved-flow face vector has the wrong size"); + } + output.assign(input.size(), 0.0); + for (std::size_t index = 0; index < input.size(); ++index) { + if (active_[index] == 0) { + continue; + } + const auto component = layout_.face_coordinates(index).first; + auto result = static_cast(face_drag_[index]) * input[index]; + for (std::size_t axis = 0; axis < 3; ++axis) { + if (layout_.dimensions()[axis] == 1) { + continue; + } + const auto inverse_square = + 1.0 / (static_cast(layout_.spacing()[axis]) * layout_.spacing()[axis]); + for (const auto offset : {-1, 1}) { + const auto neighbor_index = layout_.neighbor_face(index, axis, offset); + auto neighbor = 0.0; + if (axis == component) { + neighbor = neighbor_index.has_value() ? input[*neighbor_index] : input[index]; + } else if (neighbor_index.has_value() && exists_[*neighbor_index] != 0) { + neighbor = input[*neighbor_index]; + } else { + neighbor = -input[index]; + } + result -= (neighbor - input[index]) * inverse_square; + } + } + output[index] = result; + } + } + + [[nodiscard]] std::vector gradient(std::span pressure) const { + if (pressure.size() != layout_.site_count()) { + throw std::invalid_argument("resolved-flow pressure vector has the wrong size"); + } + std::vector result(layout_.total_face_count(), 0.0); + for (std::size_t index = 0; index < result.size(); ++index) { + if (active_[index] == 0) { + continue; + } + const auto [component, face] = layout_.face_coordinates(index); + const auto lower = layout_.adjacent_site(component, face, -1); + const auto upper = layout_.adjacent_site(component, face, 1); + const auto lower_value = lower.has_value() && fluid_[*lower] != 0 ? pressure[*lower] : 0.0; + const auto upper_value = upper.has_value() && fluid_[*upper] != 0 ? pressure[*upper] : 0.0; + result[index] = (upper_value - lower_value) / layout_.spacing()[component]; + } + return result; + } + + [[nodiscard]] std::vector divergence(std::span velocity) const { + if (velocity.size() != layout_.total_face_count()) { + throw std::invalid_argument("resolved-flow velocity vector has the wrong size"); + } + std::vector result(layout_.site_count(), 0.0); + for (std::size_t site = 0; site < result.size(); ++site) { + if (fluid_[site] == 0) { + continue; + } + const auto coordinates = layout_.site_coordinates(site); + for (std::size_t component = 0; component < 3; ++component) { + auto upper = coordinates; + ++upper[component]; + const auto upper_face = layout_.face_index(component, upper[0], upper[1], upper[2]); + const auto lower_face = + layout_.face_index(component, coordinates[0], coordinates[1], coordinates[2]); + result[site] += + (velocity[upper_face] - velocity[lower_face]) / layout_.spacing()[component]; + } + } + return result; + } + + [[nodiscard]] std::vector pressure_diagonal() const { + std::vector result(layout_.site_count(), 0.0F); + for (std::size_t site = 0; site < result.size(); ++site) { + if (fluid_[site] != 0) { + result[site] = 1.0F; + } + } + return result; + } + + [[nodiscard]] std::vector open_inlet_faces() const { + std::vector result(layout_.total_face_count(), 0); + const auto component = layout_.flow_axis(); + for (std::size_t index = layout_.face_offsets()[component]; + index < layout_.face_offsets()[component] + layout_.face_counts()[component]; ++index) { + const auto [face_component, face] = layout_.face_coordinates(index); + if (face_component == component && face[component] == 0 && active_[index] != 0) { + result[index] = 1; + } + } + return result; + } + + [[nodiscard]] std::uint32_t minimum_gap_voxels() const { + std::uint32_t shortest = 0; + for (std::size_t axis = 0; axis < 3; ++axis) { + if (axis == layout_.flow_axis() || layout_.dimensions()[axis] <= 1) { + continue; + } + const auto first_axis = (axis + 1) % 3; + const auto second_axis = (axis + 2) % 3; + for (std::uint32_t first = 0; first < layout_.dimensions()[first_axis]; ++first) { + for (std::uint32_t second = 0; second < layout_.dimensions()[second_axis]; ++second) { + std::uint32_t run = 0; + for (std::uint32_t along = 0; along < layout_.dimensions()[axis]; ++along) { + std::array coordinates{}; + coordinates[axis] = along; + coordinates[first_axis] = first; + coordinates[second_axis] = second; + if (fluid_[layout_.site_index(coordinates[0], coordinates[1], coordinates[2])] != 0) { + ++run; + } else if (run != 0) { + shortest = shortest == 0 ? run : std::min(shortest, run); + run = 0; + } + } + if (run != 0) { + shortest = shortest == 0 ? run : std::min(shortest, run); + } + } + } + } + return shortest; + } + + private: + const SignalGridSpec& spec_; + FlowGridLayout layout_; + std::vector fluid_; + std::vector drag_; + std::vector active_; + std::vector exists_; + std::vector face_drag_; + std::vector diagonal_; + std::vector force_; +}; + +struct ScaledVelocity { + SignalGridVelocityField field; + float solved_mean{0.0F}; + float max_speed{0.0F}; + float factor{1.0F}; +}; + +[[nodiscard]] inline ScaledVelocity scale_velocity(const SignalGridSpec& spec, + const FlowGridLayout& layout, + std::span values, + std::span open_inlet, + float requested_mean) { + if (values.size() != layout.total_face_count() || open_inlet.size() != values.size()) { + throw std::invalid_argument("flow velocity scaling arrays have inconsistent sizes"); + } + double inlet_sum = 0.0; + std::size_t inlet_count = 0; + float peak = 0.0F; + for (std::size_t index = 0; index < values.size(); ++index) { + if (!std::isfinite(values[index])) { + throw std::runtime_error("flow solve produced a non-finite velocity"); + } + peak = std::max(peak, std::abs(values[index])); + if (open_inlet[index] != 0) { + inlet_sum += values[index]; + ++inlet_count; + } + } + if (inlet_count == 0) { + throw std::logic_error("flow solve has no open inlet faces"); + } + const auto solved_mean = static_cast(inlet_sum / static_cast(inlet_count)); + if (peak == 0.0F || solved_mean <= 1.0e-9F * peak) { + throw std::runtime_error("the device carries no through-flow: the outlet is unreachable"); + } + const auto factor = requested_mean / solved_mean; + std::vector scaled(values.size()); + float scaled_peak = 0.0F; + for (std::size_t index = 0; index < values.size(); ++index) { + scaled[index] = values[index] * factor; + scaled_peak = std::max(scaled_peak, std::abs(scaled[index])); + } + + SignalGridVelocityField field; + const auto offsets = layout.face_offsets(); + const auto counts = layout.face_counts(); + field.x_faces.assign(scaled.begin(), scaled.begin() + static_cast(counts[0])); + field.y_faces.assign(scaled.begin() + static_cast(offsets[1]), + scaled.begin() + static_cast(offsets[1] + counts[1])); + field.z_faces.assign(scaled.begin() + static_cast(offsets[2]), scaled.end()); + auto candidate = spec; + candidate.velocity_field = field; + for (auto& advection : candidate.advection) { + advection = {}; + } + candidate.validate(); + return {.field = std::move(field), + .solved_mean = solved_mean, + .max_speed = scaled_peak, + .factor = factor}; +} + +} // namespace cm::detail diff --git a/cpp/core/simulation.cpp b/cpp/core/simulation.cpp index 589e790..e96f76e 100644 --- a/cpp/core/simulation.cpp +++ b/cpp/core/simulation.cpp @@ -265,6 +265,24 @@ MechanicsSolveResult Simulation::relax_cell_mechanics( return result; } +DepthAveragedFlowResult Simulation::solve_depth_averaged_flow( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters) { + if (!backend_->supports(BackendFeature::depth_averaged_flow)) { + throw std::runtime_error("selected backend does not implement depth-averaged flow"); + } + return backend_->solve_depth_averaged_flow(spec, mobility, parameters); +} + +ResolvedFlowResult Simulation::solve_resolved_flow(const SignalGridSpec& spec, + std::span drag, + const ResolvedFlowParameters& parameters) { + if (!backend_->supports(BackendFeature::resolved_flow)) { + throw std::runtime_error("selected backend does not implement resolved flow"); + } + return backend_->solve_resolved_flow(spec, drag, parameters); +} + CellSnapshot Simulation::cell(CellId id) const { return state_.cell(id); } std::vector Simulation::cells() const { return state_.cells(); } diff --git a/cpp/cpu/cpu_backend.cpp b/cpp/cpu/cpu_backend.cpp index a0ea93c..58daca1 100644 --- a/cpp/cpu/cpu_backend.cpp +++ b/cpp/cpu/cpu_backend.cpp @@ -22,7 +22,9 @@ class CpuBackend final : public ComputeBackend { return feature == BackendFeature::growth || feature == BackendFeature::species || feature == BackendFeature::cell_contacts || feature == BackendFeature::cell_mechanics || feature == BackendFeature::external_constraints || feature == BackendFeature::signals || - feature == BackendFeature::coupled_rates; + feature == BackendFeature::coupled_rates || + feature == BackendFeature::depth_averaged_flow || + feature == BackendFeature::resolved_flow; } void advance_growth(WorldState& state, float dt) override { state.advance_growth(dt); } @@ -59,6 +61,18 @@ class CpuBackend final : public ComputeBackend { const MechanicsParameters& parameters) override { return solve_cell_mechanics_cpu(state, contacts, external_contacts, parameters); } + + [[nodiscard]] DepthAveragedFlowResult solve_depth_averaged_flow( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters) override { + return solve_depth_averaged_flow_cpu(spec, mobility, parameters); + } + + [[nodiscard]] ResolvedFlowResult solve_resolved_flow( + const SignalGridSpec& spec, std::span drag, + const ResolvedFlowParameters& parameters) override { + return solve_resolved_flow_cpu(spec, drag, parameters); + } }; } // namespace diff --git a/cpp/cpu/cpu_flow.cpp b/cpp/cpu/cpu_flow.cpp new file mode 100644 index 0000000..a36367b --- /dev/null +++ b/cpp/cpu/cpu_flow.cpp @@ -0,0 +1,270 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cm/flow.hpp" +#include "core/flow_system.hpp" + +namespace cm { +namespace { + +struct ConjugateGradientResult { + std::vector solution; + std::uint32_t iterations{0}; + double relative_residual{0.0}; +}; + +[[nodiscard]] double dot_product(std::span left, std::span right) { + if (left.size() != right.size()) { + throw std::logic_error("conjugate-gradient vectors have inconsistent sizes"); + } + double result = 0.0; + for (std::size_t index = 0; index < left.size(); ++index) { + result += left[index] * right[index]; + } + return result; +} + +template +[[nodiscard]] ConjugateGradientResult conjugate_gradient( + Apply&& apply, std::span right_hand_side, std::span diagonal, + float tolerance, std::uint32_t max_iterations, const char* label) { + if (right_hand_side.size() != diagonal.size()) { + throw std::logic_error(std::string(label) + " arrays have inconsistent sizes"); + } + std::vector solution(right_hand_side.size(), 0.0); + std::vector residual(right_hand_side.begin(), right_hand_side.end()); + const auto rhs_norm_squared = dot_product(residual, residual); + if (rhs_norm_squared == 0.0) { + return {.solution = std::move(solution)}; + } + const auto rhs_norm = std::sqrt(rhs_norm_squared); + std::vector preconditioned(residual.size(), 0.0); + for (std::size_t index = 0; index < residual.size(); ++index) { + if (diagonal[index] > 0.0F) { + preconditioned[index] = residual[index] / diagonal[index]; + } + } + auto direction = preconditioned; + auto rho = dot_product(residual, preconditioned); + auto relative = 1.0; + for (std::uint32_t iteration = 1; iteration <= max_iterations; ++iteration) { + std::vector transformed; + apply(direction, transformed); + const auto curvature = dot_product(direction, transformed); + if (!std::isfinite(curvature) || curvature <= 0.0) { + throw std::runtime_error(std::string(label) + + " conjugate gradient encountered non-positive curvature"); + } + const auto alpha = rho / curvature; + for (std::size_t index = 0; index < solution.size(); ++index) { + solution[index] += alpha * direction[index]; + residual[index] -= alpha * transformed[index]; + } + relative = std::sqrt(dot_product(residual, residual)) / rhs_norm; + if (!std::isfinite(relative)) { + throw std::runtime_error(std::string(label) + + " conjugate gradient produced a non-finite residual"); + } + if (relative <= tolerance) { + return { + .solution = std::move(solution), .iterations = iteration, .relative_residual = relative}; + } + for (std::size_t index = 0; index < residual.size(); ++index) { + preconditioned[index] = diagonal[index] > 0.0F ? residual[index] / diagonal[index] : 0.0; + } + const auto next_rho = dot_product(residual, preconditioned); + if (!std::isfinite(next_rho) || rho == 0.0) { + throw std::runtime_error(std::string(label) + + " conjugate gradient encountered a preconditioner breakdown"); + } + const auto beta = next_rho / rho; + for (std::size_t index = 0; index < direction.size(); ++index) { + direction[index] = preconditioned[index] + beta * direction[index]; + } + rho = next_rho; + } + throw std::runtime_error(std::string(label) + " conjugate gradient did not converge: relative " + + std::to_string(relative)); +} + +[[nodiscard]] std::array axis_boundaries(const SignalGridSpec& spec, + std::size_t axis) { + const std::array, 3> boundaries{{ + {&spec.x_lower, &spec.x_upper}, + {&spec.y_lower, &spec.y_upper}, + {&spec.z_lower, &spec.z_upper}, + }}; + return boundaries[axis]; +} + +void validate_relative_tolerance(float tolerance, const char* name) { + if (!std::isfinite(tolerance) || tolerance < std::numeric_limits::epsilon()) { + throw std::invalid_argument(std::string(name) + + " must be finite and at least float machine epsilon"); + } +} + +} // namespace + +void DepthAveragedFlowParameters::validate() const { + if (!std::isfinite(mean_inlet_speed) || mean_inlet_speed == 0.0F) { + throw std::invalid_argument("depth-averaged mean inlet speed must be finite and nonzero"); + } + validate_relative_tolerance(relative_tolerance, "depth-averaged relative tolerance"); + if (max_iterations == 0) { + throw std::invalid_argument("depth-averaged iteration limit must be positive"); + } + switch (axis) { + case FlowAxis::x: + case FlowAxis::y: + case FlowAxis::z: + return; + } + throw std::invalid_argument("unknown depth-averaged flow axis"); +} + +void ResolvedFlowParameters::validate() const { + if (!std::isfinite(mean_inlet_speed) || mean_inlet_speed == 0.0F) { + throw std::invalid_argument("resolved-flow mean inlet speed must be finite and nonzero"); + } + validate_relative_tolerance(relative_tolerance, "resolved-flow outer relative tolerance"); + validate_relative_tolerance(inner_relative_tolerance, "resolved-flow inner relative tolerance"); + if (max_outer_iterations == 0 || max_inner_iterations == 0) { + throw std::invalid_argument("resolved-flow iteration limits must be positive"); + } + switch (axis) { + case FlowAxis::x: + case FlowAxis::y: + case FlowAxis::z: + return; + } + throw std::invalid_argument("unknown resolved-flow axis"); +} + +void validate_flow_grid(const SignalGridSpec& spec, FlowAxis axis) { + spec.validate(); + const auto axis_index = static_cast(axis); + if (axis_index >= 3) { + throw std::invalid_argument("unknown flow axis"); + } + for (std::size_t candidate = 0; candidate < 3; ++candidate) { + const auto boundaries = axis_boundaries(spec, candidate); + if (boundaries[0]->kind == GridBoundaryKind::periodic || + boundaries[1]->kind == GridBoundaryKind::periodic) { + throw std::invalid_argument("native flow solvers do not support periodic boundaries"); + } + } + for (const auto* boundary : axis_boundaries(spec, axis_index)) { + if (boundary->kind != GridBoundaryKind::fixed) { + throw std::invalid_argument( + "native flow-axis boundaries must be fixed to identify inlet and outlet"); + } + } +} + +DepthAveragedFlowResult solve_depth_averaged_flow_cpu( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters) { + parameters.validate(); + const detail::DepthAveragedFlowSystem system(spec, mobility, parameters.axis); + const auto solve = + conjugate_gradient([&](std::span input, + std::vector& output) { system.apply(input, output); }, + system.right_hand_side(), system.diagonal(), parameters.relative_tolerance, + parameters.max_iterations, "depth-averaged flow"); + const auto unscaled = system.velocity(solve.solution); + const auto scaled = detail::scale_velocity( + spec, system.layout(), unscaled, system.open_inlet_faces(), parameters.mean_inlet_speed); + return { + .field = scaled.field, + .report = {.iterations = solve.iterations, + .relative_residual = static_cast(solve.relative_residual), + .mean_inlet_speed = parameters.mean_inlet_speed, + .max_speed = scaled.max_speed}, + }; +} + +ResolvedFlowResult solve_resolved_flow_cpu(const SignalGridSpec& spec, std::span drag, + const ResolvedFlowParameters& parameters) { + parameters.validate(); + const detail::ResolvedFlowSystem system(spec, drag, parameters.axis); + std::uint64_t inner_iterations = 0; + const auto solve_momentum = [&](std::span right_hand_side) { + std::vector rhs(right_hand_side.size()); + std::transform(right_hand_side.begin(), right_hand_side.end(), rhs.begin(), + [](double value) { return static_cast(value); }); + const auto result = conjugate_gradient( + [&](std::span input, std::vector& output) { + system.apply_momentum(input, output); + }, + rhs, system.diagonal(), parameters.inner_relative_tolerance, + parameters.max_inner_iterations, "resolved-flow momentum"); + if (inner_iterations > std::numeric_limits::max() - result.iterations) { + throw std::overflow_error("resolved-flow inner iteration count overflow"); + } + inner_iterations += result.iterations; + return result.solution; + }; + + std::vector force(system.force().begin(), system.force().end()); + const auto particular = solve_momentum(force); + auto schur_rhs_double = system.divergence(particular); + std::vector schur_rhs(schur_rhs_double.size()); + for (std::size_t index = 0; index < schur_rhs.size(); ++index) { + schur_rhs[index] = static_cast(-schur_rhs_double[index]); + } + const auto pressure_diagonal = system.pressure_diagonal(); + const auto pressure = conjugate_gradient( + [&](std::span input, std::vector& output) { + const auto gradient = system.gradient(input); + const auto response = solve_momentum(gradient); + output = system.divergence(response); + for (auto& value : output) { + value = -value; + } + }, + schur_rhs, pressure_diagonal, parameters.relative_tolerance, parameters.max_outer_iterations, + "resolved-flow pressure"); + + const auto correction = solve_momentum(system.gradient(pressure.solution)); + std::vector velocity(particular.size()); + for (std::size_t index = 0; index < velocity.size(); ++index) { + velocity[index] = particular[index] - correction[index]; + } + const auto divergence = system.divergence(velocity); + double divergence_square_sum = 0.0; + std::size_t fluid_count = 0; + for (std::size_t site = 0; site < divergence.size(); ++site) { + if (system.fluid()[site] != 0) { + divergence_square_sum += divergence[site] * divergence[site]; + ++fluid_count; + } + } + const auto divergence_rms = + fluid_count == 0 ? 0.0 : std::sqrt(divergence_square_sum / static_cast(fluid_count)); + std::vector unscaled(velocity.size()); + std::transform(velocity.begin(), velocity.end(), unscaled.begin(), + [](double value) { return static_cast(value); }); + const auto scaled = detail::scale_velocity( + spec, system.layout(), unscaled, system.open_inlet_faces(), parameters.mean_inlet_speed); + return { + .field = scaled.field, + .report = {.outer_iterations = pressure.iterations, + .inner_iterations = inner_iterations, + .divergence_rms = static_cast(divergence_rms * std::abs(scaled.factor)), + .mean_inlet_speed = parameters.mean_inlet_speed, + .max_speed = scaled.max_speed, + .min_gap_voxels = system.minimum_gap_voxels()}, + }; +} + +} // namespace cm diff --git a/cpp/cuda/cuda_backend.cu b/cpp/cuda/cuda_backend.cu index 94285b9..ab14c98 100644 --- a/cpp/cuda/cuda_backend.cu +++ b/cpp/cuda/cuda_backend.cu @@ -16,6 +16,7 @@ #include #include "cm/backend.hpp" +#include "cuda_flow.cuh" #include "kernels/contacts.cuh" #include "kernels/coupled_rates.cuh" #include "kernels/growth.cuh" @@ -120,7 +121,9 @@ class CudaBackend final : public ComputeBackend { feature == BackendFeature::cell_contacts || feature == BackendFeature::external_constraints || feature == BackendFeature::cell_mechanics || feature == BackendFeature::signals || - feature == BackendFeature::coupled_rates; + feature == BackendFeature::coupled_rates || + feature == BackendFeature::depth_averaged_flow || + feature == BackendFeature::resolved_flow; } void advance_growth(WorldState& state, float dt) override { @@ -832,6 +835,20 @@ class CudaBackend final : public ComputeBackend { return result; } + [[nodiscard]] DepthAveragedFlowResult solve_depth_averaged_flow( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters) override { + activate_device(); + return cuda::solve_depth_averaged_flow(spec, mobility, parameters, stream_); + } + + [[nodiscard]] ResolvedFlowResult solve_resolved_flow( + const SignalGridSpec& spec, std::span drag, + const ResolvedFlowParameters& parameters) override { + activate_device(); + return cuda::solve_resolved_flow(spec, drag, parameters, stream_); + } + private: void activate_device() { check_cuda(cudaSetDevice(device_index_), "failed to activate the CUDA device"); diff --git a/cpp/cuda/cuda_flow.cu b/cpp/cuda/cuda_flow.cu new file mode 100644 index 0000000..fb89086 --- /dev/null +++ b/cpp/cuda/cuda_flow.cu @@ -0,0 +1,388 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/flow_system.hpp" +#include "cuda_flow.cuh" +#include "kernels/flow.cuh" + +namespace cm::cuda { +namespace { + +void check_cuda(cudaError_t result, const char* operation) { + if (result != cudaSuccess) { + throw std::runtime_error(std::string(operation) + ": " + cudaGetErrorString(result)); + } +} + +void check_launch(const char* operation) { check_cuda(cudaGetLastError(), operation); } + +std::uint32_t checked_count(std::size_t count, const char* description) { + if (count == 0 || count > std::numeric_limits::max()) { + throw std::overflow_error(std::string("CUDA flow ") + description + + " must fit the nonzero uint32 index space"); + } + return static_cast(count); +} + +FlowGridParameters make_grid_parameters(const detail::FlowGridLayout& layout) { + const auto site_count = checked_count(layout.site_count(), "site count"); + const auto face_count = checked_count(layout.total_face_count(), "face count"); + const auto offsets = layout.face_offsets(); + const auto counts = layout.face_counts(); + for (const auto value : offsets) { + static_cast(checked_count(value == 0 ? 1 : value, "face offset")); + } + for (const auto value : counts) { + static_cast(checked_count(value, "component face count")); + } + const auto spacing = layout.spacing(); + FlowGridParameters result{}; + for (std::size_t axis = 0; axis < 3; ++axis) { + result.dimensions[axis] = layout.dimensions()[axis]; + result.spacing[axis] = spacing[axis]; + result.inverse_spacing_squared[axis] = 1.0F / (spacing[axis] * spacing[axis]); + result.face_offsets[axis] = static_cast(offsets[axis]); + result.face_counts[axis] = static_cast(counts[axis]); + } + result.face_offsets[3] = face_count; + result.face_counts[3] = face_count; + result.flow_axis = static_cast(layout.flow_axis()); + result.site_count = site_count; + result.total_face_count = face_count; + return result; +} + +template +class DeviceBuffer { + public: + DeviceBuffer(std::size_t count, const char* description) : count_(count) { + if (count == 0 || count > std::numeric_limits::max() / sizeof(T)) { + throw std::overflow_error(std::string("invalid CUDA flow buffer size for ") + description); + } + check_cuda(cudaMalloc(reinterpret_cast(&data_), count * sizeof(T)), description); + } + + DeviceBuffer(const DeviceBuffer&) = delete; + DeviceBuffer& operator=(const DeviceBuffer&) = delete; + + ~DeviceBuffer() { + if (data_ != nullptr) { + static_cast(cudaFree(data_)); + } + } + + [[nodiscard]] T* data() noexcept { return data_; } + [[nodiscard]] const T* data() const noexcept { return data_; } + [[nodiscard]] std::size_t count() const noexcept { return count_; } + + private: + T* data_{nullptr}; + std::size_t count_{0}; +}; + +template +void upload(DeviceBuffer& destination, std::span source, cudaStream_t stream, + const char* operation) { + if (destination.count() != source.size()) { + throw std::logic_error(std::string(operation) + ": buffer size mismatch"); + } + check_cuda(cudaMemcpyAsync(destination.data(), source.data(), source.size_bytes(), + cudaMemcpyHostToDevice, stream), + operation); +} + +template +std::vector download(const DeviceBuffer& source, cudaStream_t stream, const char* operation) { + std::vector result(source.count()); + check_cuda(cudaMemcpyAsync(result.data(), source.data(), result.size() * sizeof(T), + cudaMemcpyDeviceToHost, stream), + operation); + check_cuda(cudaStreamSynchronize(stream), operation); + return result; +} + +struct PcgWorkspace { + explicit PcgWorkspace(std::uint32_t count) + : residual(count, "failed to allocate CUDA flow residual"), + preconditioned(count, "failed to allocate CUDA flow preconditioned residual"), + direction(count, "failed to allocate CUDA flow direction"), + transformed(count, "failed to allocate CUDA flow transformed vector"), + partials((count + flow_reduction_width - 1) / flow_reduction_width, + "failed to allocate CUDA flow reduction partials"), + host_partials(partials.count()) {} + + DeviceBuffer residual; + DeviceBuffer preconditioned; + DeviceBuffer direction; + DeviceBuffer transformed; + DeviceBuffer partials; + std::vector host_partials; +}; + +struct PcgReport { + std::uint32_t iterations{0}; + float relative_residual{0.0F}; +}; + +double dot(const float* left, const float* right, std::uint32_t count, PcgWorkspace& workspace, + cudaStream_t stream) { + launch_flow_dot_partial(left, right, workspace.partials.data(), count, stream); + check_launch("failed to launch CUDA flow reduction"); + check_cuda(cudaMemcpyAsync(workspace.host_partials.data(), workspace.partials.data(), + workspace.host_partials.size() * sizeof(float), cudaMemcpyDeviceToHost, + stream), + "failed to download CUDA flow reduction"); + check_cuda(cudaStreamSynchronize(stream), "CUDA flow reduction failed"); + double result = 0.0; + for (const auto value : workspace.host_partials) { + result += value; + } + return result; +} + +template +PcgReport solve_pcg(const float* right_hand_side, const float* diagonal, float* solution, + PcgWorkspace& workspace, std::uint32_t count, float tolerance, + std::uint32_t max_iterations, const char* label, cudaStream_t stream, + Apply&& apply) { + launch_flow_pcg_initialize(right_hand_side, diagonal, solution, workspace.residual.data(), + workspace.preconditioned.data(), workspace.direction.data(), count, + stream); + check_launch("failed to launch CUDA flow PCG initialization"); + const auto rhs_norm_squared = dot(right_hand_side, right_hand_side, count, workspace, stream); + if (rhs_norm_squared == 0.0) { + return {}; + } + const auto rhs_norm = std::sqrt(rhs_norm_squared); + auto rho = + dot(workspace.residual.data(), workspace.preconditioned.data(), count, workspace, stream); + auto relative = 1.0; + for (std::uint32_t iteration = 1; iteration <= max_iterations; ++iteration) { + apply(workspace.direction.data(), workspace.transformed.data()); + const auto curvature = + dot(workspace.direction.data(), workspace.transformed.data(), count, workspace, stream); + if (!std::isfinite(curvature) || curvature <= 0.0) { + throw std::runtime_error(std::string(label) + + " conjugate gradient encountered non-positive curvature"); + } + const auto alpha_double = rho / curvature; + if (!std::isfinite(alpha_double) || + std::abs(alpha_double) > std::numeric_limits::max()) { + throw std::runtime_error(std::string(label) + + " conjugate gradient produced a non-finite step"); + } + const auto alpha = static_cast(alpha_double); + launch_flow_pcg_update(solution, workspace.residual.data(), workspace.direction.data(), + workspace.transformed.data(), alpha, count, stream); + check_launch("failed to launch CUDA flow PCG update"); + const auto residual_squared = + dot(workspace.residual.data(), workspace.residual.data(), count, workspace, stream); + relative = std::sqrt(std::max(0.0, residual_squared)) / rhs_norm; + if (!std::isfinite(relative)) { + throw std::runtime_error(std::string(label) + + " conjugate gradient produced a non-finite residual"); + } + if (relative <= tolerance) { + return {.iterations = iteration, .relative_residual = static_cast(relative)}; + } + launch_flow_pcg_precondition(workspace.residual.data(), diagonal, + workspace.preconditioned.data(), count, stream); + check_launch("failed to launch CUDA flow PCG preconditioner"); + const auto next_rho = + dot(workspace.residual.data(), workspace.preconditioned.data(), count, workspace, stream); + if (!std::isfinite(next_rho) || rho == 0.0) { + throw std::runtime_error(std::string(label) + + " conjugate gradient encountered a preconditioner breakdown"); + } + const auto beta_double = next_rho / rho; + if (!std::isfinite(beta_double) || std::abs(beta_double) > std::numeric_limits::max()) { + throw std::runtime_error(std::string(label) + + " conjugate gradient produced a non-finite direction"); + } + launch_flow_pcg_direction(workspace.preconditioned.data(), workspace.direction.data(), + static_cast(beta_double), count, stream); + check_launch("failed to launch CUDA flow PCG direction update"); + rho = next_rho; + } + throw std::runtime_error(std::string(label) + " conjugate gradient did not converge: relative " + + std::to_string(relative)); +} + +} // namespace + +DepthAveragedFlowResult solve_depth_averaged_flow(const SignalGridSpec& spec, + std::span mobility, + const DepthAveragedFlowParameters& parameters, + cudaStream_t stream) { + parameters.validate(); + const detail::DepthAveragedFlowSystem system(spec, mobility, parameters.axis); + const auto grid = make_grid_parameters(system.layout()); + DeviceBuffer mobility_buffer(grid.site_count, "failed to allocate CUDA depth mobility"); + DeviceBuffer diagonal_buffer(grid.site_count, "failed to allocate CUDA depth diagonal"); + DeviceBuffer rhs_buffer(grid.site_count, "failed to allocate CUDA depth right-hand side"); + DeviceBuffer pressure_buffer(grid.site_count, "failed to allocate CUDA depth pressure"); + DeviceBuffer velocity_buffer(grid.total_face_count, + "failed to allocate CUDA depth velocity"); + upload(mobility_buffer, std::span(system.mobility()), stream, + "failed to upload CUDA depth mobility"); + upload(diagonal_buffer, std::span(system.diagonal()), stream, + "failed to upload CUDA depth diagonal"); + upload(rhs_buffer, std::span(system.right_hand_side()), stream, + "failed to upload CUDA depth right-hand side"); + PcgWorkspace workspace(grid.site_count); + const auto report = + solve_pcg(rhs_buffer.data(), diagonal_buffer.data(), pressure_buffer.data(), workspace, + grid.site_count, parameters.relative_tolerance, parameters.max_iterations, + "CUDA depth-averaged flow", stream, [&](const float* input, float* output) { + launch_depth_flow_operator(input, mobility_buffer.data(), diagonal_buffer.data(), + output, grid, stream); + check_launch("failed to launch CUDA depth-averaged operator"); + }); + launch_depth_flow_velocity(pressure_buffer.data(), mobility_buffer.data(), velocity_buffer.data(), + grid, stream); + check_launch("failed to launch CUDA depth-averaged velocity reconstruction"); + const auto velocity = download(velocity_buffer, stream, "failed to download CUDA depth velocity"); + const auto scaled = detail::scale_velocity( + spec, system.layout(), velocity, system.open_inlet_faces(), parameters.mean_inlet_speed); + return { + .field = scaled.field, + .report = {.iterations = report.iterations, + .relative_residual = report.relative_residual, + .mean_inlet_speed = parameters.mean_inlet_speed, + .max_speed = scaled.max_speed}, + }; +} + +ResolvedFlowResult solve_resolved_flow(const SignalGridSpec& spec, std::span drag, + const ResolvedFlowParameters& parameters, + cudaStream_t stream) { + parameters.validate(); + const detail::ResolvedFlowSystem system(spec, drag, parameters.axis); + const auto grid = make_grid_parameters(system.layout()); + DeviceBuffer fluid_buffer(grid.site_count, "failed to allocate CUDA fluid mask"); + DeviceBuffer active_buffer(grid.total_face_count, + "failed to allocate CUDA active face mask"); + DeviceBuffer exists_buffer(grid.total_face_count, + "failed to allocate CUDA face existence mask"); + DeviceBuffer face_drag_buffer(grid.total_face_count, "failed to allocate CUDA face drag"); + DeviceBuffer face_diagonal_buffer(grid.total_face_count, + "failed to allocate CUDA momentum diagonal"); + DeviceBuffer force_buffer(grid.total_face_count, "failed to allocate CUDA momentum force"); + const auto pressure_diagonal = system.pressure_diagonal(); + DeviceBuffer pressure_diagonal_buffer(grid.site_count, + "failed to allocate CUDA pressure diagonal"); + upload(fluid_buffer, std::span(system.fluid()), stream, + "failed to upload CUDA fluid mask"); + upload(active_buffer, std::span(system.active()), stream, + "failed to upload CUDA active face mask"); + upload(exists_buffer, std::span(system.exists()), stream, + "failed to upload CUDA face existence mask"); + upload(face_drag_buffer, std::span(system.face_drag()), stream, + "failed to upload CUDA face drag"); + upload(face_diagonal_buffer, std::span(system.diagonal()), stream, + "failed to upload CUDA momentum diagonal"); + upload(force_buffer, std::span(system.force()), stream, + "failed to upload CUDA momentum force"); + upload(pressure_diagonal_buffer, std::span(pressure_diagonal), stream, + "failed to upload CUDA pressure diagonal"); + + DeviceBuffer particular(grid.total_face_count, + "failed to allocate CUDA particular velocity"); + DeviceBuffer schur_rhs(grid.site_count, + "failed to allocate CUDA pressure right-hand side"); + DeviceBuffer pressure(grid.site_count, "failed to allocate CUDA pressure"); + DeviceBuffer gradient(grid.total_face_count, "failed to allocate CUDA pressure gradient"); + DeviceBuffer response(grid.total_face_count, "failed to allocate CUDA momentum response"); + DeviceBuffer correction(grid.total_face_count, + "failed to allocate CUDA velocity correction"); + DeviceBuffer velocity(grid.total_face_count, "failed to allocate CUDA resolved velocity"); + DeviceBuffer divergence(grid.site_count, "failed to allocate CUDA velocity divergence"); + PcgWorkspace inner_workspace(grid.total_face_count); + PcgWorkspace outer_workspace(grid.site_count); + + std::uint64_t inner_iterations = 0; + const auto solve_momentum = [&](const float* rhs, float* solution) { + const auto result = solve_pcg( + rhs, face_diagonal_buffer.data(), solution, inner_workspace, grid.total_face_count, + parameters.inner_relative_tolerance, parameters.max_inner_iterations, + "CUDA resolved-flow momentum", stream, [&](const float* input, float* output) { + launch_resolved_flow_momentum(input, active_buffer.data(), exists_buffer.data(), + face_drag_buffer.data(), output, grid, stream); + check_launch("failed to launch CUDA resolved-flow momentum operator"); + }); + if (inner_iterations > std::numeric_limits::max() - result.iterations) { + throw std::overflow_error("resolved-flow inner iteration count overflow"); + } + inner_iterations += result.iterations; + }; + + solve_momentum(force_buffer.data(), particular.data()); + launch_resolved_flow_divergence(particular.data(), fluid_buffer.data(), schur_rhs.data(), grid, + stream); + check_launch("failed to launch CUDA resolved-flow divergence"); + launch_flow_vector_negate(schur_rhs.data(), schur_rhs.data(), grid.site_count, stream); + check_launch("failed to launch CUDA pressure right-hand-side negation"); + const auto pressure_report = solve_pcg( + schur_rhs.data(), pressure_diagonal_buffer.data(), pressure.data(), outer_workspace, + grid.site_count, parameters.relative_tolerance, parameters.max_outer_iterations, + "CUDA resolved-flow pressure", stream, [&](const float* input, float* output) { + launch_resolved_flow_gradient(input, fluid_buffer.data(), active_buffer.data(), + gradient.data(), grid, stream); + check_launch("failed to launch CUDA resolved-flow gradient"); + solve_momentum(gradient.data(), response.data()); + launch_resolved_flow_divergence(response.data(), fluid_buffer.data(), output, grid, stream); + check_launch("failed to launch CUDA resolved-flow Schur divergence"); + launch_flow_vector_negate(output, output, grid.site_count, stream); + check_launch("failed to launch CUDA resolved-flow Schur negation"); + }); + launch_resolved_flow_gradient(pressure.data(), fluid_buffer.data(), active_buffer.data(), + gradient.data(), grid, stream); + check_launch("failed to launch CUDA resolved-flow final gradient"); + solve_momentum(gradient.data(), correction.data()); + launch_flow_vector_subtract(particular.data(), correction.data(), velocity.data(), + grid.total_face_count, stream); + check_launch("failed to launch CUDA resolved-flow velocity correction"); + launch_resolved_flow_divergence(velocity.data(), fluid_buffer.data(), divergence.data(), grid, + stream); + check_launch("failed to launch CUDA resolved-flow final divergence"); + const auto divergence_values = + download(divergence, stream, "failed to download CUDA resolved-flow divergence"); + const auto velocity_values = + download(velocity, stream, "failed to download CUDA resolved-flow velocity"); + + double divergence_square_sum = 0.0; + std::size_t fluid_count = 0; + for (std::size_t site = 0; site < system.fluid().size(); ++site) { + if (system.fluid()[site] != 0) { + const auto value = static_cast(divergence_values[site]); + divergence_square_sum += value * value; + ++fluid_count; + } + } + const auto divergence_rms = + fluid_count == 0 ? 0.0 : std::sqrt(divergence_square_sum / static_cast(fluid_count)); + const auto scaled = + detail::scale_velocity(spec, system.layout(), velocity_values, system.open_inlet_faces(), + parameters.mean_inlet_speed); + return { + .field = scaled.field, + .report = {.outer_iterations = pressure_report.iterations, + .inner_iterations = inner_iterations, + .divergence_rms = static_cast(divergence_rms * std::abs(scaled.factor)), + .mean_inlet_speed = parameters.mean_inlet_speed, + .max_speed = scaled.max_speed, + .min_gap_voxels = system.minimum_gap_voxels()}, + }; +} + +} // namespace cm::cuda diff --git a/cpp/cuda/cuda_flow.cuh b/cpp/cuda/cuda_flow.cuh new file mode 100644 index 0000000..b67653c --- /dev/null +++ b/cpp/cuda/cuda_flow.cuh @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include + +#include "cm/flow.hpp" + +namespace cm::cuda { + +[[nodiscard]] DepthAveragedFlowResult solve_depth_averaged_flow( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters, cudaStream_t stream); + +[[nodiscard]] ResolvedFlowResult solve_resolved_flow(const SignalGridSpec& spec, + std::span drag, + const ResolvedFlowParameters& parameters, + cudaStream_t stream); + +} // namespace cm::cuda diff --git a/cpp/cuda/kernels/flow.cu b/cpp/cuda/kernels/flow.cu new file mode 100644 index 0000000..8d8c601 --- /dev/null +++ b/cpp/cuda/kernels/flow.cu @@ -0,0 +1,379 @@ +#include "flow.cuh" + +namespace cm::cuda { +namespace { + +constexpr std::uint32_t threads_per_block = 256; + +struct Coordinate { + std::uint32_t values[3]; +}; + +struct FaceCoordinate { + std::uint32_t component; + Coordinate coordinate; + Coordinate dimensions; +}; + +__device__ std::uint32_t site_index(const Coordinate& coordinate, const FlowGridParameters& grid) { + return (coordinate.values[0] * grid.dimensions[1] + coordinate.values[1]) * grid.dimensions[2] + + coordinate.values[2]; +} + +__device__ Coordinate site_coordinate(std::uint32_t index, const FlowGridParameters& grid) { + Coordinate result{}; + result.values[2] = index % grid.dimensions[2]; + index /= grid.dimensions[2]; + result.values[1] = index % grid.dimensions[1]; + result.values[0] = index / grid.dimensions[1]; + return result; +} + +__device__ FaceCoordinate face_coordinate(std::uint32_t index, const FlowGridParameters& grid) { + const auto component = + index < grid.face_offsets[1] ? 0U : (index < grid.face_offsets[2] ? 1U : 2U); + auto local = index - grid.face_offsets[component]; + FaceCoordinate result{.component = component}; + for (std::uint32_t axis = 0; axis < 3; ++axis) { + result.dimensions.values[axis] = grid.dimensions[axis] + (axis == component ? 1U : 0U); + } + result.coordinate.values[2] = local % result.dimensions.values[2]; + local /= result.dimensions.values[2]; + result.coordinate.values[1] = local % result.dimensions.values[1]; + result.coordinate.values[0] = local / result.dimensions.values[1]; + return result; +} + +__device__ std::uint32_t face_index(std::uint32_t component, const Coordinate& coordinate, + const FlowGridParameters& grid) { + if (component == 0) { + return grid.face_offsets[0] + + (coordinate.values[0] * grid.dimensions[1] + coordinate.values[1]) * grid.dimensions[2] + + coordinate.values[2]; + } + if (component == 1) { + return grid.face_offsets[1] + + (coordinate.values[0] * (grid.dimensions[1] + 1) + coordinate.values[1]) * + grid.dimensions[2] + + coordinate.values[2]; + } + return grid.face_offsets[2] + + (coordinate.values[0] * grid.dimensions[1] + coordinate.values[1]) * + (grid.dimensions[2] + 1) + + coordinate.values[2]; +} + +__device__ float harmonic_mean(float first, float second) { + const auto sum = first + second; + return sum > 0.0F ? 2.0F * first * second / sum : 0.0F; +} + +__global__ void depth_flow_operator(const float* input, const float* mobility, + const float* diagonal, float* output, FlowGridParameters grid) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= grid.site_count) { + return; + } + if (diagonal[index] == 0.0F) { + output[index] = 0.0F; + return; + } + const auto coordinate = site_coordinate(index, grid); + auto result = diagonal[index] * input[index]; + for (std::uint32_t axis = 0; axis < 3; ++axis) { + if (coordinate.values[axis] > 0) { + auto neighbor = coordinate; + --neighbor.values[axis]; + const auto neighbor_index = site_index(neighbor, grid); + result -= harmonic_mean(mobility[index], mobility[neighbor_index]) * + grid.inverse_spacing_squared[axis] * input[neighbor_index]; + } + if (coordinate.values[axis] + 1 < grid.dimensions[axis]) { + auto neighbor = coordinate; + ++neighbor.values[axis]; + const auto neighbor_index = site_index(neighbor, grid); + result -= harmonic_mean(mobility[index], mobility[neighbor_index]) * + grid.inverse_spacing_squared[axis] * input[neighbor_index]; + } + } + output[index] = result; +} + +__global__ void depth_flow_velocity(const float* pressure, const float* mobility, float* velocity, + FlowGridParameters grid) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= grid.total_face_count) { + return; + } + const auto face = face_coordinate(index, grid); + const auto component = face.component; + const auto has_lower = face.coordinate.values[component] > 0; + const auto has_upper = face.coordinate.values[component] < grid.dimensions[component]; + auto lower_coordinate = face.coordinate; + if (has_lower) { + --lower_coordinate.values[component]; + } + const auto lower = has_lower ? site_index(lower_coordinate, grid) : 0; + const auto upper = has_upper ? site_index(face.coordinate, grid) : 0; + auto value = 0.0F; + if (has_lower && has_upper) { + value = -harmonic_mean(mobility[lower], mobility[upper]) * (pressure[upper] - pressure[lower]) / + grid.spacing[component]; + } else if (component == grid.flow_axis && has_upper) { + value = 2.0F * mobility[upper] * (1.0F - pressure[upper]) / grid.spacing[component]; + } else if (component == grid.flow_axis && has_lower) { + value = 2.0F * mobility[lower] * pressure[lower] / grid.spacing[component]; + } + velocity[index] = value; +} + +__global__ void resolved_flow_momentum(const float* input, const std::uint8_t* active, + const std::uint8_t* exists, const float* face_drag, + float* output, FlowGridParameters grid) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= grid.total_face_count) { + return; + } + if (active[index] == 0) { + output[index] = 0.0F; + return; + } + const auto face = face_coordinate(index, grid); + auto result = face_drag[index] * input[index]; + for (std::uint32_t axis = 0; axis < 3; ++axis) { + if (grid.dimensions[axis] == 1) { + continue; + } + for (int offset = -1; offset <= 1; offset += 2) { + const auto in_bounds = offset < 0 + ? face.coordinate.values[axis] > 0 + : face.coordinate.values[axis] + 1 < face.dimensions.values[axis]; + std::uint32_t neighbor_index = 0; + if (in_bounds) { + auto coordinate = face.coordinate; + if (offset < 0) { + --coordinate.values[axis]; + } else { + ++coordinate.values[axis]; + } + neighbor_index = face_index(face.component, coordinate, grid); + } + float neighbor = 0.0F; + if (axis == face.component) { + neighbor = in_bounds ? input[neighbor_index] : input[index]; + } else if (in_bounds && exists[neighbor_index] != 0) { + neighbor = input[neighbor_index]; + } else { + neighbor = -input[index]; + } + result -= (neighbor - input[index]) * grid.inverse_spacing_squared[axis]; + } + } + output[index] = result; +} + +__global__ void resolved_flow_gradient(const float* pressure, const std::uint8_t* fluid, + const std::uint8_t* active, float* gradient, + FlowGridParameters grid) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= grid.total_face_count) { + return; + } + if (active[index] == 0) { + gradient[index] = 0.0F; + return; + } + const auto face = face_coordinate(index, grid); + const auto component = face.component; + const auto has_lower = face.coordinate.values[component] > 0; + const auto has_upper = face.coordinate.values[component] < grid.dimensions[component]; + auto lower_coordinate = face.coordinate; + if (has_lower) { + --lower_coordinate.values[component]; + } + const auto lower = has_lower ? site_index(lower_coordinate, grid) : 0; + const auto upper = has_upper ? site_index(face.coordinate, grid) : 0; + const auto lower_value = has_lower && fluid[lower] != 0 ? pressure[lower] : 0.0F; + const auto upper_value = has_upper && fluid[upper] != 0 ? pressure[upper] : 0.0F; + gradient[index] = (upper_value - lower_value) / grid.spacing[component]; +} + +__global__ void resolved_flow_divergence(const float* velocity, const std::uint8_t* fluid, + float* divergence, FlowGridParameters grid) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= grid.site_count) { + return; + } + if (fluid[index] == 0) { + divergence[index] = 0.0F; + return; + } + const auto coordinate = site_coordinate(index, grid); + auto result = 0.0F; + for (std::uint32_t component = 0; component < 3; ++component) { + auto upper = coordinate; + ++upper.values[component]; + result += (velocity[face_index(component, upper, grid)] - + velocity[face_index(component, coordinate, grid)]) / + grid.spacing[component]; + } + divergence[index] = result; +} + +__global__ void flow_pcg_initialize(const float* right_hand_side, const float* diagonal, + float* solution, float* residual, float* preconditioned, + float* direction, std::uint32_t count) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) { + return; + } + const auto value = right_hand_side[index]; + const auto scaled = diagonal[index] > 0.0F ? value / diagonal[index] : 0.0F; + solution[index] = 0.0F; + residual[index] = value; + preconditioned[index] = scaled; + direction[index] = scaled; +} + +__global__ void flow_pcg_update(float* solution, float* residual, const float* direction, + const float* transformed, float alpha, std::uint32_t count) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) { + solution[index] += alpha * direction[index]; + residual[index] -= alpha * transformed[index]; + } +} + +__global__ void flow_pcg_precondition(const float* residual, const float* diagonal, + float* preconditioned, std::uint32_t count) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) { + preconditioned[index] = diagonal[index] > 0.0F ? residual[index] / diagonal[index] : 0.0F; + } +} + +__global__ void flow_pcg_direction(const float* preconditioned, float* direction, float beta, + std::uint32_t count) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) { + direction[index] = preconditioned[index] + beta * direction[index]; + } +} + +__global__ void flow_vector_negate(const float* input, float* output, std::uint32_t count) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) { + output[index] = -input[index]; + } +} + +__global__ void flow_vector_subtract(const float* left, const float* right, float* output, + std::uint32_t count) { + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) { + output[index] = left[index] - right[index]; + } +} + +__global__ void flow_dot_partial(const float* left, const float* right, float* partials, + std::uint32_t count) { + __shared__ float values[flow_reduction_width]; + const auto index = blockIdx.x * blockDim.x + threadIdx.x; + values[threadIdx.x] = index < count ? left[index] * right[index] : 0.0F; + __syncthreads(); + for (std::uint32_t stride = flow_reduction_width / 2; stride > 0; stride >>= 1) { + if (threadIdx.x < stride) { + values[threadIdx.x] += values[threadIdx.x + stride]; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + partials[blockIdx.x] = values[0]; + } +} + +std::uint32_t block_count(std::uint32_t count, std::uint32_t width = threads_per_block) { + return ((count - 1) / width) + 1; +} + +} // namespace + +void launch_depth_flow_operator(const float* input, const float* mobility, const float* diagonal, + float* output, const FlowGridParameters& grid, + cudaStream_t stream) { + depth_flow_operator<<>>( + input, mobility, diagonal, output, grid); +} + +void launch_depth_flow_velocity(const float* pressure, const float* mobility, float* velocity, + const FlowGridParameters& grid, cudaStream_t stream) { + depth_flow_velocity<<>>( + pressure, mobility, velocity, grid); +} + +void launch_resolved_flow_momentum(const float* input, const std::uint8_t* active, + const std::uint8_t* exists, const float* face_drag, + float* output, const FlowGridParameters& grid, + cudaStream_t stream) { + resolved_flow_momentum<<>>( + input, active, exists, face_drag, output, grid); +} + +void launch_resolved_flow_gradient(const float* pressure, const std::uint8_t* fluid, + const std::uint8_t* active, float* gradient, + const FlowGridParameters& grid, cudaStream_t stream) { + resolved_flow_gradient<<>>( + pressure, fluid, active, gradient, grid); +} + +void launch_resolved_flow_divergence(const float* velocity, const std::uint8_t* fluid, + float* divergence, const FlowGridParameters& grid, + cudaStream_t stream) { + resolved_flow_divergence<<>>( + velocity, fluid, divergence, grid); +} + +void launch_flow_pcg_initialize(const float* right_hand_side, const float* diagonal, + float* solution, float* residual, float* preconditioned, + float* direction, std::uint32_t count, cudaStream_t stream) { + flow_pcg_initialize<<>>( + right_hand_side, diagonal, solution, residual, preconditioned, direction, count); +} + +void launch_flow_pcg_update(float* solution, float* residual, const float* direction, + const float* transformed, float alpha, std::uint32_t count, + cudaStream_t stream) { + flow_pcg_update<<>>( + solution, residual, direction, transformed, alpha, count); +} + +void launch_flow_pcg_precondition(const float* residual, const float* diagonal, + float* preconditioned, std::uint32_t count, cudaStream_t stream) { + flow_pcg_precondition<<>>( + residual, diagonal, preconditioned, count); +} + +void launch_flow_pcg_direction(const float* preconditioned, float* direction, float beta, + std::uint32_t count, cudaStream_t stream) { + flow_pcg_direction<<>>(preconditioned, + direction, beta, count); +} + +void launch_flow_vector_negate(const float* input, float* output, std::uint32_t count, + cudaStream_t stream) { + flow_vector_negate<<>>(input, output, count); +} + +void launch_flow_vector_subtract(const float* left, const float* right, float* output, + std::uint32_t count, cudaStream_t stream) { + flow_vector_subtract<<>>(left, right, output, + count); +} + +void launch_flow_dot_partial(const float* left, const float* right, float* partials, + std::uint32_t count, cudaStream_t stream) { + flow_dot_partial<<>>( + left, right, partials, count); +} + +} // namespace cm::cuda diff --git a/cpp/cuda/kernels/flow.cuh b/cpp/cuda/kernels/flow.cuh new file mode 100644 index 0000000..057cffc --- /dev/null +++ b/cpp/cuda/kernels/flow.cuh @@ -0,0 +1,56 @@ +#pragma once + +#include + +#include + +namespace cm::cuda { + +struct alignas(16) FlowGridParameters { + std::uint32_t dimensions[4]; + float spacing[4]; + float inverse_spacing_squared[4]; + std::uint32_t face_offsets[4]; + std::uint32_t face_counts[4]; + std::uint32_t flow_axis; + std::uint32_t site_count; + std::uint32_t total_face_count; + std::uint32_t padding; +}; + +static_assert(sizeof(FlowGridParameters) == 96); + +void launch_depth_flow_operator(const float* input, const float* mobility, const float* diagonal, + float* output, const FlowGridParameters& grid, cudaStream_t stream); +void launch_depth_flow_velocity(const float* pressure, const float* mobility, float* velocity, + const FlowGridParameters& grid, cudaStream_t stream); +void launch_resolved_flow_momentum(const float* input, const std::uint8_t* active, + const std::uint8_t* exists, const float* face_drag, + float* output, const FlowGridParameters& grid, + cudaStream_t stream); +void launch_resolved_flow_gradient(const float* pressure, const std::uint8_t* fluid, + const std::uint8_t* active, float* gradient, + const FlowGridParameters& grid, cudaStream_t stream); +void launch_resolved_flow_divergence(const float* velocity, const std::uint8_t* fluid, + float* divergence, const FlowGridParameters& grid, + cudaStream_t stream); +void launch_flow_pcg_initialize(const float* right_hand_side, const float* diagonal, + float* solution, float* residual, float* preconditioned, + float* direction, std::uint32_t count, cudaStream_t stream); +void launch_flow_pcg_update(float* solution, float* residual, const float* direction, + const float* transformed, float alpha, std::uint32_t count, + cudaStream_t stream); +void launch_flow_pcg_precondition(const float* residual, const float* diagonal, + float* preconditioned, std::uint32_t count, cudaStream_t stream); +void launch_flow_pcg_direction(const float* preconditioned, float* direction, float beta, + std::uint32_t count, cudaStream_t stream); +void launch_flow_vector_negate(const float* input, float* output, std::uint32_t count, + cudaStream_t stream); +void launch_flow_vector_subtract(const float* left, const float* right, float* output, + std::uint32_t count, cudaStream_t stream); +void launch_flow_dot_partial(const float* left, const float* right, float* partials, + std::uint32_t count, cudaStream_t stream); + +inline constexpr std::uint32_t flow_reduction_width = 64; + +} // namespace cm::cuda diff --git a/cpp/include/cm/backend.hpp b/cpp/include/cm/backend.hpp index cdcd0e9..38ac0a7 100644 --- a/cpp/include/cm/backend.hpp +++ b/cpp/include/cm/backend.hpp @@ -8,6 +8,7 @@ #include "cm/constraints.hpp" #include "cm/contact_graph.hpp" #include "cm/coupled_rates.hpp" +#include "cm/flow.hpp" #include "cm/mechanics.hpp" #include "cm/signals.hpp" #include "cm/species.hpp" @@ -38,6 +39,12 @@ class ComputeBackend { [[nodiscard]] virtual MechanicsSolveResult solve_cell_mechanics( const WorldState& state, const ContactGraph& contacts, const ExternalContactGraph& external_contacts, const MechanicsParameters& parameters) = 0; + [[nodiscard]] virtual DepthAveragedFlowResult solve_depth_averaged_flow( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters) = 0; + [[nodiscard]] virtual ResolvedFlowResult solve_resolved_flow( + const SignalGridSpec& spec, std::span drag, + const ResolvedFlowParameters& parameters) = 0; }; [[nodiscard]] std::unique_ptr make_cpu_backend(std::uint32_t device_index = 0); diff --git a/cpp/include/cm/flow.hpp b/cpp/include/cm/flow.hpp new file mode 100644 index 0000000..e175de8 --- /dev/null +++ b/cpp/include/cm/flow.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include +#include + +#include "cm/signals.hpp" + +namespace cm { + +enum class FlowAxis : std::uint8_t { + x, + y, + z, +}; + +struct DepthAveragedFlowParameters { + float mean_inlet_speed{1.0F}; + FlowAxis axis{FlowAxis::y}; + float relative_tolerance{1.0e-6F}; + std::uint32_t max_iterations{50'000}; + + void validate() const; +}; + +struct DepthAveragedFlowReport { + std::uint32_t iterations{0}; + float relative_residual{0.0F}; + float mean_inlet_speed{0.0F}; + float max_speed{0.0F}; +}; + +struct DepthAveragedFlowResult { + SignalGridVelocityField field; + DepthAveragedFlowReport report; +}; + +struct ResolvedFlowParameters { + float mean_inlet_speed{1.0F}; + FlowAxis axis{FlowAxis::y}; + float relative_tolerance{1.0e-6F}; + std::uint32_t max_outer_iterations{500}; + float inner_relative_tolerance{1.0e-6F}; + std::uint32_t max_inner_iterations{50'000}; + + void validate() const; +}; + +struct ResolvedFlowReport { + std::uint32_t outer_iterations{0}; + std::uint64_t inner_iterations{0}; + float divergence_rms{0.0F}; + float mean_inlet_speed{0.0F}; + float max_speed{0.0F}; + std::uint32_t min_gap_voxels{0}; +}; + +struct ResolvedFlowResult { + SignalGridVelocityField field; + ResolvedFlowReport report; +}; + +void validate_flow_grid(const SignalGridSpec& spec, FlowAxis axis); + +[[nodiscard]] DepthAveragedFlowResult solve_depth_averaged_flow_cpu( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters = DepthAveragedFlowParameters{}); + +[[nodiscard]] ResolvedFlowResult solve_resolved_flow_cpu( + const SignalGridSpec& spec, std::span drag, + const ResolvedFlowParameters& parameters = ResolvedFlowParameters{}); + +} // namespace cm diff --git a/cpp/include/cm/simulation.hpp b/cpp/include/cm/simulation.hpp index 81eb6bb..7b51898 100644 --- a/cpp/include/cm/simulation.hpp +++ b/cpp/include/cm/simulation.hpp @@ -62,6 +62,12 @@ class Simulation { const MechanicsIntegrationParameters& integration_parameters = MechanicsIntegrationParameters{}, const ConstraintContactParameters& constraint_parameters = ConstraintContactParameters{}); + [[nodiscard]] DepthAveragedFlowResult solve_depth_averaged_flow( + const SignalGridSpec& spec, std::span mobility = {}, + const DepthAveragedFlowParameters& parameters = DepthAveragedFlowParameters{}); + [[nodiscard]] ResolvedFlowResult solve_resolved_flow( + const SignalGridSpec& spec, std::span drag = {}, + const ResolvedFlowParameters& parameters = ResolvedFlowParameters{}); [[nodiscard]] CellSnapshot cell(CellId id) const; [[nodiscard]] std::vector cells() const; diff --git a/cpp/include/cm/types.hpp b/cpp/include/cm/types.hpp index b20ca08..c05b510 100644 --- a/cpp/include/cm/types.hpp +++ b/cpp/include/cm/types.hpp @@ -68,6 +68,8 @@ enum class BackendFeature : std::uint8_t { external_constraints, signals, coupled_rates, + depth_averaged_flow, + resolved_flow, }; struct BackendInfo { diff --git a/cpp/metal/kernels/flow.metal b/cpp/metal/kernels/flow.metal new file mode 100644 index 0000000..618fc68 --- /dev/null +++ b/cpp/metal/kernels/flow.metal @@ -0,0 +1,316 @@ +#include + +using namespace metal; + +struct FlowGridParameters { + uint4 dimensions; + float4 spacing; + float4 inverse_spacing_squared; + uint4 face_offsets; + uint4 face_counts; + uint flow_axis; + uint site_count; + uint total_face_count; + uint padding; +}; + +struct FaceCoordinate { + uint component; + uint3 coordinate; + uint3 dimensions; +}; + +inline uint site_index(uint3 coordinate, constant FlowGridParameters& grid) { + return (coordinate.x * grid.dimensions.y + coordinate.y) * grid.dimensions.z + coordinate.z; +} + +inline uint3 site_coordinate(uint index, constant FlowGridParameters& grid) { + const uint z = index % grid.dimensions.z; + index /= grid.dimensions.z; + const uint y = index % grid.dimensions.y; + return uint3(index / grid.dimensions.y, y, z); +} + +inline FaceCoordinate face_coordinate(uint index, constant FlowGridParameters& grid) { + uint component = index < grid.face_offsets.y ? 0 : (index < grid.face_offsets.z ? 1 : 2); + const uint offset = component == 0 ? grid.face_offsets.x + : (component == 1 ? grid.face_offsets.y : grid.face_offsets.z); + uint local = index - offset; + uint3 dimensions = grid.dimensions.xyz; + dimensions[component] += 1; + const uint z = local % dimensions.z; + local /= dimensions.z; + const uint y = local % dimensions.y; + return {component, uint3(local / dimensions.y, y, z), dimensions}; +} + +inline uint face_index(uint component, uint3 coordinate, constant FlowGridParameters& grid) { + const uint offset = component == 0 ? grid.face_offsets.x + : (component == 1 ? grid.face_offsets.y : grid.face_offsets.z); + if (component == 0) { + return offset + (coordinate.x * grid.dimensions.y + coordinate.y) * grid.dimensions.z + + coordinate.z; + } + if (component == 1) { + return offset + (coordinate.x * (grid.dimensions.y + 1) + coordinate.y) * grid.dimensions.z + + coordinate.z; + } + return offset + (coordinate.x * grid.dimensions.y + coordinate.y) * (grid.dimensions.z + 1) + + coordinate.z; +} + +inline float harmonic_mean(float first, float second) { + const float sum = first + second; + return sum > 0.0f ? 2.0f * first * second / sum : 0.0f; +} + +kernel void depth_flow_operator(device const float* input [[buffer(0)]], + device const float* mobility [[buffer(1)]], + device const float* diagonal [[buffer(2)]], + device float* output [[buffer(3)]], + constant FlowGridParameters& grid [[buffer(4)]], + uint index [[thread_position_in_grid]]) { + if (index >= grid.site_count) { + return; + } + if (diagonal[index] == 0.0f) { + output[index] = 0.0f; + return; + } + const uint3 coordinate = site_coordinate(index, grid); + float result = diagonal[index] * input[index]; + for (uint axis = 0; axis < 3; ++axis) { + if (coordinate[axis] > 0) { + uint3 neighbor = coordinate; + neighbor[axis] -= 1; + const uint neighbor_index = site_index(neighbor, grid); + result -= harmonic_mean(mobility[index], mobility[neighbor_index]) * + grid.inverse_spacing_squared[axis] * input[neighbor_index]; + } + if (coordinate[axis] + 1 < grid.dimensions[axis]) { + uint3 neighbor = coordinate; + neighbor[axis] += 1; + const uint neighbor_index = site_index(neighbor, grid); + result -= harmonic_mean(mobility[index], mobility[neighbor_index]) * + grid.inverse_spacing_squared[axis] * input[neighbor_index]; + } + } + output[index] = result; +} + +kernel void depth_flow_velocity(device const float* pressure [[buffer(0)]], + device const float* mobility [[buffer(1)]], + device float* velocity [[buffer(2)]], + constant FlowGridParameters& grid [[buffer(3)]], + uint index [[thread_position_in_grid]]) { + if (index >= grid.total_face_count) { + return; + } + const FaceCoordinate face = face_coordinate(index, grid); + const uint component = face.component; + const bool has_lower = face.coordinate[component] > 0; + const bool has_upper = face.coordinate[component] < grid.dimensions[component]; + uint3 lower_coordinate = face.coordinate; + if (has_lower) { + lower_coordinate[component] -= 1; + } + const uint lower = has_lower ? site_index(lower_coordinate, grid) : 0; + const uint upper = has_upper ? site_index(face.coordinate, grid) : 0; + float value = 0.0f; + if (has_lower && has_upper) { + value = -harmonic_mean(mobility[lower], mobility[upper]) * (pressure[upper] - pressure[lower]) / + grid.spacing[component]; + } else if (component == grid.flow_axis && has_upper) { + value = 2.0f * mobility[upper] * (1.0f - pressure[upper]) / grid.spacing[component]; + } else if (component == grid.flow_axis && has_lower) { + value = 2.0f * mobility[lower] * pressure[lower] / grid.spacing[component]; + } + velocity[index] = value; +} + +kernel void resolved_flow_momentum(device const float* input [[buffer(0)]], + device const uchar* active [[buffer(1)]], + device const uchar* exists [[buffer(2)]], + device const float* face_drag [[buffer(3)]], + device float* output [[buffer(4)]], + constant FlowGridParameters& grid [[buffer(5)]], + uint index [[thread_position_in_grid]]) { + if (index >= grid.total_face_count) { + return; + } + if (active[index] == 0) { + output[index] = 0.0f; + return; + } + const FaceCoordinate face = face_coordinate(index, grid); + float result = face_drag[index] * input[index]; + for (uint axis = 0; axis < 3; ++axis) { + if (grid.dimensions[axis] == 1) { + continue; + } + for (int offset = -1; offset <= 1; offset += 2) { + const bool in_bounds = offset < 0 ? face.coordinate[axis] > 0 + : face.coordinate[axis] + 1 < face.dimensions[axis]; + float neighbor = 0.0f; + uint neighbor_index = 0; + if (in_bounds) { + uint3 coordinate = face.coordinate; + if (offset < 0) { + coordinate[axis] -= 1; + } else { + coordinate[axis] += 1; + } + neighbor_index = face_index(face.component, coordinate, grid); + } + if (axis == face.component) { + neighbor = in_bounds ? input[neighbor_index] : input[index]; + } else if (in_bounds && exists[neighbor_index] != 0) { + neighbor = input[neighbor_index]; + } else { + neighbor = -input[index]; + } + result -= (neighbor - input[index]) * grid.inverse_spacing_squared[axis]; + } + } + output[index] = result; +} + +kernel void resolved_flow_gradient(device const float* pressure [[buffer(0)]], + device const uchar* fluid [[buffer(1)]], + device const uchar* active [[buffer(2)]], + device float* gradient [[buffer(3)]], + constant FlowGridParameters& grid [[buffer(4)]], + uint index [[thread_position_in_grid]]) { + if (index >= grid.total_face_count) { + return; + } + if (active[index] == 0) { + gradient[index] = 0.0f; + return; + } + const FaceCoordinate face = face_coordinate(index, grid); + const uint component = face.component; + const bool has_lower = face.coordinate[component] > 0; + const bool has_upper = face.coordinate[component] < grid.dimensions[component]; + uint3 lower_coordinate = face.coordinate; + if (has_lower) { + lower_coordinate[component] -= 1; + } + const uint lower = has_lower ? site_index(lower_coordinate, grid) : 0; + const uint upper = has_upper ? site_index(face.coordinate, grid) : 0; + const float lower_value = has_lower && fluid[lower] != 0 ? pressure[lower] : 0.0f; + const float upper_value = has_upper && fluid[upper] != 0 ? pressure[upper] : 0.0f; + gradient[index] = (upper_value - lower_value) / grid.spacing[component]; +} + +kernel void resolved_flow_divergence(device const float* velocity [[buffer(0)]], + device const uchar* fluid [[buffer(1)]], + device float* divergence [[buffer(2)]], + constant FlowGridParameters& grid [[buffer(3)]], + uint index [[thread_position_in_grid]]) { + if (index >= grid.site_count) { + return; + } + if (fluid[index] == 0) { + divergence[index] = 0.0f; + return; + } + const uint3 coordinate = site_coordinate(index, grid); + float result = 0.0f; + for (uint component = 0; component < 3; ++component) { + uint3 upper = coordinate; + upper[component] += 1; + result += (velocity[face_index(component, upper, grid)] - + velocity[face_index(component, coordinate, grid)]) / + grid.spacing[component]; + } + divergence[index] = result; +} + +kernel void flow_pcg_initialize( + device const float* right_hand_side [[buffer(0)]], device const float* diagonal [[buffer(1)]], + device float* solution [[buffer(2)]], device float* residual [[buffer(3)]], + device float* preconditioned [[buffer(4)]], device float* direction [[buffer(5)]], + constant uint& count [[buffer(6)]], uint index [[thread_position_in_grid]]) { + if (index >= count) { + return; + } + const float value = right_hand_side[index]; + const float scaled = diagonal[index] > 0.0f ? value / diagonal[index] : 0.0f; + solution[index] = 0.0f; + residual[index] = value; + preconditioned[index] = scaled; + direction[index] = scaled; +} + +kernel void flow_pcg_update(device float* solution [[buffer(0)]], + device float* residual [[buffer(1)]], + device const float* direction [[buffer(2)]], + device const float* transformed [[buffer(3)]], + constant float& alpha [[buffer(4)]], constant uint& count [[buffer(5)]], + uint index [[thread_position_in_grid]]) { + if (index < count) { + solution[index] += alpha * direction[index]; + residual[index] -= alpha * transformed[index]; + } +} + +kernel void flow_pcg_precondition(device const float* residual [[buffer(0)]], + device const float* diagonal [[buffer(1)]], + device float* preconditioned [[buffer(2)]], + constant uint& count [[buffer(3)]], + uint index [[thread_position_in_grid]]) { + if (index < count) { + preconditioned[index] = diagonal[index] > 0.0f ? residual[index] / diagonal[index] : 0.0f; + } +} + +kernel void flow_pcg_direction(device const float* preconditioned [[buffer(0)]], + device float* direction [[buffer(1)]], + constant float& beta [[buffer(2)]], + constant uint& count [[buffer(3)]], + uint index [[thread_position_in_grid]]) { + if (index < count) { + direction[index] = preconditioned[index] + beta * direction[index]; + } +} + +kernel void flow_vector_negate(device const float* input [[buffer(0)]], + device float* output [[buffer(1)]], + constant uint& count [[buffer(2)]], + uint index [[thread_position_in_grid]]) { + if (index < count) { + output[index] = -input[index]; + } +} + +kernel void flow_vector_subtract(device const float* left [[buffer(0)]], + device const float* right [[buffer(1)]], + device float* output [[buffer(2)]], + constant uint& count [[buffer(3)]], + uint index [[thread_position_in_grid]]) { + if (index < count) { + output[index] = left[index] - right[index]; + } +} + +kernel void flow_dot_partial(device const float* left [[buffer(0)]], + device const float* right [[buffer(1)]], + device float* partials [[buffer(2)]], + constant uint& count [[buffer(3)]], + uint index [[thread_position_in_grid]], + uint local_index [[thread_index_in_threadgroup]], + uint group_index [[threadgroup_position_in_grid]]) { + threadgroup float values[64]; + values[local_index] = index < count ? left[index] * right[index] : 0.0f; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = 32; stride > 0; stride >>= 1) { + if (local_index < stride) { + values[local_index] += values[local_index + stride]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (local_index == 0) { + partials[group_index] = values[0]; + } +} diff --git a/cpp/metal/metal_backend.mm b/cpp/metal/metal_backend.mm index 80e95bb..e25af2f 100644 --- a/cpp/metal/metal_backend.mm +++ b/cpp/metal/metal_backend.mm @@ -23,6 +23,7 @@ #include "cm/metal/mechanics_source.hpp" #include "cm/metal/signals_source.hpp" #include "cm/metal/species_source.hpp" +#include "metal_flow.hpp" namespace cm { namespace { @@ -252,6 +253,7 @@ explicit MetalBackend(std::uint32_t device_index) : device_index_(device_index) mechanics_reduce_pipeline_ = compile_pipeline(device_, mechanics_library, @"reduce_sum_pairs", "failed to create the Metal mechanics-reduction pipeline"); + flow_solver_ = std::make_unique(device_index_); } } @@ -272,7 +274,9 @@ explicit MetalBackend(std::uint32_t device_index) : device_index_(device_index) return feature == BackendFeature::growth || feature == BackendFeature::species || feature == BackendFeature::cell_contacts || feature == BackendFeature::cell_mechanics || feature == BackendFeature::external_constraints || feature == BackendFeature::signals || - feature == BackendFeature::coupled_rates; + feature == BackendFeature::coupled_rates || + feature == BackendFeature::depth_averaged_flow || + feature == BackendFeature::resolved_flow; } void advance_growth(WorldState& state, float dt) override { @@ -953,6 +957,18 @@ SignalSolveReport advance_coupled(WorldState& state, SignalGrid& grid, return result; } + [[nodiscard]] DepthAveragedFlowResult solve_depth_averaged_flow( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters) override { + return flow_solver_->solve_depth_averaged(spec, mobility, parameters); + } + + [[nodiscard]] ResolvedFlowResult solve_resolved_flow( + const SignalGridSpec& spec, std::span drag, + const ResolvedFlowParameters& parameters) override { + return flow_solver_->solve_resolved(spec, drag, parameters); + } + private: void ensure_growth_capacity(std::size_t count) { if (count <= growth_capacity_) { @@ -2127,6 +2143,7 @@ void update_search_direction(std::uint32_t cell_count, float beta) { id mechanics_subtract_pipeline_{nil}; id mechanics_dot_pipeline_{nil}; id mechanics_reduce_pipeline_{nil}; + std::unique_ptr flow_solver_; id lengths_{nil}; id growth_rates_{nil}; diff --git a/cpp/metal/metal_flow.hpp b/cpp/metal/metal_flow.hpp new file mode 100644 index 0000000..3eb7aca --- /dev/null +++ b/cpp/metal/metal_flow.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +#include "cm/flow.hpp" + +namespace cm::metal { + +class FlowSolver { + public: + explicit FlowSolver(std::uint32_t device_index); + ~FlowSolver(); + + FlowSolver(const FlowSolver&) = delete; + FlowSolver& operator=(const FlowSolver&) = delete; + + [[nodiscard]] DepthAveragedFlowResult solve_depth_averaged( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters); + [[nodiscard]] ResolvedFlowResult solve_resolved(const SignalGridSpec& spec, + std::span drag, + const ResolvedFlowParameters& parameters); + + private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace cm::metal diff --git a/cpp/metal/metal_flow.mm b/cpp/metal/metal_flow.mm new file mode 100644 index 0000000..835e295 --- /dev/null +++ b/cpp/metal/metal_flow.mm @@ -0,0 +1,579 @@ +#import +#import + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cm/metal/flow_source.hpp" +#include "core/flow_system.hpp" +#include "metal_flow.hpp" + +namespace cm::metal { +namespace { + +struct alignas(16) MetalUInt4 { + std::uint32_t x; + std::uint32_t y; + std::uint32_t z; + std::uint32_t w; +}; + +struct alignas(16) MetalFloat4 { + float x; + float y; + float z; + float w; +}; + +struct alignas(16) MetalFlowGridParameters { + MetalUInt4 dimensions; + MetalFloat4 spacing; + MetalFloat4 inverse_spacing_squared; + MetalUInt4 face_offsets; + MetalUInt4 face_counts; + std::uint32_t flow_axis; + std::uint32_t site_count; + std::uint32_t total_face_count; + std::uint32_t padding; +}; + +static_assert(sizeof(MetalUInt4) == 16); +static_assert(sizeof(MetalFloat4) == 16); +static_assert(sizeof(MetalFlowGridParameters) == 96); + +[[noreturn]] void throw_metal_error(const char* operation, NSError* error) { + const char* detail = error == nil ? "unknown Metal error" : error.localizedDescription.UTF8String; + throw std::runtime_error(std::string(operation) + ": " + detail); +} + +id select_device(std::uint32_t device_index) { + NSArray>* devices = MTLCopyAllDevices(); + if (devices.count == 0) { + id default_device = MTLCreateSystemDefaultDevice(); + if (device_index == 0 && default_device != nil) { + return default_device; + } + if (default_device == nil) { + throw std::runtime_error("Metal is unavailable on this system"); + } + } + if (static_cast(device_index) >= devices.count) { + throw std::out_of_range("Metal device index is unavailable"); + } + return devices[device_index]; +} + +id make_pipeline(id device, id library, + NSString* name) { + id function = [library newFunctionWithName:name]; + if (function == nil) { + throw std::runtime_error(std::string("Metal flow function is missing: ") + name.UTF8String); + } + NSError* error = nil; + id pipeline = [device newComputePipelineStateWithFunction:function + error:&error]; + if (pipeline == nil) { + throw_metal_error("failed to create a Metal flow pipeline", error); + } + return pipeline; +} + +void wait_for_command(id command, const char* operation) { + [command commit]; + [command waitUntilCompleted]; + if (command.status == MTLCommandBufferStatusError) { + throw_metal_error(operation, command.error); + } +} + +std::uint32_t checked_count(std::size_t count, const char* description) { + if (count == 0 || count > std::numeric_limits::max()) { + throw std::overflow_error(std::string("Metal flow ") + description + + " must fit the nonzero uint32 index space"); + } + return static_cast(count); +} + +MetalFlowGridParameters make_grid_parameters(const detail::FlowGridLayout& layout) { + const auto site_count = checked_count(layout.site_count(), "site count"); + const auto face_count = checked_count(layout.total_face_count(), "face count"); + const auto offsets = layout.face_offsets(); + const auto counts = layout.face_counts(); + for (const auto value : offsets) { + static_cast(checked_count(value == 0 ? 1 : value, "face offset")); + } + for (const auto value : counts) { + static_cast(checked_count(value, "component face count")); + } + const auto spacing = layout.spacing(); + return { + .dimensions = {layout.dimensions()[0], layout.dimensions()[1], layout.dimensions()[2], 0}, + .spacing = {spacing[0], spacing[1], spacing[2], 0.0F}, + .inverse_spacing_squared = + { + 1.0F / (spacing[0] * spacing[0]), + 1.0F / (spacing[1] * spacing[1]), + 1.0F / (spacing[2] * spacing[2]), + 0.0F, + }, + .face_offsets = {static_cast(offsets[0]), + static_cast(offsets[1]), + static_cast(offsets[2]), face_count}, + .face_counts = {static_cast(counts[0]), static_cast(counts[1]), + static_cast(counts[2]), face_count}, + .flow_axis = static_cast(layout.flow_axis()), + .site_count = site_count, + .total_face_count = face_count, + .padding = 0, + }; +} + +struct PcgReport { + std::uint32_t iterations{0}; + float relative_residual{0.0F}; +}; + +} // namespace + +struct FlowSolver::Impl { + explicit Impl(std::uint32_t device_index) { + @autoreleasepool { + device = select_device(device_index); + queue = [device newCommandQueue]; + if (queue == nil) { + throw std::runtime_error("failed to create a Metal flow command queue"); + } + NSString* source = [NSString stringWithUTF8String:flow_source]; + if (source == nil) { + throw std::runtime_error("Metal flow source is not valid UTF-8"); + } + NSError* error = nil; + id library = [device newLibraryWithSource:source options:nil error:&error]; + if (library == nil) { + throw_metal_error("failed to compile Metal flow", error); + } + depth_operator = make_pipeline(device, library, @"depth_flow_operator"); + depth_velocity = make_pipeline(device, library, @"depth_flow_velocity"); + momentum = make_pipeline(device, library, @"resolved_flow_momentum"); + gradient = make_pipeline(device, library, @"resolved_flow_gradient"); + divergence = make_pipeline(device, library, @"resolved_flow_divergence"); + pcg_initialize = make_pipeline(device, library, @"flow_pcg_initialize"); + pcg_update = make_pipeline(device, library, @"flow_pcg_update"); + pcg_precondition = make_pipeline(device, library, @"flow_pcg_precondition"); + pcg_direction = make_pipeline(device, library, @"flow_pcg_direction"); + vector_negate = make_pipeline(device, library, @"flow_vector_negate"); + vector_subtract = make_pipeline(device, library, @"flow_vector_subtract"); + dot_partial = make_pipeline(device, library, @"flow_dot_partial"); + if (dot_partial.maxTotalThreadsPerThreadgroup < reduction_width) { + throw std::runtime_error("Metal flow reduction requires 64 threads per threadgroup"); + } + } + } + + id allocate(std::size_t byte_count, const char* description) const { + id buffer = [device newBufferWithLength:byte_count + options:MTLResourceStorageModeShared]; + if (buffer == nil) { + throw std::runtime_error(std::string("failed to allocate Metal flow ") + description); + } + return buffer; + } + + template + id upload(std::span values, const char* description) const { + if (values.empty()) { + throw std::logic_error(std::string("cannot upload an empty Metal flow ") + description); + } + auto buffer = allocate(values.size_bytes(), description); + std::memcpy(buffer.contents, values.data(), values.size_bytes()); + return buffer; + } + + id float_buffer(std::size_t count, const char* description) const { + return allocate(count * sizeof(float), description); + } + + template + void dispatch(id pipeline, std::uint32_t count, const char* operation, + Bind&& bind) const { + @autoreleasepool { + id command = [queue commandBuffer]; + id encoder = [command computeCommandEncoder]; + if (command == nil || encoder == nil) { + throw std::runtime_error(std::string(operation) + ": failed to create a command"); + } + [encoder setComputePipelineState:pipeline]; + bind(encoder); + const auto width = std::min(pipeline.maxTotalThreadsPerThreadgroup, 256); + [encoder dispatchThreads:MTLSizeMake(count, 1, 1) + threadsPerThreadgroup:MTLSizeMake(width, 1, 1)]; + [encoder endEncoding]; + wait_for_command(command, operation); + } + } + + struct PcgWorkspace { + id residual; + id preconditioned; + id direction; + id transformed; + id partials; + }; + + PcgWorkspace make_workspace(std::uint32_t count, const char* description) const { + const auto partial_count = (count + reduction_width - 1) / reduction_width; + return { + .residual = float_buffer(count, (std::string(description) + " residual").c_str()), + .preconditioned = + float_buffer(count, (std::string(description) + " preconditioned residual").c_str()), + .direction = float_buffer(count, (std::string(description) + " direction").c_str()), + .transformed = float_buffer(count, (std::string(description) + " transformed").c_str()), + .partials = + float_buffer(partial_count, (std::string(description) + " reduction partials").c_str()), + }; + } + + double dot(id left, id right, std::uint32_t count, + id partials) const { + const auto group_count = (count + reduction_width - 1) / reduction_width; + @autoreleasepool { + id command = [queue commandBuffer]; + id encoder = [command computeCommandEncoder]; + if (command == nil || encoder == nil) { + throw std::runtime_error("Metal flow reduction failed to create a command"); + } + [encoder setComputePipelineState:dot_partial]; + [encoder setBuffer:left offset:0 atIndex:0]; + [encoder setBuffer:right offset:0 atIndex:1]; + [encoder setBuffer:partials offset:0 atIndex:2]; + [encoder setBytes:&count length:sizeof(count) atIndex:3]; + [encoder dispatchThreadgroups:MTLSizeMake(group_count, 1, 1) + threadsPerThreadgroup:MTLSizeMake(reduction_width, 1, 1)]; + [encoder endEncoding]; + wait_for_command(command, "Metal flow reduction failed"); + } + const auto* values = static_cast(partials.contents); + double result = 0.0; + for (std::uint32_t index = 0; index < group_count; ++index) { + result += values[index]; + } + return result; + } + + template + PcgReport solve_pcg(id right_hand_side, id diagonal, id solution, + PcgWorkspace& workspace, std::uint32_t count, float tolerance, + std::uint32_t max_iterations, const char* label, Apply&& apply) const { + dispatch(pcg_initialize, count, "Metal flow PCG initialization failed", + [&](id encoder) { + [encoder setBuffer:right_hand_side offset:0 atIndex:0]; + [encoder setBuffer:diagonal offset:0 atIndex:1]; + [encoder setBuffer:solution offset:0 atIndex:2]; + [encoder setBuffer:workspace.residual offset:0 atIndex:3]; + [encoder setBuffer:workspace.preconditioned offset:0 atIndex:4]; + [encoder setBuffer:workspace.direction offset:0 atIndex:5]; + [encoder setBytes:&count length:sizeof(count) atIndex:6]; + }); + const auto rhs_norm_squared = dot(right_hand_side, right_hand_side, count, workspace.partials); + if (rhs_norm_squared == 0.0) { + return {}; + } + const auto rhs_norm = std::sqrt(rhs_norm_squared); + auto rho = dot(workspace.residual, workspace.preconditioned, count, workspace.partials); + auto relative = 1.0; + for (std::uint32_t iteration = 1; iteration <= max_iterations; ++iteration) { + apply(workspace.direction, workspace.transformed); + const auto curvature = + dot(workspace.direction, workspace.transformed, count, workspace.partials); + if (!std::isfinite(curvature) || curvature <= 0.0) { + throw std::runtime_error(std::string(label) + + " conjugate gradient encountered non-positive curvature"); + } + const auto alpha_double = rho / curvature; + if (!std::isfinite(alpha_double) || + std::abs(alpha_double) > std::numeric_limits::max()) { + throw std::runtime_error(std::string(label) + + " conjugate gradient produced a non-finite step"); + } + const auto alpha = static_cast(alpha_double); + dispatch(pcg_update, count, "Metal flow PCG update failed", + [&](id encoder) { + [encoder setBuffer:solution offset:0 atIndex:0]; + [encoder setBuffer:workspace.residual offset:0 atIndex:1]; + [encoder setBuffer:workspace.direction offset:0 atIndex:2]; + [encoder setBuffer:workspace.transformed offset:0 atIndex:3]; + [encoder setBytes:&alpha length:sizeof(alpha) atIndex:4]; + [encoder setBytes:&count length:sizeof(count) atIndex:5]; + }); + const auto residual_squared = + dot(workspace.residual, workspace.residual, count, workspace.partials); + relative = std::sqrt(std::max(0.0, residual_squared)) / rhs_norm; + if (!std::isfinite(relative)) { + throw std::runtime_error(std::string(label) + + " conjugate gradient produced a non-finite residual"); + } + if (relative <= tolerance) { + return {.iterations = iteration, .relative_residual = static_cast(relative)}; + } + dispatch(pcg_precondition, count, "Metal flow PCG preconditioner failed", + [&](id encoder) { + [encoder setBuffer:workspace.residual offset:0 atIndex:0]; + [encoder setBuffer:diagonal offset:0 atIndex:1]; + [encoder setBuffer:workspace.preconditioned offset:0 atIndex:2]; + [encoder setBytes:&count length:sizeof(count) atIndex:3]; + }); + const auto next_rho = + dot(workspace.residual, workspace.preconditioned, count, workspace.partials); + if (!std::isfinite(next_rho) || rho == 0.0) { + throw std::runtime_error(std::string(label) + + " conjugate gradient encountered a preconditioner breakdown"); + } + const auto beta_double = next_rho / rho; + if (!std::isfinite(beta_double) || + std::abs(beta_double) > std::numeric_limits::max()) { + throw std::runtime_error(std::string(label) + + " conjugate gradient produced a non-finite direction"); + } + const auto beta = static_cast(beta_double); + dispatch(pcg_direction, count, "Metal flow PCG direction update failed", + [&](id encoder) { + [encoder setBuffer:workspace.preconditioned offset:0 atIndex:0]; + [encoder setBuffer:workspace.direction offset:0 atIndex:1]; + [encoder setBytes:&beta length:sizeof(beta) atIndex:2]; + [encoder setBytes:&count length:sizeof(count) atIndex:3]; + }); + rho = next_rho; + } + throw std::runtime_error(std::string(label) + + " conjugate gradient did not converge: relative " + + std::to_string(relative)); + } + + void apply_depth(id input, id mobility_buffer, + id diagonal_buffer, id output, + const MetalFlowGridParameters& grid) const { + dispatch(depth_operator, grid.site_count, "Metal depth-averaged operator failed", + [&](id encoder) { + [encoder setBuffer:input offset:0 atIndex:0]; + [encoder setBuffer:mobility_buffer offset:0 atIndex:1]; + [encoder setBuffer:diagonal_buffer offset:0 atIndex:2]; + [encoder setBuffer:output offset:0 atIndex:3]; + [encoder setBytes:&grid length:sizeof(grid) atIndex:4]; + }); + } + + void apply_momentum(id input, id active_buffer, id exists_buffer, + id face_drag_buffer, id output, + const MetalFlowGridParameters& grid) const { + dispatch(momentum, grid.total_face_count, "Metal resolved-flow momentum operator failed", + [&](id encoder) { + [encoder setBuffer:input offset:0 atIndex:0]; + [encoder setBuffer:active_buffer offset:0 atIndex:1]; + [encoder setBuffer:exists_buffer offset:0 atIndex:2]; + [encoder setBuffer:face_drag_buffer offset:0 atIndex:3]; + [encoder setBuffer:output offset:0 atIndex:4]; + [encoder setBytes:&grid length:sizeof(grid) atIndex:5]; + }); + } + + void apply_gradient(id pressure, id fluid_buffer, + id active_buffer, id output, + const MetalFlowGridParameters& grid) const { + dispatch(gradient, grid.total_face_count, "Metal resolved-flow gradient failed", + [&](id encoder) { + [encoder setBuffer:pressure offset:0 atIndex:0]; + [encoder setBuffer:fluid_buffer offset:0 atIndex:1]; + [encoder setBuffer:active_buffer offset:0 atIndex:2]; + [encoder setBuffer:output offset:0 atIndex:3]; + [encoder setBytes:&grid length:sizeof(grid) atIndex:4]; + }); + } + + void apply_divergence(id velocity, id fluid_buffer, id output, + const MetalFlowGridParameters& grid) const { + dispatch(divergence, grid.site_count, "Metal resolved-flow divergence failed", + [&](id encoder) { + [encoder setBuffer:velocity offset:0 atIndex:0]; + [encoder setBuffer:fluid_buffer offset:0 atIndex:1]; + [encoder setBuffer:output offset:0 atIndex:2]; + [encoder setBytes:&grid length:sizeof(grid) atIndex:3]; + }); + } + + void negate(id input, id output, std::uint32_t count) const { + dispatch(vector_negate, count, "Metal flow vector negation failed", + [&](id encoder) { + [encoder setBuffer:input offset:0 atIndex:0]; + [encoder setBuffer:output offset:0 atIndex:1]; + [encoder setBytes:&count length:sizeof(count) atIndex:2]; + }); + } + + void subtract(id left, id right, id output, + std::uint32_t count) const { + dispatch(vector_subtract, count, "Metal flow vector subtraction failed", + [&](id encoder) { + [encoder setBuffer:left offset:0 atIndex:0]; + [encoder setBuffer:right offset:0 atIndex:1]; + [encoder setBuffer:output offset:0 atIndex:2]; + [encoder setBytes:&count length:sizeof(count) atIndex:3]; + }); + } + + id device; + id queue; + id depth_operator; + id depth_velocity; + id momentum; + id gradient; + id divergence; + id pcg_initialize; + id pcg_update; + id pcg_precondition; + id pcg_direction; + id vector_negate; + id vector_subtract; + id dot_partial; + static constexpr std::uint32_t reduction_width = 64; +}; + +FlowSolver::FlowSolver(std::uint32_t device_index) : impl_(std::make_unique(device_index)) {} + +FlowSolver::~FlowSolver() = default; + +DepthAveragedFlowResult FlowSolver::solve_depth_averaged( + const SignalGridSpec& spec, std::span mobility, + const DepthAveragedFlowParameters& parameters) { + parameters.validate(); + const detail::DepthAveragedFlowSystem system(spec, mobility, parameters.axis); + const auto grid = make_grid_parameters(system.layout()); + const auto mobility_buffer = impl_->upload(system.mobility(), "depth mobility"); + const auto diagonal_buffer = impl_->upload(system.diagonal(), "depth diagonal"); + const auto rhs_buffer = impl_->upload(system.right_hand_side(), "depth right-hand side"); + const auto pressure_buffer = impl_->float_buffer(grid.site_count, "depth pressure"); + const auto velocity_buffer = impl_->float_buffer(grid.total_face_count, "depth velocity"); + auto workspace = impl_->make_workspace(grid.site_count, "depth"); + const auto report = + impl_->solve_pcg(rhs_buffer, diagonal_buffer, pressure_buffer, workspace, grid.site_count, + parameters.relative_tolerance, parameters.max_iterations, + "Metal depth-averaged flow", [&](id input, id output) { + impl_->apply_depth(input, mobility_buffer, diagonal_buffer, output, grid); + }); + impl_->dispatch(impl_->depth_velocity, grid.total_face_count, + "Metal depth-averaged velocity reconstruction failed", + [&](id encoder) { + [encoder setBuffer:pressure_buffer offset:0 atIndex:0]; + [encoder setBuffer:mobility_buffer offset:0 atIndex:1]; + [encoder setBuffer:velocity_buffer offset:0 atIndex:2]; + [encoder setBytes:&grid length:sizeof(grid) atIndex:3]; + }); + const auto* velocity_values = static_cast(velocity_buffer.contents); + const std::span velocity(velocity_values, grid.total_face_count); + const auto scaled = detail::scale_velocity( + spec, system.layout(), velocity, system.open_inlet_faces(), parameters.mean_inlet_speed); + return { + .field = scaled.field, + .report = {.iterations = report.iterations, + .relative_residual = report.relative_residual, + .mean_inlet_speed = parameters.mean_inlet_speed, + .max_speed = scaled.max_speed}, + }; +} + +ResolvedFlowResult FlowSolver::solve_resolved(const SignalGridSpec& spec, + std::span drag, + const ResolvedFlowParameters& parameters) { + parameters.validate(); + const detail::ResolvedFlowSystem system(spec, drag, parameters.axis); + const auto grid = make_grid_parameters(system.layout()); + const auto fluid_buffer = impl_->upload(system.fluid(), "fluid mask"); + const auto active_buffer = impl_->upload(system.active(), "active face mask"); + const auto exists_buffer = impl_->upload(system.exists(), "face existence mask"); + const auto face_drag_buffer = impl_->upload(system.face_drag(), "face drag"); + const auto face_diagonal_buffer = impl_->upload(system.diagonal(), "momentum diagonal"); + const auto force_buffer = impl_->upload(system.force(), "momentum force"); + const auto pressure_diagonal = system.pressure_diagonal(); + const auto pressure_diagonal_buffer = + impl_->upload(pressure_diagonal, "pressure diagonal"); + + const auto particular = impl_->float_buffer(grid.total_face_count, "particular velocity"); + const auto schur_rhs = impl_->float_buffer(grid.site_count, "pressure right-hand side"); + const auto pressure = impl_->float_buffer(grid.site_count, "pressure"); + const auto gradient = impl_->float_buffer(grid.total_face_count, "pressure gradient"); + const auto response = impl_->float_buffer(grid.total_face_count, "momentum response"); + const auto correction = impl_->float_buffer(grid.total_face_count, "velocity correction"); + const auto velocity = impl_->float_buffer(grid.total_face_count, "resolved velocity"); + const auto divergence = impl_->float_buffer(grid.site_count, "velocity divergence"); + auto inner_workspace = impl_->make_workspace(grid.total_face_count, "resolved momentum"); + auto outer_workspace = impl_->make_workspace(grid.site_count, "resolved pressure"); + + std::uint64_t inner_iterations = 0; + const auto solve_momentum = [&](id rhs, id solution) { + const auto result = impl_->solve_pcg( + rhs, face_diagonal_buffer, solution, inner_workspace, grid.total_face_count, + parameters.inner_relative_tolerance, parameters.max_inner_iterations, + "Metal resolved-flow momentum", [&](id input, id output) { + impl_->apply_momentum(input, active_buffer, exists_buffer, face_drag_buffer, output, + grid); + }); + if (inner_iterations > std::numeric_limits::max() - result.iterations) { + throw std::overflow_error("resolved-flow inner iteration count overflow"); + } + inner_iterations += result.iterations; + }; + + solve_momentum(force_buffer, particular); + impl_->apply_divergence(particular, fluid_buffer, schur_rhs, grid); + impl_->negate(schur_rhs, schur_rhs, grid.site_count); + const auto pressure_report = impl_->solve_pcg( + schur_rhs, pressure_diagonal_buffer, pressure, outer_workspace, grid.site_count, + parameters.relative_tolerance, parameters.max_outer_iterations, + "Metal resolved-flow pressure", [&](id input, id output) { + impl_->apply_gradient(input, fluid_buffer, active_buffer, gradient, grid); + solve_momentum(gradient, response); + impl_->apply_divergence(response, fluid_buffer, output, grid); + impl_->negate(output, output, grid.site_count); + }); + impl_->apply_gradient(pressure, fluid_buffer, active_buffer, gradient, grid); + solve_momentum(gradient, correction); + impl_->subtract(particular, correction, velocity, grid.total_face_count); + impl_->apply_divergence(velocity, fluid_buffer, divergence, grid); + + const auto* divergence_values = static_cast(divergence.contents); + double divergence_square_sum = 0.0; + std::size_t fluid_count = 0; + for (std::size_t site = 0; site < system.fluid().size(); ++site) { + if (system.fluid()[site] != 0) { + const auto value = static_cast(divergence_values[site]); + divergence_square_sum += value * value; + ++fluid_count; + } + } + const auto divergence_rms = + fluid_count == 0 ? 0.0 : std::sqrt(divergence_square_sum / static_cast(fluid_count)); + const auto* velocity_values = static_cast(velocity.contents); + const std::span velocity_span(velocity_values, grid.total_face_count); + const auto scaled = detail::scale_velocity( + spec, system.layout(), velocity_span, system.open_inlet_faces(), parameters.mean_inlet_speed); + return { + .field = scaled.field, + .report = {.outer_iterations = pressure_report.iterations, + .inner_iterations = inner_iterations, + .divergence_rms = static_cast(divergence_rms * std::abs(scaled.factor)), + .mean_inlet_speed = parameters.mean_inlet_speed, + .max_speed = scaled.max_speed, + .min_gap_voxels = system.minimum_gap_voxels()}, + }; +} + +} // namespace cm::metal diff --git a/cpp/python/bindings.cpp b/cpp/python/bindings.cpp index 6713823..33dc5b7 100644 --- a/cpp/python/bindings.cpp +++ b/cpp/python/bindings.cpp @@ -27,7 +27,14 @@ NB_MODULE(_core, module) { .value("CELL_MECHANICS", cm::BackendFeature::cell_mechanics) .value("EXTERNAL_CONSTRAINTS", cm::BackendFeature::external_constraints) .value("SIGNALS", cm::BackendFeature::signals) - .value("COUPLED_RATES", cm::BackendFeature::coupled_rates); + .value("COUPLED_RATES", cm::BackendFeature::coupled_rates) + .value("DEPTH_AVERAGED_FLOW", cm::BackendFeature::depth_averaged_flow) + .value("RESOLVED_FLOW", cm::BackendFeature::resolved_flow); + + nb::enum_(module, "FlowAxis") + .value("X", cm::FlowAxis::x) + .value("Y", cm::FlowAxis::y) + .value("Z", cm::FlowAxis::z); nb::enum_(module, "GridBoundaryKind") .value("NO_FLUX", cm::GridBoundaryKind::no_flux) @@ -145,6 +152,49 @@ NB_MODULE(_core, module) { .def_rw("y_faces", &cm::SignalGridVelocityField::y_faces) .def_rw("z_faces", &cm::SignalGridVelocityField::z_faces); + nb::class_(module, "DepthAveragedFlowParameters") + .def(nb::init<>()) + .def_rw("mean_inlet_speed", &cm::DepthAveragedFlowParameters::mean_inlet_speed) + .def_rw("axis", &cm::DepthAveragedFlowParameters::axis) + .def_rw("relative_tolerance", &cm::DepthAveragedFlowParameters::relative_tolerance) + .def_rw("max_iterations", &cm::DepthAveragedFlowParameters::max_iterations) + .def("validate", &cm::DepthAveragedFlowParameters::validate); + + nb::class_(module, "DepthAveragedFlowReport") + .def_ro("iterations", &cm::DepthAveragedFlowReport::iterations) + .def_prop_ro( + "residual", + [](const cm::DepthAveragedFlowReport& report) { return report.relative_residual; }) + .def_ro("relative_residual", &cm::DepthAveragedFlowReport::relative_residual) + .def_ro("mean_inlet_speed", &cm::DepthAveragedFlowReport::mean_inlet_speed) + .def_ro("max_speed", &cm::DepthAveragedFlowReport::max_speed); + + nb::class_(module, "DepthAveragedFlowResult") + .def_ro("field", &cm::DepthAveragedFlowResult::field) + .def_ro("report", &cm::DepthAveragedFlowResult::report); + + nb::class_(module, "ResolvedFlowParameters") + .def(nb::init<>()) + .def_rw("mean_inlet_speed", &cm::ResolvedFlowParameters::mean_inlet_speed) + .def_rw("axis", &cm::ResolvedFlowParameters::axis) + .def_rw("relative_tolerance", &cm::ResolvedFlowParameters::relative_tolerance) + .def_rw("max_outer_iterations", &cm::ResolvedFlowParameters::max_outer_iterations) + .def_rw("inner_relative_tolerance", &cm::ResolvedFlowParameters::inner_relative_tolerance) + .def_rw("max_inner_iterations", &cm::ResolvedFlowParameters::max_inner_iterations) + .def("validate", &cm::ResolvedFlowParameters::validate); + + nb::class_(module, "ResolvedFlowReport") + .def_ro("outer_iterations", &cm::ResolvedFlowReport::outer_iterations) + .def_ro("inner_iterations", &cm::ResolvedFlowReport::inner_iterations) + .def_ro("divergence_rms", &cm::ResolvedFlowReport::divergence_rms) + .def_ro("mean_inlet_speed", &cm::ResolvedFlowReport::mean_inlet_speed) + .def_ro("max_speed", &cm::ResolvedFlowReport::max_speed) + .def_ro("min_gap_voxels", &cm::ResolvedFlowReport::min_gap_voxels); + + nb::class_(module, "ResolvedFlowResult") + .def_ro("field", &cm::ResolvedFlowResult::field) + .def_ro("report", &cm::ResolvedFlowResult::report); + nb::class_(module, "SignalGridSpec") .def(nb::init<>()) .def_rw("signal_count", &cm::SignalGridSpec::signal_count) @@ -504,6 +554,22 @@ NB_MODULE(_core, module) { "contact_parameters"_a = cm::ContactParameters{}, "integration_parameters"_a = cm::MechanicsIntegrationParameters{}, "constraint_parameters"_a = cm::ConstraintContactParameters{}) + .def( + "solve_depth_averaged_flow", + [](cm::Simulation& simulation, const cm::SignalGridSpec& spec, + const std::vector& mobility, + const cm::DepthAveragedFlowParameters& parameters) { + return simulation.solve_depth_averaged_flow(spec, mobility, parameters); + }, + "spec"_a, "mobility"_a = std::vector{}, + "parameters"_a = cm::DepthAveragedFlowParameters{}) + .def( + "solve_resolved_flow", + [](cm::Simulation& simulation, const cm::SignalGridSpec& spec, + const std::vector& drag, const cm::ResolvedFlowParameters& parameters) { + return simulation.solve_resolved_flow(spec, drag, parameters); + }, + "spec"_a, "drag"_a = std::vector{}, "parameters"_a = cm::ResolvedFlowParameters{}) .def("cell", &cm::Simulation::cell, "id"_a) .def("cells", &cm::Simulation::cells) .def("lineage_parent", &cm::Simulation::lineage_parent, "id"_a) diff --git a/docs/architecture/0022-brinkman-flow.md b/docs/architecture/0022-brinkman-flow.md index 1fbd60e..5e2cbf7 100644 --- a/docs/architecture/0022-brinkman-flow.md +++ b/docs/architecture/0022-brinkman-flow.md @@ -2,15 +2,11 @@ - Status: accepted - Date: 2026-08-16 +- Amended: 2026-08-29 ## Context -Device flow fields are authored analytically, which is exact only for straight channels. A -junction, bend, pillar array, or partially blocking colony needs a numerical solve. At -microfluidic scale the Reynolds number is around `1e-4`, so the governing momentum balance is -inertia-free and linear, and for a fixed geometry the flow is steady: it can be computed once -in the authoring layer and handed to the engine as the existing face-staggered velocity -field, with no fluid solver in the simulation loop. +Analytic velocity profiles are exact only for simple channels. A junction, bend, pillar array, imported mask, or partially blocking colony requires a numerical solve over the authored device geometry. At microfluidic scales the governing momentum balance is commonly inertia-free and linear, so the flow for a fixed geometry is steady. The simulation nevertheless needs to select the implementation: initial device assembly and any later colony-coupled re-solve must execute through the same CPU, Metal, or CUDA backend chosen for the rest of the model. ## Decision @@ -30,24 +26,13 @@ the model and its validation gate. The in-plane viscous term is deliberately dro wall boundary layers, whose thickness is on the order of the gap height, are not resolved. A full staggered-grid Stokes solve is the named refinement if a study needs them. -Pressure is fixed on the fluid boundary faces of one axis - inlet one, outlet zero - and -every other exterior face carries no flux. The discrete operator is symmetric positive -definite and is solved matrix-free by Jacobi-preconditioned conjugate gradient in NumPy; no -new dependency is added. The face velocities are the discrete fluxes of the solved pressure, -so per-voxel mass conservation and zero velocity on closed faces hold by construction, and -the result passes the engine's velocity-field validation unchanged. Because the problem is -linear, the solved field is rescaled to a requested mean inlet speed, so callers never handle -pressure or viscosity units. A grid whose inlet is entirely blocked, or which declares -periodic boundaries, is an error. - -`colony_mobility` builds the Brinkman drag field from cell state: each cell's volume -accumulates into its center voxel, the resulting volume fraction sets a Kozeny-Carman style -drag `phi^2 / (1 - phi)^3` scaled by a model-chosen coefficient, and resistances add to the -base mobility. The closure coefficient is a modeling choice, not a measured constant, and is -documented as such. Binning a whole capsule into its center voxel is a nearest-voxel -rasterization: a cell longer than a voxel contributes entirely to one of the voxels it -spans, so the volume fraction, and the drag field with it, is noisier than the colony at -spacings comparable to a cell. +Pressure is fixed on the fluid boundary faces of one axis, with inlet pressure one and outlet pressure zero, while every other exterior face carries no flux. For neighboring fluid sites `i` and `j`, the face coefficient is the harmonic mean `m_ij = 2 m_i m_j / (m_i + m_j)` divided by the squared center spacing. Fixed pressure boundaries use the corresponding half-cell coefficient `2 m_i / h^2`. Summing these coefficients gives the Jacobi diagonal and the inlet contribution gives the right-hand side. The resulting symmetric positive-definite system is solved matrix-free by Jacobi-preconditioned conjugate gradient. + +The solve is a domain operation on `ComputeBackend` and `Simulation`. C++ implements the readable CPU reference, MSL implements the Metal operator, Krylov vector updates, and reductions, and CUDA C++ implements the CUDA equivalents. Device vectors remain on the selected accelerator throughout each solve; the host receives reduction partials needed for convergence control and the final face field. No accelerator backend calls the CPU reference. The portable field and solver contract is binary32, with a default relative residual tolerance of `1e-6`. + +The reconstructed face velocities are the discrete fluxes of the solved pressure, so per-voxel mass conservation and zero velocity on closed faces hold by construction. Because the problem is linear, the field is rescaled to a requested mean inlet speed. A grid whose inlet is entirely blocked, whose outlet is unreachable, or which declares periodic boundaries is rejected. + +Python remains the device-authoring surface. `gap_mobility` converts a solid mask into a gap-height mobility and `colony_mobility` rasterizes cell volume into a Kozeny-Carman-style resistance field. These helpers construct backend-neutral dense input arrays; they do not solve the pressure system. The closure coefficient is a modeling choice, not a measured constant. Binning a whole capsule into its center voxel is a nearest-voxel approximation, so mobility becomes noisy when the grid spacing approaches a cell length. For colony feedback the field must change mid-run, so the engine adds one mutation: `Simulation.set_velocity_field` validates a replacement field against the full grid @@ -66,10 +51,9 @@ uses whichever field is current. Model code chooses the re-solve cadence. ## Consequences -- Arbitrary mask geometry, including CAD-derived layouts, gets a conservative flow field - from one build-time solve. -- Colony blockage feeds back on flow at a model-chosen cadence without any native fluid - solver. +- Arbitrary mask geometry, including CAD-derived layouts, gets a conservative flow field from a native backend solve. +- Colony blockage feeds back on flow at a model-chosen cadence through the already selected backend. +- The solver, not only downstream transport, can therefore use CUDA or Metal acceleration. - In-plane boundary layers are the stated accuracy limit of the closure. - The solved field is a depth-averaged velocity: every voxel in a column carries the column's mean. Advection of signals stays conservative, but a cell drifting near a floor diff --git a/docs/architecture/0023-mac-stokes.md b/docs/architecture/0023-mac-stokes.md index 24e1307..1f97619 100644 --- a/docs/architecture/0023-mac-stokes.md +++ b/docs/architecture/0023-mac-stokes.md @@ -2,6 +2,7 @@ - Status: accepted - Date: 2026-08-18 +- Amended: 2026-08-29 ## Context @@ -34,19 +35,15 @@ drops out; the Brinkman drag field is an inverse permeability (`colony_drag` builds it from the colony's volume fraction). Collapsed axes are invariant directions, matching engine transport semantics. -The saddle-point system is solved through the pressure Schur complement -`S = D A^-1 D^T`, symmetric positive definite, by outer conjugate gradient -with three independent inner component-Laplacian conjugate gradient solves per -application - matrix-free NumPy throughout, no new dependency. The cost sits -well above the Hele-Shaw solve, which remains the default for device authoring -and the in-model re-solve cadence; the MAC solver is for resolved studies and -for anchoring the closure. +Write the discrete momentum equation as `A v + G p = f` and incompressibility as `D v = 0`, where `G = -D^T` under the declared face and cell inner products. Eliminating velocity gives the positive pressure Schur system `(-D A^-1 G) p = -D A^-1 f`. An outer Jacobi-preconditioned conjugate-gradient solve applies this operator matrix-free; each application invokes an inner Jacobi-preconditioned conjugate-gradient solve for the block-diagonal face momentum operator. The three component blocks are stored in one concatenated face vector, which preserves their mathematical independence while allowing one backend-native Krylov operation. + +Resolved flow is a `ComputeBackend` domain operation. The CPU implementation evaluates the same operators in C++, while Metal and CUDA keep pressure, velocity, Krylov work vectors, gradients, and divergences in device memory and execute independent MSL and CUDA kernels. The host controls the iteration from reduced scalar data and downloads the final velocity and divergence report. Neither accelerator implementation calls the CPU solver. The portable field contract is binary32 and both outer and inner relative tolerances default to `1e-6`. + +The cost remains above the depth-averaged solve, but it is no longer restricted to a Python build-time calculation. Models can execute either solver on their selected backend, including a resolved re-solve when that cost is justified. ## Validation -`scripts/run_flow_benchmarks.py` runs both solvers against literature and -exact references and fails nonzero on any tolerance miss; `test_stokes.py` -enforces the same physics at test sizes. +`scripts/run_flow_benchmarks.py` runs both solvers through an explicitly selected backend against literature and exact references and fails nonzero on any tolerance miss; `test_stokes.py` enforces the same physics at test sizes. The shared C++ `flow_conformance` scenario separately compares every available native backend with the CPU reference for heterogeneous mobility and Brinkman drag. - Plane Poiseuille: exact parabola, observed convergence order 2. The duct peak is interpolated to the centerline, since cell centers straddle the axis @@ -69,8 +66,7 @@ enforces the same physics at test sizes. ## Consequences -- Resolved wall shear and cross-channel profiles are available where a study - needs them, at build-time cost. +- Resolved wall shear and cross-channel profiles are available where a study needs them on CPU, Metal, and CUDA. - The Hele-Shaw closure's domain of validity is now measured, not asserted. - Resolution bounds the MAC solve as the closure bounds the depth-averaged one. Every solve reports `min_gap_voxels`, the fluid voxels across its diff --git a/docs/development/validation.md b/docs/development/validation.md index 210c4f2..acee6d6 100644 --- a/docs/development/validation.md +++ b/docs/development/validation.md @@ -19,7 +19,7 @@ Passing compilation is necessary but not sufficient. GPU conformance always mean ## Backend contract -Every test-enabled build runs the shared scenarios against every enumerated device compiled into that build. `backend_contract_conformance` requires each constructed device to advertise growth, species, contacts, mechanics, constraints, signals, and coupled rates. Capability guards in individual tests may help diagnose partial development builds, but they cannot turn a missing capability into a green complete-backend result. +Every test-enabled build runs the shared scenarios against every enumerated device compiled into that build. `backend_contract_conformance` requires each constructed device to advertise growth, species, contacts, mechanics, constraints, signals, coupled rates, depth-averaged flow, and resolved flow. Capability guards in individual tests may help diagnose partial development builds, but they cannot turn a missing capability into a green complete-backend result. `trajectory_conformance` composes coupled rates and transport, contact and constraint geometry, fixed-cell mechanics, integration, and division over three steps. It catches cross-feature errors that isolated one-step tests cannot expose. The exact scenarios, problem sizes, and numerical tolerances are maintained in the [conformance test reference](../../tests/conformance/README.md). @@ -39,6 +39,7 @@ Together, these gates cover: - growth, equal and asymmetric division, stable identity, and lineage; - cell contacts, plane, sphere, box, and cylinder constraints, full-capsule finite-obstacle contact, fixed cells, and mechanics relaxation; - species, signal transport, Forward Euler, Crank-Nicolson, and coupled rates; +- depth-averaged Darcy-Brinkman flow and resolved MAC Stokes-Brinkman flow; - checkpoint migration, exact controller resume, and deterministic runtime random state; - batch execution, stopping rules, output collision handling, and run manifests; - scene capture, live-viewer reset and checkpoint behavior, and protocol validation; diff --git a/python/src/cellmodeller2/__init__.py b/python/src/cellmodeller2/__init__.py index 7069102..597d264 100644 --- a/python/src/cellmodeller2/__init__.py +++ b/python/src/cellmodeller2/__init__.py @@ -15,9 +15,13 @@ ContactParameters, CoupledRatePlan, CylinderConstraintInit, + DepthAveragedFlowParameters, + DepthAveragedFlowReport, + DepthAveragedFlowResult, ExternalConstraintKind, ExternalContact, ExternalContactGraph, + FlowAxis, GridBoundary, GridBoundaryKind, GridShape, @@ -27,6 +31,9 @@ PlaneConstraintInit, RateInstruction, RateOp, + ResolvedFlowParameters, + ResolvedFlowReport, + ResolvedFlowResult, RodContactLocation, RodEndpoint, SignalGridAffineReaction, @@ -168,12 +175,16 @@ "ControllerStep", "CoupledRatePlan", "CylinderConstraintInit", + "DepthAveragedFlowParameters", + "DepthAveragedFlowReport", + "DepthAveragedFlowResult", "DivisionCallback", "DivisionEvent", "DivisionRequest", "ExternalConstraintKind", "ExternalContact", "ExternalContactGraph", + "FlowAxis", "GridBoundary", "GridBoundaryKind", "GridShape", @@ -199,6 +210,9 @@ "RatePlanBuilder", "RatePlanError", "RegulationCallback", + "ResolvedFlowParameters", + "ResolvedFlowReport", + "ResolvedFlowResult", "RodContactLocation", "RodEndpoint", "RunJob", diff --git a/python/src/cellmodeller2/_core.pyi b/python/src/cellmodeller2/_core.pyi index 8fb910c..fffd835 100644 --- a/python/src/cellmodeller2/_core.pyi +++ b/python/src/cellmodeller2/_core.pyi @@ -14,6 +14,13 @@ class BackendFeature(Enum): EXTERNAL_CONSTRAINTS: BackendFeature SIGNALS: BackendFeature COUPLED_RATES: BackendFeature + DEPTH_AVERAGED_FLOW: BackendFeature + RESOLVED_FLOW: BackendFeature + +class FlowAxis(Enum): + X: FlowAxis + Y: FlowAxis + Z: FlowAxis class GridBoundaryKind(Enum): NO_FLUX: GridBoundaryKind @@ -150,6 +157,64 @@ class SignalGridVelocityField: def __init__(self) -> None: ... +class DepthAveragedFlowParameters: + mean_inlet_speed: float + axis: FlowAxis + relative_tolerance: float + max_iterations: int + + def __init__(self) -> None: ... + def validate(self) -> None: ... + +class DepthAveragedFlowReport: + @property + def iterations(self) -> int: ... + @property + def residual(self) -> float: ... + @property + def relative_residual(self) -> float: ... + @property + def mean_inlet_speed(self) -> float: ... + @property + def max_speed(self) -> float: ... + +class DepthAveragedFlowResult: + @property + def field(self) -> SignalGridVelocityField: ... + @property + def report(self) -> DepthAveragedFlowReport: ... + +class ResolvedFlowParameters: + mean_inlet_speed: float + axis: FlowAxis + relative_tolerance: float + max_outer_iterations: int + inner_relative_tolerance: float + max_inner_iterations: int + + def __init__(self) -> None: ... + def validate(self) -> None: ... + +class ResolvedFlowReport: + @property + def outer_iterations(self) -> int: ... + @property + def inner_iterations(self) -> int: ... + @property + def divergence_rms(self) -> float: ... + @property + def mean_inlet_speed(self) -> float: ... + @property + def max_speed(self) -> float: ... + @property + def min_gap_voxels(self) -> int: ... + +class ResolvedFlowResult: + @property + def field(self) -> SignalGridVelocityField: ... + @property + def report(self) -> ResolvedFlowReport: ... + class SignalGridSpec: signal_count: int shape: GridShape @@ -550,6 +615,18 @@ class Simulation: integration_parameters: MechanicsIntegrationParameters = ..., constraint_parameters: ConstraintContactParameters = ..., ) -> MechanicsSolveResult: ... + def solve_depth_averaged_flow( + self, + spec: SignalGridSpec, + mobility: list[float] = ..., + parameters: DepthAveragedFlowParameters = ..., + ) -> DepthAveragedFlowResult: ... + def solve_resolved_flow( + self, + spec: SignalGridSpec, + drag: list[float] = ..., + parameters: ResolvedFlowParameters = ..., + ) -> ResolvedFlowResult: ... def cell(self, id: int) -> CellSnapshot: ... def cells(self) -> list[CellSnapshot]: ... def lineage_parent(self, id: int) -> int | None: ... diff --git a/python/src/cellmodeller2/flow.py b/python/src/cellmodeller2/flow.py index 98addfd..2b64b67 100644 --- a/python/src/cellmodeller2/flow.py +++ b/python/src/cellmodeller2/flow.py @@ -16,38 +16,44 @@ outlet zero) and every other exterior face carries no flux; the flow axis boundaries must therefore be `FIXED` and no axis may be periodic. The discrete operator is symmetric positive definite and is solved matrix-free with -Jacobi-preconditioned conjugate gradient. Side-wall boundary layers, whose -thickness is on the order of the gap height, are outside the closure. +Jacobi-preconditioned conjugate gradient by the CPU, Metal, or CUDA backend +selected for the simulation. Side-wall boundary layers, whose thickness is on +the order of the gap height, are outside the closure. """ from __future__ import annotations import math -from collections.abc import Callable, Iterable, Sequence -from dataclasses import dataclass +from collections.abc import Iterable, Sequence from typing import Protocol import numpy as np from numpy.typing import NDArray -from ._core import GridBoundaryKind, SignalGridSpec, SignalGridVelocityField, Vec3 +from ._core import ( # pyright: ignore[reportMissingModuleSource] + BackendKind, + DepthAveragedFlowParameters, + DepthAveragedFlowReport, + FlowAxis, + GridBoundaryKind, + SignalGridSpec, + SignalGridVelocityField, + Simulation, + Vec3, +) _FloatGrid = NDArray[np.float64] _BoolGrid = NDArray[np.bool_] _AXES = {"x": 0, "y": 1, "z": 2} +_NATIVE_AXES = {"x": FlowAxis.X, "y": FlowAxis.Y, "z": FlowAxis.Z} class FlowError(ValueError): """Raised when a flow problem is ill-posed or its solve fails.""" -@dataclass(frozen=True, slots=True) -class FlowSolveReport: - iterations: int - residual: float - mean_inlet_speed: float - max_speed: float +FlowSolveReport = DepthAveragedFlowReport class _RodLike(Protocol): @@ -59,89 +65,6 @@ def length(self) -> float: ... def radius(self) -> float: ... -def _slices(axis: int, along: slice) -> tuple[slice, slice, slice]: - index: list[slice] = [slice(None), slice(None), slice(None)] - index[axis] = along - return index[0], index[1], index[2] - - -def _mobility_grid(spec: SignalGridSpec, mobility: Sequence[float] | None) -> _FloatGrid: - dims = (spec.shape.x, spec.shape.y, spec.shape.z) - sites = dims[0] * dims[1] * dims[2] - if mobility is None: - values = np.ones(dims, dtype=np.float64) - else: - if len(mobility) != sites: - raise FlowError("mobility must hold one value per grid site") - values = np.asarray(mobility, dtype=np.float64).reshape(dims) - if not bool(np.all(np.isfinite(values))) or bool(np.any(values < 0.0)): - raise FlowError("mobility values must be finite and non-negative") - obstacles = spec.obstacles - if obstacles: - if len(obstacles) != sites: - raise FlowError("obstacles must hold one flag per grid site") - solid = np.asarray(obstacles, dtype=np.uint8).reshape(dims) != 0 - values[solid] = 0.0 - return values - - -def _harmonic_faces(mobility: _FloatGrid, axis: int, spacing: float) -> _FloatGrid: - lower = mobility[_slices(axis, slice(None, -1))] - upper = mobility[_slices(axis, slice(1, None))] - total = lower + upper - product = 2.0 * lower * upper - return np.divide( - product, - total, - out=np.zeros_like(total), - where=total > 0.0, - ) / (spacing * spacing) - - -def _conjugate_gradient( - apply_operator: Callable[[_FloatGrid], _FloatGrid], - rhs: _FloatGrid, - diagonal: _FloatGrid, - tolerance: float, - max_iterations: int, - *, - mask: _BoolGrid | None = None, - label: str = "flow", -) -> tuple[_FloatGrid, int, float]: - """Solve a symmetric positive definite system by Jacobi-preconditioned CG. - - ``mask`` restricts the solve to the sites the operator acts on; entries - outside it stay at zero. - """ - - solution = np.zeros_like(rhs) - residual = rhs.copy() if mask is None else np.where(mask, rhs, 0.0) - rhs_norm = float(np.sqrt(np.sum(residual * residual))) - if rhs_norm == 0.0: - return solution, 0, 0.0 - scale = np.where(diagonal > 0.0, diagonal, 1.0) - preconditioned = residual / scale - direction = preconditioned.copy() - rho = float(np.sum(residual * preconditioned)) - relative = 1.0 - for iteration in range(1, max_iterations + 1): - transformed = apply_operator(direction) - curvature = float(np.sum(direction * transformed)) - if curvature <= 0.0: - break - alpha = rho / curvature - solution += alpha * direction - residual -= alpha * transformed - relative = float(np.sqrt(np.sum(residual * residual))) / rhs_norm - if relative <= tolerance: - return solution, iteration, relative - preconditioned = residual / scale - next_rho = float(np.sum(residual * preconditioned)) - direction = preconditioned + (next_rho / rho) * direction - rho = next_rho - raise FlowError(f"{label} solve did not converge: relative residual {relative:.3e}") - - def _flow_axis_index(spec: SignalGridSpec, axis: str) -> int: """Validate a flow problem's axis and boundary kinds, and return the axis.""" @@ -176,96 +99,40 @@ def solve_flow_field( mean_inlet_speed: float, axis: str = "y", mobility: Sequence[float] | None = None, - tolerance: float = 1.0e-10, + tolerance: float = 1.0e-6, max_iterations: int = 50_000, + simulation: Simulation | None = None, + backend: BackendKind = BackendKind.CPU, + device_index: int = 0, ) -> tuple[SignalGridVelocityField, FlowSolveReport]: """Solve the device flow and return the face-staggered velocity field. Flow runs from the lower to the upper boundary of ``axis``; a negative ``mean_inlet_speed`` reverses it. The grid's shape, spacing, obstacles, and boundary kinds are read from ``spec``; ``mobility`` optionally gives - one relative mobility per site (default uniform, the Stokes limit). + one relative mobility per site (default uniform, the Stokes limit). If a + simulation is supplied, its native backend executes the solve. Otherwise + a temporary simulation uses ``backend`` and ``device_index``. """ - if not math.isfinite(mean_inlet_speed) or mean_inlet_speed == 0.0: - raise FlowError("mean inlet speed must be finite and nonzero") - flow_axis = _flow_axis_index(spec, axis) - - dims = (spec.shape.x, spec.shape.y, spec.shape.z) - spacing = (spec.spacing.x, spec.spacing.y, spec.spacing.z) - mobility_grid = _mobility_grid(spec, mobility) - - conductances = tuple( - _harmonic_faces(mobility_grid, index, spacing[index]) for index in range(3) + _flow_axis_index(spec, axis) + parameters = DepthAveragedFlowParameters() + parameters.mean_inlet_speed = mean_inlet_speed + parameters.axis = _NATIVE_AXES[axis] + parameters.relative_tolerance = tolerance + parameters.max_iterations = max_iterations + selected = ( + simulation if simulation is not None else Simulation(backend, device_index=device_index) ) - step = spacing[flow_axis] - inlet_conductance = 2.0 * mobility_grid[_slices(flow_axis, slice(0, 1))] / (step * step) - outlet_conductance = 2.0 * mobility_grid[_slices(flow_axis, slice(-1, None))] / (step * step) - if not bool(np.any(inlet_conductance > 0.0)): - raise FlowError("the inlet boundary is entirely blocked") - - diagonal = np.zeros(dims, dtype=np.float64) - for index in range(3): - diagonal[_slices(index, slice(1, None))] += conductances[index] - diagonal[_slices(index, slice(None, -1))] += conductances[index] - diagonal[_slices(flow_axis, slice(0, 1))] += inlet_conductance - diagonal[_slices(flow_axis, slice(-1, None))] += outlet_conductance - - def apply_operator(pressure: _FloatGrid) -> _FloatGrid: - result = diagonal * pressure - for index in range(3): - faces = conductances[index] - result[_slices(index, slice(1, None))] -= ( - faces * pressure[_slices(index, slice(None, -1))] - ) - result[_slices(index, slice(None, -1))] -= ( - faces * pressure[_slices(index, slice(1, None))] - ) - return result - - rhs = np.zeros(dims, dtype=np.float64) - rhs[_slices(flow_axis, slice(0, 1))] += inlet_conductance - pressure, iterations, residual = _conjugate_gradient( - apply_operator, rhs, diagonal, tolerance, max_iterations - ) - - face_grids: list[_FloatGrid] = [] - for index in range(3): - face_dims = list(dims) - face_dims[index] += 1 - faces = np.zeros(tuple(face_dims), dtype=np.float64) - gradient = ( - pressure[_slices(index, slice(1, None))] - pressure[_slices(index, slice(None, -1))] + try: + result = selected.solve_depth_averaged_flow( + spec, + [] if mobility is None else [float(value) for value in mobility], + parameters, ) - faces[_slices(index, slice(1, -1))] = -conductances[index] * spacing[index] * gradient - face_grids.append(faces) - inlet_faces = -inlet_conductance * step * (pressure[_slices(flow_axis, slice(0, 1))] - 1.0) - outlet_faces = outlet_conductance * step * pressure[_slices(flow_axis, slice(-1, None))] - face_grids[flow_axis][_slices(flow_axis, slice(0, 1))] = inlet_faces - face_grids[flow_axis][_slices(flow_axis, slice(-1, None))] = outlet_faces - - open_inlet = inlet_conductance > 0.0 - solved_mean = float(np.mean(inlet_faces[open_inlet])) - # The inlet must carry a real share of whatever the solve moved anywhere, - # which makes the test independent of mobility, spacing, and grid size. - peak = max(float(np.max(np.abs(faces))) for faces in face_grids) - if peak == 0.0 or solved_mean <= 1.0e-9 * peak: - raise FlowError("the device carries no through-flow: the outlet is unreachable") - factor = mean_inlet_speed / solved_mean - scaled = [faces * factor for faces in face_grids] - - field = SignalGridVelocityField() - field.x_faces = [float(value) for value in scaled[0].ravel()] - field.y_faces = [float(value) for value in scaled[1].ravel()] - field.z_faces = [float(value) for value in scaled[2].ravel()] - max_speed = max(float(np.max(np.abs(faces))) for faces in scaled) - report = FlowSolveReport( - iterations=iterations, - residual=residual, - mean_inlet_speed=mean_inlet_speed, - max_speed=max_speed, - ) - return field, report + except (OverflowError, RuntimeError, ValueError) as error: + raise FlowError(str(error)) from error + return result.field, result.report def gap_mobility(spec: SignalGridSpec) -> list[float]: diff --git a/python/src/cellmodeller2/microfluidics.py b/python/src/cellmodeller2/microfluidics.py index a6a3e23..bcf6968 100644 --- a/python/src/cellmodeller2/microfluidics.py +++ b/python/src/cellmodeller2/microfluidics.py @@ -19,6 +19,7 @@ from dataclasses import dataclass from ._core import ( # pyright: ignore[reportMissingModuleSource] + BackendKind, BoxConstraintInit, ConstraintRegion, GridBoundaryKind, @@ -89,8 +90,16 @@ def apply_to_grid( spec: SignalGridSpec, inlet_values: list[float], outlet_values: list[float], + *, + simulation: Simulation | None = None, + backend: BackendKind = BackendKind.CPU, + device_index: int = 0, ) -> None: - """Materialize the device's solid mask, solved flow field, and y inlet and outlet.""" + """Materialize the device's mask, boundaries, and backend-solved flow field. + + If a simulation is supplied, its native backend executes the flow solve. + Otherwise a temporary simulation uses ``backend`` and ``device_index``. + """ shape = spec.shape origin = spec.origin @@ -123,6 +132,9 @@ def apply_to_grid( mean_inlet_speed=self.mean_flow_speed, axis="y", mobility=gap_mobility(spec), + simulation=simulation, + backend=backend, + device_index=device_index, ) spec.velocity_field = field diff --git a/python/src/cellmodeller2/stokes.py b/python/src/cellmodeller2/stokes.py index 92adae7..fc94c87 100644 --- a/python/src/cellmodeller2/stokes.py +++ b/python/src/cellmodeller2/stokes.py @@ -26,55 +26,39 @@ The saddle-point system is solved by the pressure Schur complement: an outer conjugate gradient on `S = D A^-1 D^T` (symmetric positive definite), with each application solving three independent component Laplacians by inner -conjugate gradient. Everything is matrix-free NumPy. This costs far more than -the Hele-Shaw solve - it is the build-time and benchmark solver, not the -per-hundred-steps re-solve inside a running model. +conjugate gradient. The selected CPU, Metal, or CUDA backend executes the +matrix-free operator and Krylov iterations. This costs far more than the +Hele-Shaw solve. """ # pyright: reportPrivateUsage=false from __future__ import annotations -import math from collections.abc import Iterable, Sequence -from dataclasses import dataclass -from typing import TypeVar import numpy as np -from ._core import SignalGridSpec, SignalGridVelocityField +from ._core import ( # pyright: ignore[reportMissingModuleSource] + BackendKind, + FlowAxis, + ResolvedFlowParameters, + ResolvedFlowReport, + SignalGridSpec, + SignalGridVelocityField, + Simulation, +) from .flow import ( FlowError, - _BoolGrid, - _conjugate_gradient, - _FloatGrid, _flow_axis_index, _kozeny_carman_drag, _RodLike, colony_volume_fraction, ) -# Either a float field or a mask: shifting and reflecting treat them alike. -_Grid = TypeVar("_Grid", _FloatGrid, _BoolGrid) - +_NATIVE_AXES = {"x": FlowAxis.X, "y": FlowAxis.Y, "z": FlowAxis.Z} -@dataclass(frozen=True, slots=True) -class StokesSolveReport: - outer_iterations: int - inner_iterations: int - divergence_rms: float - mean_inlet_speed: float - max_speed: float - min_gap_voxels: int - """Fluid voxels across the narrowest channel, transverse to the flow. - - A no-slip profile needs several voxels to resolve, so this number bounds - the solve's accuracy: a channel one voxel across carries roughly two and a - half times the flux its true parabolic profile would, four voxels bring - that within about ten percent, and eight within a few percent. Below four, - the depth-averaged Hele-Shaw closure of `cellmodeller2.flow` is the more - accurate model of a shallow channel. - """ +StokesSolveReport = ResolvedFlowReport def colony_drag( @@ -102,307 +86,47 @@ def colony_drag( return [float(value) for value in drag.ravel()] -def _minimum_gap_voxels(fluid: _BoolGrid, dims: tuple[int, int, int], flow_axis: int) -> int: - """The shortest run of fluid voxels across any axis transverse to the flow.""" - - shortest = 0 - for axis in range(3): - if axis == flow_axis or dims[axis] <= 1: - continue - lines = np.moveaxis(fluid, axis, -1) - padded = np.zeros((*lines.shape[:-1], lines.shape[-1] + 2), dtype=np.int8) - padded[..., 1:-1] = lines - edges = np.diff(padded, axis=-1) - starts = np.argwhere(edges == 1) - ends = np.argwhere(edges == -1) - if starts.size == 0: - continue - runs = ends[:, -1] - starts[:, -1] - axis_shortest = int(runs.min()) - shortest = axis_shortest if shortest == 0 else min(shortest, axis_shortest) - return shortest - - -class _StokesOperator: - """The masked component Laplacians, divergence, and gradient of one problem.""" - - def __init__( - self, - dims: tuple[int, int, int], - spacing: tuple[float, float, float], - fluid: _BoolGrid, - drag: _FloatGrid, - flow_axis: int, - ) -> None: - self.dims = dims - self.spacing = spacing - self.fluid = fluid - # Collapsed axes (a single site) are invariant directions, matching - # the engine's transport semantics: no wall reflection across them. - self.live_axes = tuple(a for a in range(3) if dims[a] > 1) - - self.active: list[_BoolGrid] = [] - self.exists: list[_BoolGrid] = [] - self.face_drag: list[_FloatGrid] = [] - for c in range(3): - lower = self._cell_beside(fluid, c, -1) - upper = self._cell_beside(fluid, c, +1) - active = lower & upper - exists = lower | upper - if c == flow_axis: - # The flow-axis boundary faces carry prescribed ghost pressures - # rather than a wall, so they stay active beside one fluid cell. - active[self._edge_slice(c, 0)] = fluid[self._edge_slice(c, 0)] - active[self._edge_slice(c, -1)] = fluid[self._edge_slice(c, -1)] - self.active.append(active) - self.exists.append(exists) - drag_low = self._cell_beside_values(drag, c, -1) - drag_high = self._cell_beside_values(drag, c, +1) - counts = lower.astype(np.float64) + upper.astype(np.float64) - face_drag = np.divide( - drag_low + drag_high, - counts, - out=np.zeros_like(drag_low), - where=counts > 0.0, - ) - self.face_drag.append(face_drag) - - def _face_dims(self, c: int) -> tuple[int, int, int]: - dims = list(self.dims) - dims[c] += 1 - return dims[0], dims[1], dims[2] - - def _edge_slice(self, axis: int, edge: int) -> tuple[slice, slice, slice]: - index: list[slice] = [slice(None)] * 3 - index[axis] = slice(0, 1) if edge == 0 else slice(-1, None) - return index[0], index[1], index[2] - - def _cell_beside(self, cells: _BoolGrid, c: int, side: int) -> _BoolGrid: - """Whether the cell on ``side`` of each c-face exists and is fluid.""" - - face_dims = self._face_dims(c) - result = np.zeros(face_dims, dtype=bool) - target: list[slice] = [slice(None)] * 3 - target[c] = slice(1, None) if side < 0 else slice(0, -1) - result[target[0], target[1], target[2]] = cells - return result - - def _cell_beside_values(self, values: _FloatGrid, c: int, side: int) -> _FloatGrid: - face_dims = self._face_dims(c) - result = np.zeros(face_dims, dtype=np.float64) - target: list[slice] = [slice(None)] * 3 - target[c] = slice(1, None) if side < 0 else slice(0, -1) - result[target[0], target[1], target[2]] = values - return result - - def _shift(self, field: _Grid, axis: int, offset: int) -> _Grid: - """The field sampled at ``index + offset`` along ``axis``, zero beyond. - - Values past the array edge read as zero, and a mask shifted this way - reads as false, which is the wall the reflection tests look for. - """ - - result = np.zeros_like(field) - source: list[slice] = [slice(None)] * 3 - target: list[slice] = [slice(None)] * 3 - if offset > 0: - source[axis] = slice(1, None) - target[axis] = slice(0, -1) - else: - source[axis] = slice(0, -1) - target[axis] = slice(1, None) - result[target[0], target[1], target[2]] = field[source[0], source[1], source[2]] - return result - - def apply_momentum(self, c: int, u: _FloatGrid) -> _FloatGrid: - """``A u = -lap(u) + d u`` on active c-faces, zero elsewhere.""" - - active = self.active[c] - result = self.face_drag[c] * u - for a in self.live_axes: - inv_h2 = 1.0 / (self.spacing[a] * self.spacing[a]) - for offset in (-1, +1): - neighbor = self._shift(u, a, offset) - if a == c: - # Along the component axis neighbor faces hold genuine - # velocities (zero on walls). Active faces on the array - # edge - the flow-axis inlet and outlet - use a - # zero-gradient ghost equal to the face value. - edge = self._edge_slice(a, 0 if offset < 0 else -1) - ghost = np.zeros_like(u) - ghost[edge] = u[edge] - neighbor = neighbor + ghost - else: - # Across the component axis a neighbor location with no - # adjacent fluid cell lies inside a wall whose plane sits - # half a spacing away: reflect for no-slip. - reflect = ~self._shift(self.exists[c], a, offset) - neighbor = np.where(reflect, -u, neighbor) - result -= (neighbor - u) * inv_h2 - result[~active] = 0.0 - return result - - def divergence(self, velocity: list[_FloatGrid]) -> _FloatGrid: - result = np.zeros(self.dims, dtype=np.float64) - for c in range(3): - faces = velocity[c] - inv_h = 1.0 / self.spacing[c] - upper: list[slice] = [slice(None)] * 3 - lower: list[slice] = [slice(None)] * 3 - upper[c] = slice(1, None) - lower[c] = slice(0, -1) - result += ( - faces[upper[0], upper[1], upper[2]] - faces[lower[0], lower[1], lower[2]] - ) * inv_h - result[~self.fluid] = 0.0 - return result - - def gradient(self, pressure: _FloatGrid) -> list[_FloatGrid]: - """``-D^T p`` per component: the pressure gradient on active faces.""" - - fields: list[_FloatGrid] = [] - clean = np.where(self.fluid, pressure, 0.0) - for c in range(3): - face = np.zeros(self._face_dims(c), dtype=np.float64) - face += self._cell_beside_values(clean, c, +1) - face -= self._cell_beside_values(clean, c, -1) - face *= 1.0 / self.spacing[c] - face[~self.active[c]] = 0.0 - fields.append(face) - return fields - - def solve_stokes_field( spec: SignalGridSpec, *, mean_inlet_speed: float, axis: str = "y", drag: Sequence[float] | None = None, - tolerance: float = 1.0e-8, + tolerance: float = 1.0e-6, max_outer_iterations: int = 500, - inner_tolerance: float = 1.0e-10, + inner_tolerance: float = 1.0e-6, max_inner_iterations: int = 50_000, + simulation: Simulation | None = None, + backend: BackendKind = BackendKind.CPU, + device_index: int = 0, ) -> tuple[SignalGridVelocityField, StokesSolveReport]: """Solve the staggered Stokes-Brinkman flow and return the velocity field. Flow runs from the lower to the upper boundary of ``axis``; a negative ``mean_inlet_speed`` reverses it. ``drag`` optionally gives one Brinkman drag value (inverse permeability, units 1/length^2) per site; omitted or - zero drag is pure Stokes. + zero drag is pure Stokes. If a simulation is supplied, its native backend + executes the solve. Otherwise a temporary simulation uses ``backend`` and + ``device_index``. """ - if not math.isfinite(mean_inlet_speed) or mean_inlet_speed == 0.0: - raise FlowError("mean inlet speed must be finite and nonzero") - flow_axis = _flow_axis_index(spec, axis) - - dims = (spec.shape.x, spec.shape.y, spec.shape.z) - spacing = (spec.spacing.x, spec.spacing.y, spec.spacing.z) - sites = dims[0] * dims[1] * dims[2] - obstacles = spec.obstacles - if obstacles: - if len(obstacles) != sites: - raise FlowError("obstacles must hold one flag per grid site") - fluid = np.asarray(obstacles, dtype=np.uint8).reshape(dims) == 0 - else: - fluid = np.ones(dims, dtype=bool) - if drag is None: - drag_grid = np.zeros(dims, dtype=np.float64) - else: - if len(drag) != sites: - raise FlowError("drag must hold one value per grid site") - drag_grid = np.asarray(drag, dtype=np.float64).reshape(dims) - if not bool(np.all(np.isfinite(drag_grid))) or bool(np.any(drag_grid < 0.0)): - raise FlowError("drag values must be finite and non-negative") - drag_grid = np.where(fluid, drag_grid, 0.0) - - operator = _StokesOperator(dims, spacing, fluid, drag_grid, flow_axis) - if not bool(np.any(operator.active[flow_axis][operator._edge_slice(flow_axis, 0)])): - raise FlowError("the inlet boundary is entirely blocked") - - # Momentum right-hand side from the prescribed inlet and outlet ghost - # pressures (one and zero). - force: list[_FloatGrid] = [ - np.zeros(operator._face_dims(c), dtype=np.float64) for c in range(3) - ] - inlet_slice = operator._edge_slice(flow_axis, 0) - inlet_active = operator.active[flow_axis][inlet_slice] - force[flow_axis][inlet_slice] = np.where( - inlet_active, 1.0 / spacing[flow_axis], 0.0 - ) - - # Momentum diagonals for the inner Jacobi preconditioner. - diagonals: list[_FloatGrid] = [] - for c in range(3): - diagonal = operator.face_drag[c].copy() - for a in operator.live_axes: - diagonal += 2.0 / (spacing[a] * spacing[a]) - diagonals.append(diagonal) - - inner_total = 0 - - def solve_momentum(rhs: list[_FloatGrid]) -> list[_FloatGrid]: - nonlocal inner_total - solution: list[_FloatGrid] = [] - for c in range(3): - component, iterations, _ = _conjugate_gradient( - lambda u, c=c: operator.apply_momentum(c, u), - rhs[c], - diagonals[c], - inner_tolerance, - max_inner_iterations, - mask=operator.active[c], - label="stokes momentum", - ) - inner_total += iterations - solution.append(component) - return solution - - particular = solve_momentum(force) - schur_rhs = -operator.divergence(particular) - - def apply_schur(q: _FloatGrid) -> _FloatGrid: - # gradient() is -D^T, so negating the divergence gives S = D A^-1 D^T. - return -operator.divergence(solve_momentum(operator.gradient(q))) - - pressure, outer_iterations, _ = _conjugate_gradient( - apply_schur, - schur_rhs, - np.ones(dims, dtype=np.float64), - tolerance, - max_outer_iterations, - mask=fluid, - label="stokes pressure", - ) - - correction = solve_momentum(operator.gradient(pressure)) - velocity = [particular[c] - correction[c] for c in range(3)] - divergence = operator.divergence(velocity) - divergence_rms = float(np.sqrt(np.mean(divergence[fluid] ** 2))) if bool( - np.any(fluid) - ) else 0.0 - - inlet_values = velocity[flow_axis][inlet_slice] - open_inlet = operator.active[flow_axis][inlet_slice] - solved_mean = float(np.mean(inlet_values[open_inlet])) - # The inlet must carry a real share of whatever the solve moved anywhere, - # which makes the test independent of drag, spacing, and grid size. - peak = max(float(np.max(np.abs(component))) for component in velocity) - if peak == 0.0 or solved_mean <= 1.0e-9 * peak: - raise FlowError("the device carries no through-flow: the outlet is unreachable") - factor = mean_inlet_speed / solved_mean - scaled = [component * factor for component in velocity] - - field = SignalGridVelocityField() - field.x_faces = [float(value) for value in scaled[0].ravel()] - field.y_faces = [float(value) for value in scaled[1].ravel()] - field.z_faces = [float(value) for value in scaled[2].ravel()] - max_speed = max(float(np.max(np.abs(component))) for component in scaled) - report = StokesSolveReport( - outer_iterations=outer_iterations, - inner_iterations=inner_total, - divergence_rms=divergence_rms * abs(factor), - mean_inlet_speed=mean_inlet_speed, - max_speed=max_speed, - min_gap_voxels=_minimum_gap_voxels(fluid, dims, flow_axis), + _flow_axis_index(spec, axis) + parameters = ResolvedFlowParameters() + parameters.mean_inlet_speed = mean_inlet_speed + parameters.axis = _NATIVE_AXES[axis] + parameters.relative_tolerance = tolerance + parameters.max_outer_iterations = max_outer_iterations + parameters.inner_relative_tolerance = inner_tolerance + parameters.max_inner_iterations = max_inner_iterations + selected = ( + simulation if simulation is not None else Simulation(backend, device_index=device_index) ) - return field, report + try: + result = selected.solve_resolved_flow( + spec, + [] if drag is None else [float(value) for value in drag], + parameters, + ) + except (OverflowError, RuntimeError, ValueError) as error: + raise FlowError(str(error)) from error + return result.field, result.report diff --git a/python/tests/test_flow.py b/python/tests/test_flow.py index 7e603c5..f0649cf 100644 --- a/python/tests/test_flow.py +++ b/python/tests/test_flow.py @@ -5,6 +5,8 @@ import pytest from cellmodeller2 import ( + BackendFeature, + BackendKind, GridBoundaryKind, GridShape, SignalGridSpec, @@ -12,6 +14,7 @@ SignalIntegrationKind, Simulation, Vec3, + backend_available, ) from cellmodeller2.flow import FlowError, colony_mobility, gap_mobility, solve_flow_field from cellmodeller2.microfluidics import TrapChannelDevice @@ -56,23 +59,39 @@ def _cross_section_fluxes(spec: SignalGridSpec, field: SignalGridVelocityField) def test_uniform_duct_is_exact_plug_flow() -> None: spec = _duct() field, report = solve_flow_field(spec, mean_inlet_speed=5.0) - assert all(math.isclose(value, 5.0, abs_tol=1.0e-8) for value in field.y_faces) - assert all(abs(value) < 1.0e-8 for value in field.x_faces) - assert all(abs(value) < 1.0e-8 for value in field.z_faces) - assert math.isclose(report.max_speed, 5.0, rel_tol=1.0e-9) + assert all(math.isclose(value, 5.0, abs_tol=5.0e-5) for value in field.y_faces) + assert all(abs(value) < 2.0e-5 for value in field.x_faces) + assert all(abs(value) < 2.0e-5 for value in field.z_faces) + assert math.isclose(report.max_speed, 5.0, rel_tol=1.0e-5) spec.velocity_field = field spec.validate() +@pytest.mark.parametrize("backend", list(BackendKind)) +def test_depth_averaged_flow_uses_the_selected_native_backend(backend: BackendKind) -> None: + if not backend_available(backend): + pytest.skip(f"{backend.name} backend is unavailable") + spec = _duct(nx=3, ny=5, nz=1) + expected, _ = solve_flow_field(spec, mean_inlet_speed=2.0) + simulation = Simulation(backend) + assert simulation.supports(BackendFeature.DEPTH_AVERAGED_FLOW) + actual, report = solve_flow_field(spec, mean_inlet_speed=2.0, simulation=simulation) + assert report.residual <= 1.0e-6 + assert all( + math.isclose(observed, reference, abs_tol=5.0e-4, rel_tol=5.0e-4) + for observed, reference in zip(actual.y_faces, expected.y_faces, strict=True) + ) + + def test_parallel_channels_split_flux_in_the_mobility_ratio() -> None: spec = _duct(nx=2, ny=6, nz=1) mobility = [1.0 if x == 0 else 3.0 for x in range(2) for _ in range(6)] field, _ = solve_flow_field(spec, mean_inlet_speed=4.0, mobility=mobility) slow = field.y_faces[_y_face(spec, 0, 3, 0)] fast = field.y_faces[_y_face(spec, 1, 3, 0)] - assert math.isclose(fast / slow, 3.0, rel_tol=1.0e-6) - assert math.isclose((slow + fast) / 2.0, 4.0, rel_tol=1.0e-9) - assert all(abs(value) < 1.0e-8 for value in field.x_faces) + assert math.isclose(fast / slow, 3.0, rel_tol=1.0e-5) + assert math.isclose((slow + fast) / 2.0, 4.0, rel_tol=1.0e-6) + assert all(abs(value) < 2.0e-5 for value in field.x_faces) def test_a_pillar_routes_flow_around_itself_conservatively() -> None: @@ -302,7 +321,7 @@ def test_anisotropic_spacing_scales_the_solved_speeds() -> None: field, _ = solve_flow_field(spec, mean_inlet_speed=7.0) spec.velocity_field = field spec.validate() - assert all(math.isclose(value, 7.0, rel_tol=1.0e-8) for value in field.y_faces) + assert all(math.isclose(value, 7.0, rel_tol=1.0e-5) for value in field.y_faces) def test_reversed_and_transverse_flow_axes_solve() -> None: @@ -310,7 +329,7 @@ def test_reversed_and_transverse_flow_axes_solve() -> None: field, _ = solve_flow_field(spec, mean_inlet_speed=-3.0) spec.velocity_field = field spec.validate() - assert all(math.isclose(value, -3.0, abs_tol=1.0e-8) for value in field.y_faces) + assert all(math.isclose(value, -3.0, abs_tol=2.0e-5) for value in field.y_faces) across = _duct() for name in ("y_lower", "y_upper"): @@ -326,7 +345,7 @@ def test_reversed_and_transverse_flow_axes_solve() -> None: sideways, _ = solve_flow_field(across, mean_inlet_speed=2.0, axis="x") across.velocity_field = sideways across.validate() - assert all(math.isclose(value, 2.0, abs_tol=1.0e-8) for value in sideways.x_faces) + assert all(math.isclose(value, 2.0, abs_tol=2.0e-5) for value in sideways.x_faces) def test_partly_blocked_inlets_and_walled_off_pockets_solve() -> None: diff --git a/python/tests/test_stokes.py b/python/tests/test_stokes.py index 1216921..d261008 100644 --- a/python/tests/test_stokes.py +++ b/python/tests/test_stokes.py @@ -4,7 +4,14 @@ import numpy as np import pytest -from cellmodeller2 import GridBoundaryKind, Vec3 +from cellmodeller2 import ( + BackendFeature, + BackendKind, + GridBoundaryKind, + Simulation, + Vec3, + backend_available, +) from cellmodeller2.flow import FlowError, gap_mobility, solve_flow_field from cellmodeller2.flow_reference import ( SQUARE_DUCT_PEAK_TO_MEAN, @@ -19,7 +26,7 @@ def _plane_poiseuille_error(nx: int) -> float: spec = duct_grid(nx, 6, 1, (1.0 / nx, 0.25, 1.0)) - field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-10) + field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-6) profile = np.asarray(field.y_faces).reshape(nx, 7, 1)[:, 3, 0] positions = (np.arange(nx) + 0.5) / nx exact = plane_poiseuille(positions) @@ -34,15 +41,31 @@ def test_plane_poiseuille_profile_converges_at_second_order() -> None: assert 3.0 < coarse / fine < 5.0 +@pytest.mark.parametrize("backend", list(BackendKind)) +def test_resolved_flow_uses_the_selected_native_backend(backend: BackendKind) -> None: + if not backend_available(backend): + pytest.skip(f"{backend.name} backend is unavailable") + spec = duct_grid(6, 5, 1, (1.0 / 6.0, 0.25, 1.0)) + expected, _ = solve_stokes_field(spec, mean_inlet_speed=1.0) + simulation = Simulation(backend) + assert simulation.supports(BackendFeature.RESOLVED_FLOW) + actual, report = solve_stokes_field(spec, mean_inlet_speed=1.0, simulation=simulation) + assert report.divergence_rms < 2.0e-5 + assert all( + math.isclose(observed, reference, abs_tol=8.0e-4, rel_tol=8.0e-4) + for observed, reference in zip(actual.y_faces, expected.y_faces, strict=True) + ) + + def test_square_duct_peak_to_mean_matches_shah_and_london() -> None: # u_max / u_mean = 2.0962 for a square duct (Shah & London 1978). n = 16 spec = duct_grid(n, 6, n, (1.0 / n, 0.25, 1.0 / n)) - field, report = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + field, report = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-6) cross = np.asarray(field.y_faces).reshape(n, 7, n)[:, 3, :] ratio = centerline_value(cross) / float(cross.mean()) assert abs(ratio - SQUARE_DUCT_PEAK_TO_MEAN) / SQUARE_DUCT_PEAK_TO_MEAN < 0.015 - assert report.divergence_rms < 1.0e-6 + assert report.divergence_rms < 2.0e-5 def test_two_layer_brinkman_channel_matches_the_exact_solution() -> None: @@ -55,7 +78,7 @@ def test_two_layer_brinkman_channel_matches_the_exact_solution() -> None: for _ in range(6) for z in range(nz) ] - field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, drag=drag, tolerance=1.0e-9) + field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, drag=drag, tolerance=1.0e-6) profile = np.asarray(field.y_faces).reshape(1, 7, nz)[0, 3, :] positions = (np.arange(nz) + 0.5) / nz # The solve rescales to the requested mean speed, so both profiles are @@ -84,10 +107,10 @@ def test_stokes_field_is_engine_valid_and_conservative_around_a_pillar() -> None for x in (4, 5): obstacles[site_index(spec, x, y, 0)] = 1 spec.obstacles = obstacles - field, report = solve_stokes_field(spec, mean_inlet_speed=6.0, tolerance=1.0e-9) + field, report = solve_stokes_field(spec, mean_inlet_speed=6.0, tolerance=1.0e-6) spec.velocity_field = field spec.validate() - assert report.divergence_rms < 1.0e-6 + assert report.divergence_rms < 2.0e-5 def y_face(x: int, fy: int) -> float: return field.y_faces[x * 13 + fy] @@ -111,7 +134,7 @@ def test_thin_gap_stokes_depth_averages_to_the_hele_shaw_solution() -> None: obstacles[site_index(spec, x, y, z)] = 1 spec.obstacles = obstacles - stokes_field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + stokes_field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-6) hele_shaw_field, _ = solve_flow_field( spec, mean_inlet_speed=1.0, mobility=gap_mobility(spec) ) @@ -196,7 +219,7 @@ def test_thin_gaps_over_predict_flux_until_they_are_resolved() -> None: for y in range(8): obstacles[site_index(spec, 0, y, thin)] = 1 spec.obstacles = obstacles - field, report = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + field, report = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-6) profile = np.asarray(field.y_faces).reshape(1, 9, nz)[0, 4, :] ratio = float(profile[:thin].mean() / profile[thin + 1 :].mean()) errors.append(ratio / lubrication) @@ -242,14 +265,14 @@ def test_partly_blocked_inlets_and_walled_off_pockets_solve() -> None: for y in range(6): obstacles[site_index(spec, 0, y, 0)] = 1 spec.obstacles = obstacles - field, report = solve_stokes_field(spec, mean_inlet_speed=2.0, tolerance=1.0e-9) + field, report = solve_stokes_field(spec, mean_inlet_speed=2.0, tolerance=1.0e-6) spec.velocity_field = field spec.validate() inlet = np.asarray(field.y_faces).reshape(4, 7, 1)[:, 0, 0] # The mean is taken over open inlet faces, and the blocked column is still. assert math.isclose(float(inlet[1:].mean()), 2.0, rel_tol=1.0e-6) assert inlet[0] == 0.0 - assert report.divergence_rms < 1.0e-6 + assert report.divergence_rms < 2.0e-5 # A fluid site sealed off from the flow leaves the solve well posed. pocket = duct_grid(5, 6, 1, (1.0, 1.0, 1.0)) @@ -264,5 +287,5 @@ def test_partly_blocked_inlets_and_walled_off_pockets_solve() -> None: sealed_field, sealed_report = solve_stokes_field(pocket, mean_inlet_speed=1.0) pocket.velocity_field = sealed_field pocket.validate() - assert sealed_report.divergence_rms < 1.0e-6 + assert sealed_report.divergence_rms < 2.0e-5 assert sealed_field.y_faces[(3 * 7) + 3] == 0.0 diff --git a/scripts/run_flow_benchmarks.py b/scripts/run_flow_benchmarks.py index a3bb122..69aff23 100644 --- a/scripts/run_flow_benchmarks.py +++ b/scripts/run_flow_benchmarks.py @@ -11,7 +11,8 @@ `cellmodeller2.flow_reference`, so this script and the test suite measure the same physics. -Usage: uv run python scripts/run_flow_benchmarks.py [--fine] +Usage: uv run python scripts/run_flow_benchmarks.py + [--backend cpu|metal|cuda] [--device-index N] [--fine] `--fine` doubles every benchmark's resolution to demonstrate mesh convergence. """ @@ -24,6 +25,7 @@ from dataclasses import dataclass import numpy as np +from cellmodeller2 import BackendKind, Simulation, backend_available from cellmodeller2.flow import gap_mobility, solve_flow_field from cellmodeller2.flow_reference import ( SQUARE_DUCT_PEAK_TO_MEAN, @@ -59,13 +61,15 @@ def passed(self) -> bool: return self.error <= self.tolerance -def bench_plane_poiseuille_order(coarse: int) -> list[Result]: +def bench_plane_poiseuille_order(coarse: int, simulation: Simulation) -> list[Result]: results: list[Result] = [] errors: list[float] = [] for n in (coarse, coarse * 2): start = time.perf_counter() spec = duct_grid(n, 6, 1, (1.0 / n, 0.25, 1.0)) - field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-10) + field, _ = solve_stokes_field( + spec, mean_inlet_speed=1.0, tolerance=1.0e-6, simulation=simulation + ) profile = np.asarray(field.y_faces).reshape(n, 7, 1)[:, 3, 0] positions = (np.arange(n) + 0.5) / n exact = plane_poiseuille(positions) @@ -97,10 +101,12 @@ def bench_plane_poiseuille_order(coarse: int) -> list[Result]: return results -def bench_square_duct(n: int) -> Result: +def bench_square_duct(n: int, simulation: Simulation) -> Result: start = time.perf_counter() spec = duct_grid(n, 6, n, (1.0 / n, 0.25, 1.0 / n)) - field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + field, _ = solve_stokes_field( + spec, mean_inlet_speed=1.0, tolerance=1.0e-6, simulation=simulation + ) cross = np.asarray(field.y_faces).reshape(n, 7, n)[:, 3, :] # Cell centers straddle the duct axis, so the peak is interpolated rather # than taken from the largest sample, which would understate it. @@ -116,7 +122,7 @@ def bench_square_duct(n: int) -> Result: ) -def bench_two_layer_brinkman(coarse: int) -> list[Result]: +def bench_two_layer_brinkman(coarse: int, simulation: Simulation) -> list[Result]: drag_value = 200.0 results: list[Result] = [] errors: list[float] = [] @@ -129,7 +135,11 @@ def bench_two_layer_brinkman(coarse: int) -> list[Result]: for z in range(nz) ] field, _ = solve_stokes_field( - spec, mean_inlet_speed=1.0, drag=drag, tolerance=1.0e-9 + spec, + mean_inlet_speed=1.0, + drag=drag, + tolerance=1.0e-6, + simulation=simulation, ) profile = np.asarray(field.y_faces).reshape(1, 7, nz)[0, 3, :] positions = (np.arange(nz) + 0.5) / nz @@ -164,10 +174,10 @@ def bench_two_layer_brinkman(coarse: int) -> list[Result]: return results -def bench_hele_shaw_duct(scale: int) -> Result: +def bench_hele_shaw_duct(scale: int, simulation: Simulation) -> Result: start = time.perf_counter() spec = duct_grid(4 * scale, 8 * scale, 3 * scale, (1.0, 1.0, 1.0)) - field, _ = solve_flow_field(spec, mean_inlet_speed=5.0) + field, _ = solve_flow_field(spec, mean_inlet_speed=5.0, simulation=simulation) error = float(max(abs(v - 5.0) for v in field.y_faces)) return Result( "hele-shaw", @@ -175,19 +185,21 @@ def bench_hele_shaw_duct(scale: int) -> Result: "max |u - mean| (exact plug flow)", error, 0.0, - 1.0e-6, + 5.0e-5, time.perf_counter() - start, ) -def bench_hele_shaw_mobility_split(scale: int) -> Result: +def bench_hele_shaw_mobility_split(scale: int, simulation: Simulation) -> Result: start = time.perf_counter() columns, rows = 2 * scale, 6 * scale spec = duct_grid(columns, rows, 1, (1.0, 1.0, 1.0)) mobility = [ 1.0 if x < columns // 2 else 3.0 for x in range(columns) for _ in range(rows) ] - field, _ = solve_flow_field(spec, mean_inlet_speed=4.0, mobility=mobility) + field, _ = solve_flow_field( + spec, mean_inlet_speed=4.0, mobility=mobility, simulation=simulation + ) middle = rows // 2 slow = field.y_faces[0 * (rows + 1) + middle] fast = field.y_faces[(columns - 1) * (rows + 1) + middle] @@ -202,7 +214,7 @@ def bench_hele_shaw_mobility_split(scale: int) -> Result: ) -def bench_cross_solver_consistency(scale: int) -> Result: +def bench_cross_solver_consistency(scale: int, simulation: Simulation) -> Result: start = time.perf_counter() nx, ny, nz = 6 * scale, 10 * scale, 6 * scale spec = duct_grid(nx, ny, nz, (1.0 / scale, 1.0 / scale, 0.05 / scale)) @@ -212,9 +224,11 @@ def bench_cross_solver_consistency(scale: int) -> Result: for z in range(nz): obstacles[site_index(spec, x, y, z)] = 1 spec.obstacles = obstacles - stokes_field, _ = solve_stokes_field(spec, mean_inlet_speed=1.0, tolerance=1.0e-9) + stokes_field, _ = solve_stokes_field( + spec, mean_inlet_speed=1.0, tolerance=1.0e-6, simulation=simulation + ) hele_shaw_field, _ = solve_flow_field( - spec, mean_inlet_speed=1.0, mobility=gap_mobility(spec) + spec, mean_inlet_speed=1.0, mobility=gap_mobility(spec), simulation=simulation ) def column_flux(values: list[float], x: int, fy: int) -> float: @@ -244,18 +258,38 @@ def main() -> int: parser.add_argument( "--fine", action="store_true", help="double the benchmark resolutions" ) + parser.add_argument( + "--backend", + choices=("cpu", "metal", "cuda"), + default="cpu", + help="native backend used for every flow solve", + ) + parser.add_argument("--device-index", type=int, default=0) arguments = parser.parse_args() + backends = { + "cpu": BackendKind.CPU, + "metal": BackendKind.METAL, + "cuda": BackendKind.CUDA, + } + backend = backends[arguments.backend] + if arguments.device_index < 0 or not backend_available(backend, arguments.device_index): + parser.error( + f"backend {arguments.backend!r} has no device at index {arguments.device_index}" + ) + simulation = Simulation(backend, device_index=arguments.device_index) scale = 2 if arguments.fine else 1 results: list[Result] = [] - results.extend(bench_plane_poiseuille_order(8 * scale)) - results.append(bench_square_duct(16 * scale)) - results.extend(bench_two_layer_brinkman(32 * scale)) - results.append(bench_hele_shaw_duct(scale)) - results.append(bench_hele_shaw_mobility_split(scale)) - results.append(bench_cross_solver_consistency(scale)) + results.extend(bench_plane_poiseuille_order(8 * scale, simulation)) + results.append(bench_square_duct(16 * scale, simulation)) + results.extend(bench_two_layer_brinkman(32 * scale, simulation)) + results.append(bench_hele_shaw_duct(scale, simulation)) + results.append(bench_hele_shaw_mobility_split(scale, simulation)) + results.append(bench_cross_solver_consistency(scale, simulation)) width = max(len(r.benchmark) for r in results) + info = simulation.backend_info + print(f"backend: {info.name} ({info.device}), device index {info.device_index}") print(f"{'solver':<11} {'benchmark':<{width}} {'computed':>10} {'reference':>10} " f"{'error':>9} {'tol':>7} {'time':>7} status") failures = 0 diff --git a/tests/conformance/README.md b/tests/conformance/README.md index d529d66..a00f2d7 100644 --- a/tests/conformance/README.md +++ b/tests/conformance/README.md @@ -16,7 +16,11 @@ A backend only earns conformance when this executable passes with that native ba ## Backend contract -The backend contract scenario constructs every enumerated device and requires it to advertise growth, species, cell contacts, cell mechanics, external constraints, signals, and coupled rates. Individual scientific fixtures may retain capability guards to diagnose partially implemented development builds, but a feature-complete Metal or CUDA build cannot pass the conformance suite by opting out of one of those fixtures. +The backend contract scenario constructs every enumerated device and requires it to advertise growth, species, cell contacts, cell mechanics, external constraints, signals, coupled rates, depth-averaged flow, and resolved flow. Individual scientific fixtures may retain capability guards to diagnose partially implemented development builds, but a feature-complete Metal or CUDA build cannot pass the conformance suite by opting out of one of those fixtures. + +## Flow + +The flow scenario exercises two native domain operations. The depth-averaged case uses a 5-by-8-by-2 anisotropic grid with an internal obstacle and spatially varying mobility. The resolved case uses a 6-by-7-by-2 anisotropic grid with a two-layer Brinkman drag field. Every backend solves through its own matrix-free operator and Krylov kernels, reports convergence, and returns all face components. Velocity fields are compared with the CPU reference using absolute and relative tolerances of `8e-4`; the resolved divergence RMS must remain below `5e-5`, and the minimum transverse gap is exact. CPU-only execution validates the reference fixture, Metal conformance requires execution on an Apple GPU, and CUDA conformance requires execution on an NVIDIA GPU. ## Species diff --git a/tests/conformance/backend_contract_conformance_test.cpp b/tests/conformance/backend_contract_conformance_test.cpp index 98c2932..4c1bd55 100644 --- a/tests/conformance/backend_contract_conformance_test.cpp +++ b/tests/conformance/backend_contract_conformance_test.cpp @@ -15,6 +15,8 @@ constexpr std::array required_features{ cm::BackendFeature::external_constraints, cm::BackendFeature::signals, cm::BackendFeature::coupled_rates, + cm::BackendFeature::depth_averaged_flow, + cm::BackendFeature::resolved_flow, }; void require_complete_backend(cm::BackendKind backend, std::uint32_t device_index) { diff --git a/tests/conformance/flow_conformance_test.cpp b/tests/conformance/flow_conformance_test.cpp new file mode 100644 index 0000000..0ce425a --- /dev/null +++ b/tests/conformance/flow_conformance_test.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "backend_devices.hpp" +#include "cm/simulation.hpp" + +namespace { + +cm::SignalGridSpec duct(std::uint32_t nx, std::uint32_t ny, std::uint32_t nz, + cm::Vec3 spacing = {1.0F, 1.0F, 1.0F}) { + cm::SignalGridSpec spec; + spec.signal_count = 1; + spec.shape = {.x = nx, .y = ny, .z = nz}; + spec.spacing = spacing; + spec.diffusion = {0.0F}; + spec.advection = {{0.0F, 0.0F, 0.0F}}; + spec.y_lower = {.kind = cm::GridBoundaryKind::fixed, .values = {0.0F}}; + spec.y_upper = {.kind = cm::GridBoundaryKind::fixed, .values = {0.0F}}; + return spec; +} + +std::size_t site_index(const cm::SignalGridSpec& spec, std::uint32_t x, std::uint32_t y, + std::uint32_t z) { + return (static_cast(x) * spec.shape.y + y) * spec.shape.z + z; +} + +bool close(float actual, float expected) { + constexpr float absolute_tolerance = 8.0e-4F; + constexpr float relative_tolerance = 8.0e-4F; + return std::abs(actual - expected) <= + absolute_tolerance + relative_tolerance * std::abs(expected); +} + +void compare_component(std::span actual, std::span expected, + std::string_view scenario, std::string_view component) { + assert(actual.size() == expected.size()); + for (std::size_t index = 0; index < actual.size(); ++index) { + if (!close(actual[index], expected[index])) { + std::cerr << scenario << ' ' << component << " face " << index << ": actual=" << actual[index] + << " expected=" << expected[index] << '\n'; + std::abort(); + } + } +} + +void compare_fields(const cm::SignalGridVelocityField& actual, + const cm::SignalGridVelocityField& expected, std::string_view scenario) { + compare_component(actual.x_faces, expected.x_faces, scenario, "x"); + compare_component(actual.y_faces, expected.y_faces, scenario, "y"); + compare_component(actual.z_faces, expected.z_faces, scenario, "z"); +} + +void run_depth_case(cm::BackendKind backend, std::uint32_t device_index) { + auto spec = duct(5, 8, 2, {0.7F, 1.1F, 0.6F}); + spec.obstacles.assign(spec.site_count(), 0); + for (std::uint32_t y = 3; y <= 4; ++y) { + spec.obstacles[site_index(spec, 2, y, 0)] = 1; + } + std::vector mobility(spec.site_count(), 1.0F); + for (std::uint32_t x = 0; x < spec.shape.x; ++x) { + for (std::uint32_t y = 0; y < spec.shape.y; ++y) { + for (std::uint32_t z = 0; z < spec.shape.z; ++z) { + mobility[site_index(spec, x, y, z)] = + spec.solid_site(site_index(spec, x, y, z)) + ? 0.0F + : 0.4F + 0.1F * static_cast(x) + 0.05F * static_cast(z); + } + } + } + cm::DepthAveragedFlowParameters parameters; + parameters.mean_inlet_speed = 3.5F; + parameters.relative_tolerance = 1.0e-6F; + cm::Simulation reference; + cm::Simulation candidate(backend, 0, 0, device_index); + const auto expected = reference.solve_depth_averaged_flow(spec, mobility, parameters); + const auto actual = candidate.solve_depth_averaged_flow(spec, mobility, parameters); + assert(actual.report.relative_residual <= 1.1e-6F); + compare_fields(actual.field, expected.field, "depth-averaged flow"); +} + +void run_resolved_case(cm::BackendKind backend, std::uint32_t device_index) { + auto spec = duct(6, 7, 2, {0.2F, 0.35F, 0.3F}); + std::vector drag(spec.site_count(), 0.0F); + for (std::uint32_t x = 0; x < spec.shape.x; ++x) { + for (std::uint32_t y = 0; y < spec.shape.y; ++y) { + for (std::uint32_t z = 0; z < spec.shape.z; ++z) { + if (x >= 3) { + drag[site_index(spec, x, y, z)] = 12.0F; + } + } + } + } + cm::ResolvedFlowParameters parameters; + parameters.mean_inlet_speed = 2.0F; + parameters.relative_tolerance = 1.0e-6F; + parameters.inner_relative_tolerance = 1.0e-6F; + cm::Simulation reference; + cm::Simulation candidate(backend, 0, 0, device_index); + const auto expected = reference.solve_resolved_flow(spec, drag, parameters); + const auto actual = candidate.solve_resolved_flow(spec, drag, parameters); + assert(actual.report.divergence_rms < 5.0e-5F); + assert(actual.report.min_gap_voxels == expected.report.min_gap_voxels); + compare_fields(actual.field, expected.field, "resolved flow"); +} + +} // namespace + +int main() { + cm::test::for_each_backend_device([](cm::BackendKind backend, std::uint32_t device_index) { + cm::Simulation probe(backend, 0, 0, device_index); + assert(probe.supports(cm::BackendFeature::depth_averaged_flow)); + assert(probe.supports(cm::BackendFeature::resolved_flow)); + run_depth_case(backend, device_index); + run_resolved_case(backend, device_index); + }); +} diff --git a/tests/cpp/flow_test.cpp b/tests/cpp/flow_test.cpp new file mode 100644 index 0000000..955bf72 --- /dev/null +++ b/tests/cpp/flow_test.cpp @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "cm/simulation.hpp" + +namespace { + +cm::SignalGridSpec duct(std::uint32_t nx, std::uint32_t ny, std::uint32_t nz, + cm::Vec3 spacing = {1.0F, 1.0F, 1.0F}) { + cm::SignalGridSpec spec; + spec.signal_count = 1; + spec.shape = {.x = nx, .y = ny, .z = nz}; + spec.spacing = spacing; + spec.diffusion = {0.0F}; + spec.advection = {{0.0F, 0.0F, 0.0F}}; + spec.y_lower = {.kind = cm::GridBoundaryKind::fixed, .values = {0.0F}}; + spec.y_upper = {.kind = cm::GridBoundaryKind::fixed, .values = {0.0F}}; + return spec; +} + +std::size_t y_face(const cm::SignalGridSpec& spec, std::uint32_t x, std::uint32_t y, + std::uint32_t z) { + return (static_cast(x) * (spec.shape.y + 1) + y) * spec.shape.z + z; +} + +template +void assert_throws(Function&& function) { + bool rejected = false; + try { + function(); + } catch (const Exception&) { + rejected = true; + } + assert(rejected); +} + +} // namespace + +int main() { + { + const auto spec = duct(4, 8, 3); + cm::Simulation simulation; + const auto result = simulation.solve_depth_averaged_flow( + spec, {}, {.mean_inlet_speed = 5.0F, .axis = cm::FlowAxis::y}); + assert(simulation.supports(cm::BackendFeature::depth_averaged_flow)); + assert(result.report.iterations > 0); + assert(result.report.relative_residual <= 1.0e-6F); + assert(std::ranges::all_of(result.field.y_faces, + [](float value) { return std::abs(value - 5.0F) <= 1.0e-5F; })); + assert(std::ranges::all_of(result.field.x_faces, + [](float value) { return std::abs(value) <= 2.0e-5F; })); + assert(std::ranges::all_of(result.field.z_faces, + [](float value) { return std::abs(value) <= 2.0e-5F; })); + } + + { + const auto spec = duct(2, 6, 1); + std::vector mobility(spec.site_count()); + for (std::uint32_t x = 0; x < spec.shape.x; ++x) { + for (std::uint32_t y = 0; y < spec.shape.y; ++y) { + mobility[(static_cast(x) * spec.shape.y) + y] = x == 0 ? 1.0F : 3.0F; + } + } + const auto result = cm::solve_depth_averaged_flow_cpu( + spec, mobility, {.mean_inlet_speed = 4.0F, .axis = cm::FlowAxis::y}); + const auto slow = result.field.y_faces[y_face(spec, 0, 3, 0)]; + const auto fast = result.field.y_faces[y_face(spec, 1, 3, 0)]; + assert(std::abs((fast / slow) - 3.0F) <= 2.0e-5F); + } + + { + constexpr std::uint32_t nx = 8; + const auto spec = duct(nx, 6, 1, {1.0F / static_cast(nx), 0.25F, 1.0F}); + cm::Simulation simulation; + const auto result = simulation.solve_resolved_flow( + spec, {}, {.mean_inlet_speed = 1.0F, .axis = cm::FlowAxis::y}); + assert(simulation.supports(cm::BackendFeature::resolved_flow)); + float max_error = 0.0F; + for (std::uint32_t x = 0; x < nx; ++x) { + const auto position = (static_cast(x) + 0.5F) / static_cast(nx); + const auto exact = 6.0F * position * (1.0F - position); + max_error = + std::max(max_error, std::abs(result.field.y_faces[y_face(spec, x, 3, 0)] - exact)); + } + assert(max_error / 1.5F < 0.02F); + assert(result.report.outer_iterations > 0); + assert(result.report.inner_iterations > 0); + assert(result.report.divergence_rms < 2.0e-5F); + assert(result.report.min_gap_voxels == nx); + } + + { + auto spec = duct(3, 4, 1); + spec.obstacles.assign(spec.site_count(), 0); + for (std::uint32_t x = 0; x < spec.shape.x; ++x) { + spec.obstacles[(static_cast(x) * spec.shape.y) + 2] = 1; + } + assert_throws( + [&] { static_cast(cm::solve_depth_averaged_flow_cpu(spec, {})); }); + assert_throws( + [&] { static_cast(cm::solve_resolved_flow_cpu(spec, {})); }); + } +}