Faster node tables

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## Unreleased
+
+### Added
+- `table_format` keyword on `write_section_aero`, `generate_airfoils`, `obj_to_yaml`
+  and `surfplan_to_aero_yaml`: `:csv` (default, readable) or `:arrow` (binary, ~40×
+  faster to load and 2.6× smaller). `read_section_aero` detects the format from the
+  file suffix, so a geometry YAML can reference either.
+- `convert_node_table` and `write_node_rows` rewrite a per-node table in the format
+  the destination suffix names. `obj_to_yaml` migrates an existing dataset with
+  them when `table_format` differs from what the directory holds, so a dataset
+  changes format without re-running the airfoil solver that produced it.
+
+### Changed
+- `read_node_table` parses into a preallocated matrix instead of `reduce(vcat, …)`
+  over a generator, which was quadratic in the row count: ~21× faster on a 16 MB
+  surface table (2.49 s → 0.12 s), benefiting every existing dataset.
+
 ## VortexStepMethod v4.0.0 2026-08-03
 
 ### Added

Project.toml

@@ -7,6 +7,7 @@ version = "4.0.0"
 projects = ["examples", "docs", "test"]
 
 [deps]
+Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 DefaultApplication = "3f0dd361-4fe0-5fc6-8523-80b14ec94d85"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
@@ -42,6 +43,7 @@ MakieControlPlots = "6d616b69-6563-4f6e-8472-6f6c706c6f74"
 VortexStepMethodMakieExt = ["MakieControlPlots"]
 
 [compat]
+Arrow = "2.8"
 Colors = "0.13"
 DefaultApplication = "1"
 DelimitedFiles = "1"

bin/run_julia

@@ -5,6 +5,7 @@ if [[ $(basename $(pwd)) == "bin" ]]; then
 fi
 
 export JULIA_PKG_SERVER_REGISTRY_PREFERENCE=eager
+export JULIA_NUM_THREADS="${JULIA_NUM_THREADS:-auto}"
 
 # If any local project preferences file disables CondaPkg, forward that as an
 # env var so that Julia reads it at runtime even when activating a subproject

docs/make.jl

@@ -14,7 +14,6 @@ makedocs(;
              VortexStepMethod.ObjAdapter],
     authors="Uwe Fechner <uwe.fechner.msc@gmail.com>, Bart van de Lint <bart@vandelint.net> and contributors",
     sitename="VortexStepMethod.jl",
-    warnonly=[:cross_references],
     format = Documenter.HTML(prettyurls = haskey(ENV, "CI")),
     pages=[
         "Home" => "index.md",

docs/src/functions.md

@@ -84,6 +84,7 @@ CurrentModule = VortexStepMethod
 set_va!
 solve
 solve!
+solve_base!
 reinit!(body_aero::BodyAerodynamics{P, W, T}) where {P, W, T}
 linearize
 calculate_results

docs/src/private_functions.md

@@ -78,6 +78,8 @@ load_matrix_polar_data
 read_aero_matrix
 read_dat
 read_node_table
+write_node_rows
+convert_node_table
 delta_suffix
 interpolate_matrix_nans!
 remove_vector_nans
@@ -175,6 +177,7 @@ airfoils_from_yaml
 write_geometry_yaml
 resolve_aero_geometry
 plot_airfoil_fit
+migrate_node_tables
 ```
 
 ## Makie plotting internals
@@ -192,6 +195,7 @@ airfoil_skin_geometry
 panel_normal
 plate_hinge_local
 panel_plate_geometry
+PLATE_FACES
 Makie.plot!(ax, panel::VortexStepMethod.Panel)
 Makie.plot!(ax, body::VortexStepMethod.BodyAerodynamics)
 Makie.plot!(body::VortexStepMethod.BodyAerodynamics)

ext/VortexStepMethodMakieExt.jl

@@ -13,6 +13,12 @@ const PANEL_MESH_OBSERVABLES = Ref{Union{Nothing,Dict}}(nothing)
 # Global storage for airfoil-skin observables, keyed by body objectid.
 const AIRFOIL_SKIN_OBSERVABLES = Ref{Union{Nothing,Dict}}(nothing)
 
+"""
+    PLATE_FACES
+
+The 4 triangles that mesh the 6 [`panel_plate_geometry`](@ref) vertices of a panel's
+flat-plate skin, two per side of the hinge.
+"""
 const PLATE_FACES = [Makie.GLTriangleFace(1, 2, 5), Makie.GLTriangleFace(1, 5, 6),
     Makie.GLTriangleFace(2, 3, 4), Makie.GLTriangleFace(2, 4, 5)]
 const PLATE_BORDER_IDX = [1, 2, 3, 4, 5, 6, 1]
@@ -1510,8 +1516,8 @@ end
 Shrink-wrap a section's sliced contour with `wrap_method` and map it back into 3D
 through the section's local airfoil frame, for overlaying on the 3D slice diagnostic.
 A nonzero `delta` (degrees) deflects the trailing edge and re-wraps
-([`deform_section`](@ref)), showing the geometry the solvers consume.
-Returns `nothing` for a degenerate slice.
+([`deform_section`](@ref VortexStepMethod.AirfoilAero.deform_section)), showing the
+geometry the solvers consume. Returns `nothing` for a degenerate slice.
 """
 function fitted_airfoil_3d(s, wrap_method; delta=0.0, crease_frac=0.75)
     frame = ObjAdapter.airfoil_frame(s.LE_point, s.TE_point, s.span_dir)
@@ -1551,10 +1557,13 @@ end
 """
     generated_slices(out_dir, delta, fit_pts) -> (slices, le, te)
 
-Read the stations of a generated [`obj_to_yaml`](@ref) output directory and their
-written `.dat` airfoils — raw slice, wrap, and the `delta`-degree deformed wrap when
-it was generated — assembled for [`plot_slices_3d`](@ref). Nothing is re-sliced or
-re-wrapped; only the Kulfan fits of the stored coordinates are recomputed (via
+Read the stations of a generated
+[`obj_to_yaml`](@ref VortexStepMethod.ObjAdapter.obj_to_yaml) output directory and
+their written `.dat` airfoils — raw slice, wrap, and the `delta`-degree deformed wrap
+when it was generated — assembled for
+[`plot_slices_3d`](@ref VortexStepMethod.ObjAdapter.plot_slices_3d). Nothing is
+re-sliced or re-wrapped; only the Kulfan fits of the stored coordinates are
+recomputed (via
 `fit_pts`), exactly as the polar pipeline fits them.
 """
 function generated_slices(out_dir, delta, fit_pts)
@@ -1605,14 +1614,16 @@ end
 3D slice diagnostic with a hover 2D airfoil panel: raw slice points (green), the
 shrink-wrapped airfoil (crimson) and its Kulfan fit (black dashed), and, for nonzero
 `delta` (degrees), the deflected re-wrapped airfoil (purple — the
-[`deform_section`](@ref) geometry the solvers consume) and its Kulfan fit (orange
-dashed). **Hovering a slice** updates the 2D panel.
+[`deform_section`](@ref VortexStepMethod.AirfoilAero.deform_section) geometry the
+solvers consume) and its Kulfan fit (orange dashed). **Hovering a slice** updates the
+2D panel.
 
 `path` selects the source:
 - a mesh `.obj` file: live preview — slices and wraps here with `wrap_method`
   (`n_slices`, `n_bins`, `wingtip_distance`, `crease_frac`).
-- a generated [`obj_to_yaml`](@ref) output directory: audit mode — stations and
-  airfoils are read from `geometry.yaml` and the written `.dat` files, so the plot
+- a generated [`obj_to_yaml`](@ref VortexStepMethod.ObjAdapter.obj_to_yaml) output
+  directory: audit mode — stations and airfoils are read from `geometry.yaml` and the
+  written `.dat` files, so the plot
   shows exactly what the polar pipeline analysed. `delta` must then match a
   generated deflection value; pass `obj_path` to also draw the mesh (with the same
   `rotation` used at generation).

src/VortexStepMethod.jl

@@ -1,5 +1,6 @@
 module VortexStepMethod
 
+using Arrow
 using LinearAlgebra
 using StaticArrays
 using Logging
@@ -261,7 +262,7 @@ end
 """
     LEI_AIRFOIL_BREUKELS
 
-Deprecated alias of [`POLY`](@ref). The Breukels `(tube_diameter, camber)` → coeff
+Deprecated alias of `POLY` (see [`AeroModel`](@ref)). The Breukels `(tube_diameter, camber)` → coeff
 derivation now lives in `AirfoilAero.lei_poly_coeffs`; sections carry the resulting
 `(cl_coeffs, cd_coeffs, cm_coeffs)`.
 """

src/airfoil_aero/AirfoilAero.jl

@@ -7,7 +7,8 @@ using Interpolations
 using NPZ
 using Xfoil
 using Printf: @sprintf
-using ..VortexStepMethod: SectionAero, interpolate_matrix_nans!, delta_suffix
+using ..VortexStepMethod: SectionAero, interpolate_matrix_nans!, delta_suffix,
+                          write_node_rows
 
 include("kulfan.jl")
 include("shrink_wrap.jl")

src/airfoil_aero/geometry_gen.jl

@@ -1,8 +1,8 @@
 """
     generate_airfoils(airfoils, output_dir; Re, alpha_range=-180:1:180,
                       delta_range=nothing, aero_solver=NeuralFoilSolver(),
-                      reuse_valid_airfoils=true, crease_frac=0.75, verbose=true)
-        -> (airfoil_rows, ok)
+                      reuse_valid_airfoils=true, crease_frac=0.75, verbose=true,
+                      table_format=:csv) -> (airfoil_rows, ok)
 
 Run the 2D solver over a set of already-shrink-wrapped airfoils and write the per
 section files each geometry route references. Shared by the `.obj` and Surfplan
@@ -15,9 +15,10 @@ the wrapped airfoil the solver analyses; `x_raw`/`y_raw` the raw points it enclo
 
 Writes into `output_dir` (indexed by `id`), one directory per file kind:
 `airfoils/{id}.dat` (wrapped shape), `airfoils/{id}_{delta_suffix(δ)}.dat` (per
-deflection), `airfoils/{id}_raw.dat` (raw points); `pressure/{id}_cp.csv` / `_cf.csv`
-(per-node surface pressure and skin friction); and `polars/{id}.csv` (`POLAR_VECTORS`,
-or a `POLAR_MATRICES` grid when `delta_range` is set). `aero_solver` selects the backend
+deflection), `airfoils/{id}_raw.dat` (raw points); `pressure/{id}_cp.{table_format}` /
+`_cf.{table_format}` (per-node surface pressure and skin friction, `:csv` or the far
+faster-loading `:arrow`); and `polars/{id}.csv` (`POLAR_VECTORS`, or a `POLAR_MATRICES`
+grid when `delta_range` is set). `aero_solver` selects the backend
 ([`NeuralFoilSolver`](@ref) default, [`XFoilSolver`](@ref) opt-in).
 
 With `reuse_valid_airfoils=true` an airfoil the solver cannot converge is skipped
@@ -26,7 +27,8 @@ With `reuse_valid_airfoils=true` an airfoil the solver cannot converge is skippe
 function generate_airfoils(airfoils, output_dir::String;
         Re::Real, alpha_range=-180:1:180, delta_range=nothing,
         aero_solver::AbstractAirfoilSolver=NeuralFoilSolver(),
-        reuse_valid_airfoils::Bool=true, crease_frac=0.75, verbose::Bool=true)
+        reuse_valid_airfoils::Bool=true, crease_frac=0.75, verbose::Bool=true,
+        table_format::Symbol=:csv)
     mkpath(joinpath(output_dir, "airfoils"))
     mkpath(joinpath(output_dir, "polars"))
     mkpath(joinpath(output_dir, "pressure"))
@@ -54,7 +56,7 @@ function generate_airfoils(airfoils, output_dir::String;
                     coeff(s -> s.cl), coeff(s -> s.cd), coeff(s -> s.cm))
             end
             paths = write_section_aero(joinpath(output_dir, "airfoils", "$j"), aero;
-                table_prefix=joinpath(output_dir, "pressure", "$j"))
+                table_prefix=joinpath(output_dir, "pressure", "$j"), table_format)
             dat_rel, cp_rel, cf_rel = (relpath(p, output_dir) for p in paths)
             write_dat(joinpath(output_dir, raw_rel), "section_$(j)_raw",
                       af.x_raw, af.y_raw)

src/airfoil_aero/section_aero_gen.jl

@@ -94,36 +94,44 @@ end
 """
     write_node_table(path, aero, values) -> path
 
-Write a per-node aero table (`Cp` or `cf`, shaped `n_node × n_alpha × n_delta`) as a
-human-readable CSV: header `alpha, delta, n0, n1, …` (node columns in the contour node
-order), one row per `(alpha, delta)` with angles in degrees.
+Write a per-node aero table (`Cp` or `cf`, shaped `n_node × n_alpha × n_delta`), one row
+per `(alpha, delta)` with angles in degrees. The file suffix picks the format: `.arrow`
+(columns `alpha`, `delta` and a per-row list column `values`, an order of magnitude
+faster to load), anything else a human-readable CSV with header `alpha, delta, n0, n1, …`
+(node columns in the contour node order).
+[`read_node_table`](@ref VortexStepMethod.read_node_table) reads both.
 """
 function write_node_table(path::AbstractString, aero::SectionAero, values)
-    open(String(path), "w") do io
-        println(io, "alpha,delta," * join(("n$(k - 1)" for k in 1:size(values, 1)), ","))
-        for jd in eachindex(aero.delta_range), ia in eachindex(aero.alpha_range)
-            row = [rad2deg(aero.alpha_range[ia]), rad2deg(aero.delta_range[jd])]
-            append!(row, values[:, ia, jd])
-            println(io, join(row, ","))
-        end
+    grid = [(jd, ia) for jd in eachindex(aero.delta_range)
+                     for ia in eachindex(aero.alpha_range)]
+    rows = Matrix{Float64}(undef, length(grid), size(values, 1))
+    for (k, (jd, ia)) in enumerate(grid)
+        rows[k, :] .= @view values[:, ia, jd]
     end
-    return path
+    return write_node_rows(path,
+        [aero.alpha_range[ia] for (_, ia) in grid],
+        [aero.delta_range[jd] for (jd, _) in grid], rows)
 end
 
 """
-    write_section_aero(dat_prefix, aero::SectionAero; table_prefix=dat_prefix)
-        -> (dat, cp_csv, cf_csv)
+    write_section_aero(dat_prefix, aero::SectionAero; table_prefix=dat_prefix,
+                       table_format=:csv) -> (dat, cp_table, cf_table)
 
 Write a [`SectionAero`](@ref) as human-readable files. The airfoil contours share
 `dat_prefix`: `{dat_prefix}.dat` (contour at `delta=0`) plus
 `{dat_prefix}_{delta_suffix(δ)}.dat` per non-zero deflection. The per-node `Cp`/`cf`
-tables share `table_prefix` (defaults to `dat_prefix`): `{table_prefix}_cp.csv` and
-`{table_prefix}_cf.csv`. Pass a separate `table_prefix` to keep the surface tables out
-of the airfoil-shape directory. `read_section_aero` reads them back. The single writer
-for surface aero (submodule side); loading lives in the main package.
+tables share `table_prefix` (defaults to `dat_prefix`): `{table_prefix}_cp.{ext}` and
+`{table_prefix}_cf.{ext}`, where `ext` is `table_format`, either `:csv` (default,
+readable) or `:arrow` (binary, an order of magnitude faster to load). Pass a separate
+`table_prefix` to keep the surface tables out of the airfoil-shape directory.
+`read_section_aero` reads either format back, detecting it from the suffix. The single
+writer for surface aero (submodule side); loading lives in the main package.
 """
 function write_section_aero(dat_prefix::AbstractString, aero::SectionAero;
-                            table_prefix::AbstractString=dat_prefix)
+                            table_prefix::AbstractString=dat_prefix,
+                            table_format::Symbol=:csv)
+    table_format in (:csv, :arrow) ||
+        throw(ArgumentError("table_format must be :csv or :arrow, got :$table_format"))
     mkpath(dirname(dat_prefix))
     mkpath(dirname(table_prefix))
     for (jd, d) in enumerate(aero.delta_range)
@@ -136,7 +144,9 @@ function write_section_aero(dat_prefix::AbstractString, aero::SectionAero;
     jz = findfirst(iszero, aero.delta_range)
     jdat = jz !== nothing && finite(jz) ? jz : findfirst(finite, eachindex(aero.delta_range))
     write_dat("$dat_prefix.dat", "section", aero.x[:, jdat], aero.y[:, jdat])
-    write_node_table("$(table_prefix)_cp.csv", aero, aero.cp)
-    write_node_table("$(table_prefix)_cf.csv", aero, aero.cf)
-    return "$dat_prefix.dat", "$(table_prefix)_cp.csv", "$(table_prefix)_cf.csv"
+    cp_path = "$(table_prefix)_cp.$table_format"
+    cf_path = "$(table_prefix)_cf.$table_format"
+    write_node_table(cp_path, aero, aero.cp)
+    write_node_table(cf_path, aero, aero.cf)
+    return "$dat_prefix.dat", cp_path, cf_path
 end

src/obj_adapter/obj_to_yaml.jl

@@ -81,11 +81,63 @@ share a neighbour's id):
   `{tag}` is the deflection in degrees with `m` for minus and `p` for the decimal point
   (e.g. `_dm3.dat`, `_d2p5.dat`)
 - `polars/{j}.csv`    — NeuralFoil polar (alpha, Cd, Cs, Cl, Cm)
+- `pressure/{j}_cp.{table_format}`, `_cf.{table_format}` — per-node surface pressure and
+  skin friction; `table_format` is `:csv` (default, readable) or `:arrow` (binary, an
+  order of magnitude faster to load)
 - `geometry.yaml`     — `wing_sections` + `wing_airfoils` referencing the above
 
 # Returns
 - Path to the written `geometry.yaml`.
 """
+function obj_to_yaml end
+
+"""
+    migrate_node_tables(yaml_path, output_dir, table_format; verbose=true)
+
+Rewrite a generated dataset's per-node `Cp`/`cf` tables in `table_format` and point
+`geometry.yaml` at them, when they are not in that format already. This is what lets
+an existing directory change format without re-running the airfoil solver that
+produced it — the polars are the slow part and they are untouched. The source tables
+are left in place.
+"""
+function migrate_node_tables(yaml_path::String, output_dir::String,
+                             table_format::Symbol; verbose::Bool=true)
+    table_format in (:csv, :arrow) ||
+        throw(ArgumentError("table_format must be :csv or :arrow, got :$table_format"))
+    data = YAML.load_file(yaml_path)
+    airfoils = get(data, "wing_airfoils", nothing)
+    airfoils === nothing && return yaml_path
+    headers = airfoils["headers"]
+    info_col = findfirst(==("info_dict"), headers)
+    info_col === nothing && return yaml_path
+    converted = 0
+    for row in airfoils["data"]
+        info = row[info_col]
+        info isa AbstractDict || continue
+        for key in ("cp_file", "cf_file")
+            haskey(info, key) || continue
+            relative = String(info[key])
+            endswith(relative, ".$table_format") && continue
+            fresh = string(splitext(relative)[1], ".", table_format)
+            source = joinpath(output_dir, relative)
+            isfile(source) || error("Table $source referenced by $yaml_path is missing")
+            isfile(joinpath(output_dir, fresh)) ||
+                VortexStepMethod.convert_node_table(source,
+                                                    joinpath(output_dir, fresh))
+            info[key] = fresh
+            converted += 1
+        end
+    end
+    converted == 0 && return yaml_path
+    id_col = findfirst(==("airfoil_id"), headers)
+    type_col = findfirst(==("type"), headers)
+    write_geometry_yaml(yaml_path, data["wing_sections"]["data"],
+                        [Any[row[id_col], row[type_col], row[info_col]]
+                         for row in airfoils["data"]])
+    verbose && @info "Converted $converted node tables to :$table_format" yaml_path
+    return yaml_path
+end
+
 function obj_to_yaml(obj_path::String, output_dir::String;
                      n_sections::Int, Re::Real,
                      alpha_range=-180:1:180, delta_range=nothing,
@@ -94,7 +146,7 @@ function obj_to_yaml(obj_path::String, output_dir::String;
                      reuse_valid_airfoils::Bool=true, max_thickness_ratio::Real=2.0,
                      spanwise_direction=[0.0, 1.0, 0.0], rotation=I,
                      wingtip_distance=0.05, crease_frac=0.75, force::Bool=false,
-                     verbose::Bool=true)
+                     verbose::Bool=true, table_format::Symbol=:csv)
     (!endswith(obj_path, ".obj")) && (obj_path *= ".obj")
     isfile(obj_path) || error("OBJ file not found: $obj_path")
     !isapprox(spanwise_direction, [0.0, 1.0, 0.0]) &&
@@ -103,6 +155,7 @@ function obj_to_yaml(obj_path::String, output_dir::String;
     yaml_path = joinpath(output_dir, "geometry.yaml")
     if !force && isfile(yaml_path)
         verbose && @info "Reusing existing geometry (force=true to regenerate)" yaml_path
+        migrate_node_tables(yaml_path, output_dir, table_format; verbose)
         return yaml_path
     end
 
@@ -132,7 +185,8 @@ function obj_to_yaml(obj_path::String, output_dir::String;
     airfoils = [(; id = j, x_fit = stations[j].x_fit, y_fit = stations[j].y_fit,
                  x_raw = stations[j].xa, y_raw = stations[j].ya) for j in unique(ids)]
     airfoil_rows, ok = generate_airfoils(airfoils, output_dir; Re, alpha_range,
-        delta_range, aero_solver, reuse_valid_airfoils, crease_frac, verbose)
+        delta_range, aero_solver, reuse_valid_airfoils, crease_frac, verbose,
+        table_format)
     isempty(ok) && error("No section produced a valid polar in $obj_path")
 
     # Each section uses its nearest airfoil that actually produced a polar — covering