From a10963a6f14e6982c41683543687235f32159a77 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 22:51:10 -0700 Subject: [PATCH 01/14] Restructure the C binding into a CAPI submodule with verified ABI Replace src/C_interface.jl with a ReadStat.CAPI submodule that wraps the readstat 1.1.9 parser-side C API one-to-one: proper Julia enums for every C enum (including all 42 error codes), opaque typed pointers plus C getters for every struct except readstat_value_t, and @ccall-based wrappers covering the metadata, variable, value, and missing-range accessors, handler and custom-IO setters, encodings, and row limit/offset. readstat_value_t remains the one by-value struct mirror. Its layout is now verified behaviorally against the C library by a new ABI test item that round-trips missing ranges through by-value returns; that test showed MinGW GCC uses MS-compatible bitfield layout, so the struct is 24 bytes on Windows (bitfields in a fresh 4-byte unit at offset 16) and 16 bytes elsewhere. The parser is now freed via try/finally so a throwing handler cannot leak it. Version 2.0.0-DEV; minimum Julia raised to 1.12. Reader behavior unchanged. Co-Authored-By: Claude Fable 5 --- Project.toml | 4 +- src/C_interface.jl | 80 -------------- src/ReadStat.jl | 260 ++++++++++++++++++++------------------------- src/capi/enums.jl | 112 +++++++++++++++++++ src/capi/parser.jl | 195 ++++++++++++++++++++++++++++++++++ src/capi/value.jl | 109 +++++++++++++++++++ test/runtests.jl | 1 + test/test_abi.jl | 56 ++++++++++ 8 files changed, 589 insertions(+), 228 deletions(-) delete mode 100644 src/C_interface.jl create mode 100644 src/capi/enums.jl create mode 100644 src/capi/parser.jl create mode 100644 src/capi/value.jl create mode 100644 test/test_abi.jl diff --git a/Project.toml b/Project.toml index 1891f22..32b4519 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "ReadStat" uuid = "d71aba96-b539-5138-91ee-935c3ee1374c" -version = "1.2.1-DEV" +version = "2.0.0-DEV" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" @@ -12,7 +12,7 @@ TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] -julia = "1.10" +julia = "1.12" DataValues = "0.4.13, 0.5, 1" ReadStat_jll = "1.1.9" diff --git a/src/C_interface.jl b/src/C_interface.jl deleted file mode 100644 index eeede3a..0000000 --- a/src/C_interface.jl +++ /dev/null @@ -1,80 +0,0 @@ -function readstat_get_file_label(metadata::Ptr{Nothing}) - ptr = ccall((:readstat_get_file_label, libreadstat), Cstring, (Ptr{Nothing},), metadata) - return ptr == C_NULL ? "" : unsafe_string(ptr) -end - -function readstat_get_modified_time(metadata::Ptr{Nothing}) - return ccall((:readstat_get_modified_time, libreadstat), Int64, (Ptr{Nothing},), metadata) -end - -function readstat_get_file_format_version(metadata::Ptr{Nothing}) - return ccall((:readstat_get_file_format_version, libreadstat), Cint, (Ptr{Nothing},), metadata) -end - -function readstat_get_row_count(metadata::Ptr{Nothing}) - return ccall((:readstat_get_row_count, libreadstat), Cint, (Ptr{Nothing},), metadata) -end - -function readstat_get_var_count(metadata::Ptr{Nothing}) - return ccall((:readstat_get_var_count, libreadstat), Cint, (Ptr{Nothing},), metadata) -end - -function readstat_value_is_missing(value::ReadStatValue, variable::Ptr{Nothing}) - return Bool(ccall((:readstat_value_is_missing, libreadstat), Cint, (ReadStatValue,Ptr{Nothing}), value, variable)) -end - -function readstat_variable_get_index(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_index, libreadstat), Cint, (Ptr{Nothing},), variable) -end - -function readstat_variable_get_name(variable::Ptr{Nothing}) - return unsafe_string(ccall((:readstat_variable_get_name, libreadstat), Cstring, (Ptr{Nothing},), variable)) -end - -function readstat_variable_get_type(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_type, libreadstat), Cint, (Ptr{Nothing},), variable) -end - -function readstat_variable_get_storage_width(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_storage_width, libreadstat), Csize_t, (Ptr{Nothing},), variable) -end - -function readstat_variable_get_measure(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_measure, libreadstat), Cint, (Ptr{Nothing},), variable) -end - -function readstat_variable_get_alignment(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_alignment, libreadstat), Cint, (Ptr{Nothing},), variable) -end - -function readstat_parser_free(parser::Ptr{Nothing}) - return ccall((:readstat_parser_free, libreadstat), Nothing, (Ptr{Nothing},), parser) -end - -function readstat_value_type(val::Value) - return ccall((:readstat_value_type, libreadstat), Cint, (Value,), val) -end - -function readstat_parse(filename::String, type::Val{:dta}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_dta, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_parse(filename::String, type::Val{:sav}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_sav, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_parse(filename::String, type::Val{:por}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_por, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_parse(filename::String, type::Val{:sas7bdat}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_sas7bdat, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_parse(filename::String, type::Val{:xport}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_xport, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_variable_get_missing_ranges_count(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_missing_ranges_count, libreadstat), Cint, (Ptr{Nothing},), variable) -end diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 2d26bbf..edb09d4 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -1,63 +1,48 @@ module ReadStat -using ReadStat_jll - -############################################################################## -## -## Import -## -############################################################################## - using DataValues: DataValueVector import DataValues using Dates export ReadStatDataFrame, read_dta, read_sav, read_por, read_sas7bdat, read_xport -############################################################################## -## -## Julia types that mirror C types -## -############################################################################## +public CAPI + +""" + ReadStat.CAPI + +Thin wrappers around the public C API of the readstat library (readstat.h, +v1.1.9). Function names, argument order, and semantics follow the C header +one-to-one; C structs are handled as opaque pointers except `ReadStatValue`, +which the C API passes by value. Higher-level functionality lives in the +parent `ReadStat` module — reach for `CAPI` only when the high-level API does +not expose what you need. +""" +module CAPI + +using ReadStat_jll: libreadstat -const READSTAT_TYPE_STRING = Cint(0) -const READSTAT_TYPE_INT8 = Cint(1) -const READSTAT_TYPE_INT16 = Cint(2) -const READSTAT_TYPE_INT32 = Cint(3) -const READSTAT_TYPE_FLOAT = Cint(4) -const READSTAT_TYPE_DOUBLE = Cint(5) -const READSTAT_TYPE_STRING_REF = Cint(6) - -# Julia type for each readstat_type_t, indexed by the enum value plus one. -# STRING_REF is a reference into a string table, so it surfaces as a String -# just like STRING does. +include("capi/enums.jl") +include("capi/value.jl") +include("capi/parser.jl") + +end # module CAPI + +using .CAPI + +# Julia type corresponding to each ReadStatType, indexed by the enum value +# plus one. STRING_REF is a reference into a string table, so it surfaces as +# a String just like STRING does. const READSTAT_TYPES = (String, Int8, Int16, Int32, Float32, Float64, String) -const READSTAT_ERROR_OPEN = Cint(1) -const READSTAT_ERROR_READ = Cint(2) -const READSTAT_ERROR_MALLOC = Cint(3) -const READSTAT_ERROR_USER_ABORT = Cint(4) -const READSTAT_ERROR_PARSE = Cint(5) +jltype(t::ReadStatType) = READSTAT_TYPES[Int(t) + 1] ############################################################################## ## -## Pure Julia types +## Result type ## ############################################################################## -struct ReadStatValue - union::Int64 - readstat_types_t::Cint - tag::Cchar - @static if Sys.iswindows() - bits::Cuint - else - bits::UInt8 - end -end - -const Value = ReadStatValue - mutable struct ReadStatDataFrame data::Vector{Any} headers::Vector{Symbol} @@ -82,91 +67,72 @@ mutable struct ReadStatDataFrame String[], Dict{String, Dict{Any,String}}(), 0, 0, "", Dates.unix2datetime(0), 0, Cint[], Bool[]) end -include("C_interface.jl") - ############################################################################## ## -## Julia functions +## Parse handlers ## ############################################################################## -function handle_metadata!(metadata::Ptr{Nothing}, ds_ptr::Ptr{ReadStatDataFrame}) - ds = unsafe_pointer_to_objref(ds_ptr) +function handle_metadata!(metadata::MetadataPtr, ds_ptr::Ptr{ReadStatDataFrame}) + ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame ds.filelabel = readstat_get_file_label(metadata) ds.timestamp = Dates.unix2datetime(readstat_get_modified_time(metadata)) ds.format = readstat_get_file_format_version(metadata) ds.rows = readstat_get_row_count(metadata) ds.columns = readstat_get_var_count(metadata) - return Cint(0) -end - -get_name(variable::Ptr{Nothing}) = Symbol(readstat_variable_get_name(variable)) - -function get_label(var::Ptr{Nothing}) - ptr = ccall((:readstat_variable_get_label, libreadstat), Cstring, (Ptr{Nothing},), var) - ptr == C_NULL ? "" : unsafe_string(ptr) + return READSTAT_HANDLER_OK end -function get_format(var::Ptr{Nothing}) - ptr = ccall((:readstat_variable_get_format, libreadstat), Cstring, (Ptr{Nothing},), var) - ptr == C_NULL ? "" : unsafe_string(ptr) -end - -get_type(data_type::Cint) = READSTAT_TYPES[data_type + 1] -get_type(variable::Ptr{Nothing}) = get_type(readstat_variable_get_type(variable)) - -get_storagewidth(variable::Ptr{Nothing}) = readstat_variable_get_storage_width(variable) - -get_measure(variable::Ptr{Nothing}) = readstat_variable_get_measure(variable) - -get_alignment(variable::Ptr{Nothing}) = readstat_variable_get_alignment(variable) - -function handle_variable!(var_index::Cint, variable::Ptr{Nothing}, - val_label::Cstring, ds_ptr::Ptr{ReadStatDataFrame}) - col = var_index + 1 +function handle_variable!(var_index::Cint, variable::VariablePtr, + val_label::Cstring, ds_ptr::Ptr{ReadStatDataFrame}) ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame missing_count = readstat_variable_get_missing_ranges_count(variable) - push!(ds.val_label_keys, (val_label == C_NULL ? "" : unsafe_string(val_label))) - push!(ds.headers, get_name(variable)) - push!(ds.labels, get_label(variable)) - push!(ds.formats, get_format(variable)) - jtype = get_type(variable) - push!(ds.types, jtype) - push!(ds.types_as_int, readstat_variable_get_type(variable)) + push!(ds.val_label_keys, val_label == C_NULL ? "" : unsafe_string(val_label)) + push!(ds.headers, Symbol(readstat_variable_get_name(variable))) + push!(ds.labels, readstat_variable_get_label(variable)) + push!(ds.formats, readstat_variable_get_format(variable)) + ctype = readstat_variable_get_type(variable) + T = jltype(ctype) + push!(ds.types, T) + push!(ds.types_as_int, Cint(Int(ctype))) push!(ds.hasmissings, missing_count > 0) - # SAS XPORT sets ds.rows == -1 + # XPORT and POR report an unknown row count as -1. if ds.rows >= 0 - push!(ds.data, DataValueVector{jtype}(Vector{jtype}(undef, ds.rows), fill(false, ds.rows))) + push!(ds.data, DataValueVector{T}(Vector{T}(undef, ds.rows), fill(false, ds.rows))) else - push!(ds.data, DataValueVector{jtype}(Vector{jtype}(undef, 0), fill(false, 0))) + push!(ds.data, DataValueVector{T}(Vector{T}(undef, 0), fill(false, 0))) end - push!(ds.storagewidths, get_storagewidth(variable)) - push!(ds.measures, get_measure(variable)) - push!(ds.alignments, get_alignment(variable)) + push!(ds.storagewidths, readstat_variable_get_storage_width(variable)) + push!(ds.measures, Cint(Int(readstat_variable_get_measure(variable)))) + push!(ds.alignments, Cint(Int(readstat_variable_get_alignment(variable)))) - return Cint(0) + return READSTAT_HANDLER_OK end -get_type(val::Value) = get_type(readstat_value_type(val)) - -Base.convert(::Type{Int8}, val::Value) = ccall((:readstat_int8_value, libreadstat), Int8, (Value,), val) -Base.convert(::Type{Int16}, val::Value) = ccall((:readstat_int16_value, libreadstat), Int16, (Value,), val) -Base.convert(::Type{Int32}, val::Value) = ccall((:readstat_int32_value, libreadstat), Int32, (Value,), val) -Base.convert(::Type{Float32}, val::Value) = ccall((:readstat_float_value, libreadstat), Float32, (Value,), val) -Base.convert(::Type{Float64}, val::Value) = ccall((:readstat_double_value, libreadstat), Float64, (Value,), val) -function Base.convert(::Type{String}, val::Value) - ptr = ccall((:readstat_string_value, libreadstat), Cstring, (Value,), val) - ptr ≠ C_NULL ? unsafe_string(ptr) : "" +function as_native(val::ReadStatValue) + t = readstat_value_type(val) + if t == READSTAT_TYPE_INT8 + return readstat_int8_value(val) + elseif t == READSTAT_TYPE_INT16 + return readstat_int16_value(val) + elseif t == READSTAT_TYPE_INT32 + return readstat_int32_value(val) + elseif t == READSTAT_TYPE_FLOAT + return readstat_float_value(val) + elseif t == READSTAT_TYPE_DOUBLE + return readstat_double_value(val) + else + ptr = readstat_string_value(val) + return ptr == C_NULL ? "" : unsafe_string(ptr) + end end -as_native(val::Value) = convert(get_type(val), val) -function handle_value!(obs_index::Cint, variable::Ptr{Nothing}, +function handle_value!(obs_index::Cint, variable::VariablePtr, value::ReadStatValue, ds_ptr::Ptr{ReadStatDataFrame}) ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame var_index = readstat_variable_get_index(variable) + 1 data = ds.data - @inbounds type_as_int = ds.types_as_int[var_index] ismissing = if @inbounds(ds.hasmissings[var_index]) readstat_value_is_missing(value, variable) @@ -175,7 +141,6 @@ function handle_value!(obs_index::Cint, variable::Ptr{Nothing}, end col = data[var_index] - @assert eltype(eltype(col)) == get_type(type_as_int) if ismissing if obs_index < length(col) @@ -187,83 +152,86 @@ function handle_value!(obs_index::Cint, variable::Ptr{Nothing}, readfield!(col, obs_index + 1, value) end - return Cint(0) + return READSTAT_HANDLER_OK end function readfield!(dest::DataValueVector{String}, row, val::ReadStatValue) - ptr = ccall((:readstat_string_value, libreadstat), Cstring, (ReadStatValue,), val) + ptr = readstat_string_value(val) if row <= length(dest) - if ptr ≠ C_NULL + if ptr != C_NULL @inbounds DataValues.unsafe_setindex_value!(dest, unsafe_string(ptr), row) end elseif row == length(dest) + 1 - _val = ptr ≠ C_NULL ? unsafe_string(ptr) : "" - DataValues.push!(dest, _val) + push!(dest, ptr != C_NULL ? unsafe_string(ptr) : "") else throw(ArgumentError("illegal row index: $row")) end end -for (j_type, rs_name) in ( - (Int8, :readstat_int8_value), - (Int16, :readstat_int16_value), - (Int32, :readstat_int32_value), - (Float32, :readstat_float_value), - (Float64, :readstat_double_value)) - @eval function readfield!(dest::DataValueVector{$j_type}, row, val::ReadStatValue) - _val = ccall(($(QuoteNode(rs_name)), libreadstat), $j_type, (ReadStatValue,), val) +for (T, value_fn) in ((Int8, :readstat_int8_value), + (Int16, :readstat_int16_value), + (Int32, :readstat_int32_value), + (Float32, :readstat_float_value), + (Float64, :readstat_double_value)) + @eval function readfield!(dest::DataValueVector{$T}, row, val::ReadStatValue) + _val = $value_fn(val) if row <= length(dest) @inbounds DataValues.unsafe_setindex_value!(dest, _val, row) elseif row == length(dest) + 1 - DataValues.push!(dest, _val) + push!(dest, _val) else throw(ArgumentError("illegal row index: $row")) end end end -function handle_value_label!(val_labels::Cstring, value::Value, label::Cstring, ds_ptr::Ptr{ReadStatDataFrame}) - val_labels ≠ C_NULL || return Cint(0) - ds = unsafe_pointer_to_objref(ds_ptr) +function handle_value_label!(val_labels::Cstring, value::ReadStatValue, label::Cstring, + ds_ptr::Ptr{ReadStatDataFrame}) + val_labels != C_NULL || return READSTAT_HANDLER_OK + ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame dict = get!(ds.val_label_dict, unsafe_string(val_labels), Dict{Any,String}()) dict[as_native(value)] = unsafe_string(label) - return Cint(0) + return READSTAT_HANDLER_OK end -function read_data_file(filename::AbstractString, filetype::Val) - # initialize ds - ds = ReadStatDataFrame() - # initialize parser - parser = Parser() - # parse - parse_data_file!(ds, parser, filename, filetype) - # return dataframe instead of ReadStatDataFrame - return ds -end +############################################################################## +## +## Parse driver +## +############################################################################## function Parser() - parser = ccall((:readstat_parser_init, libreadstat), Ptr{Nothing}, ()) - meta_fxn = @cfunction(handle_metadata!, Cint, (Ptr{Nothing}, Ptr{ReadStatDataFrame})) - var_fxn = @cfunction(handle_variable!, Cint, (Cint, Ptr{Nothing}, Cstring, Ptr{ReadStatDataFrame})) - val_fxn = @cfunction(handle_value!, Cint, (Cint, Ptr{Nothing}, ReadStatValue, Ptr{ReadStatDataFrame})) - label_fxn = @cfunction(handle_value_label!, Cint, (Cstring, Value, Cstring, Ptr{ReadStatDataFrame})) - ccall((:readstat_set_metadata_handler, libreadstat), Cint, (Ptr{Nothing}, Ptr{Nothing}), parser, meta_fxn) - ccall((:readstat_set_variable_handler, libreadstat), Cint, (Ptr{Nothing}, Ptr{Nothing}), parser, var_fxn) - ccall((:readstat_set_value_handler, libreadstat), Cint, (Ptr{Nothing}, Ptr{Nothing}), parser, val_fxn) - ccall((:readstat_set_value_label_handler, libreadstat), Cint, (Ptr{Nothing}, Ptr{Nothing}), parser, label_fxn) + parser = readstat_parser_init() + meta_fxn = @cfunction(handle_metadata!, Cint, (MetadataPtr, Ptr{ReadStatDataFrame})) + var_fxn = @cfunction(handle_variable!, Cint, (Cint, VariablePtr, Cstring, Ptr{ReadStatDataFrame})) + val_fxn = @cfunction(handle_value!, Cint, (Cint, VariablePtr, ReadStatValue, Ptr{ReadStatDataFrame})) + label_fxn = @cfunction(handle_value_label!, Cint, (Cstring, ReadStatValue, Cstring, Ptr{ReadStatDataFrame})) + readstat_set_metadata_handler(parser, meta_fxn) + readstat_set_variable_handler(parser, var_fxn) + readstat_set_value_handler(parser, val_fxn) + readstat_set_value_label_handler(parser, label_fxn) return parser end -function error_message(retval::Integer) - unsafe_string(ccall((:readstat_error_message, libreadstat), Ptr{Cchar}, (Cint,), retval)) -end +readstat_parse(parser, path, ::Val{:dta}, ctx) = CAPI.readstat_parse_dta(parser, path, ctx) +readstat_parse(parser, path, ::Val{:sav}, ctx) = CAPI.readstat_parse_sav(parser, path, ctx) +readstat_parse(parser, path, ::Val{:por}, ctx) = CAPI.readstat_parse_por(parser, path, ctx) +readstat_parse(parser, path, ::Val{:sas7bdat}, ctx) = CAPI.readstat_parse_sas7bdat(parser, path, ctx) +readstat_parse(parser, path, ::Val{:xport}, ctx) = CAPI.readstat_parse_xport(parser, path, ctx) -function parse_data_file!(ds::ReadStatDataFrame, parser::Ptr{Nothing}, filename::AbstractString, filetype::Val) - retval = readstat_parse(filename, filetype, parser, ds) - readstat_parser_free(parser) - retval == 0 || error("Error parsing $filename: $(error_message(retval))") +function read_data_file(filename::AbstractString, filetype::Val) + ds = ReadStatDataFrame() + parser = Parser() + local retval + try + retval = readstat_parse(parser, filename, filetype, ds) + finally + readstat_parser_free(parser) + end + retval == READSTAT_OK || error("Error parsing $filename: $(readstat_error_message(retval))") + return ds end read_dta(filename::AbstractString) = read_data_file(filename, Val(:dta)) @@ -272,4 +240,4 @@ read_por(filename::AbstractString) = read_data_file(filename, Val(:por)) read_sas7bdat(filename::AbstractString) = read_data_file(filename, Val(:sas7bdat)) read_xport(filename::AbstractString) = read_data_file(filename, Val(:xport)) -end #module ReadStat +end # module ReadStat diff --git a/src/capi/enums.jl b/src/capi/enums.jl new file mode 100644 index 0000000..c95bdcd --- /dev/null +++ b/src/capi/enums.jl @@ -0,0 +1,112 @@ +# Enums and handler status codes mirroring readstat.h (v1.1.9). + +# Return codes for parse handlers. SKIP_VARIABLE is only valid from the +# variable handler; subsequent variables keep their original index values. +const READSTAT_HANDLER_OK = Cint(0) +const READSTAT_HANDLER_ABORT = Cint(1) +const READSTAT_HANDLER_SKIP_VARIABLE = Cint(2) + +@enum ReadStatType::Cint begin + READSTAT_TYPE_STRING = 0 + READSTAT_TYPE_INT8 + READSTAT_TYPE_INT16 + READSTAT_TYPE_INT32 + READSTAT_TYPE_FLOAT + READSTAT_TYPE_DOUBLE + READSTAT_TYPE_STRING_REF +end + +@enum ReadStatTypeClass::Cint begin + READSTAT_TYPE_CLASS_STRING = 0 + READSTAT_TYPE_CLASS_NUMERIC +end + +@enum ReadStatMeasure::Cint begin + READSTAT_MEASURE_UNKNOWN = 0 + READSTAT_MEASURE_NOMINAL + READSTAT_MEASURE_ORDINAL + READSTAT_MEASURE_SCALE +end + +@enum ReadStatAlignment::Cint begin + READSTAT_ALIGNMENT_UNKNOWN = 0 + READSTAT_ALIGNMENT_LEFT + READSTAT_ALIGNMENT_CENTER + READSTAT_ALIGNMENT_RIGHT +end + +@enum ReadStatCompress::Cint begin + READSTAT_COMPRESS_NONE = 0 + READSTAT_COMPRESS_ROWS + READSTAT_COMPRESS_BINARY +end + +@enum ReadStatEndian::Cint begin + READSTAT_ENDIAN_NONE = 0 + READSTAT_ENDIAN_LITTLE + READSTAT_ENDIAN_BIG +end + +@enum ReadStatIOFlags::Cint begin + READSTAT_SEEK_SET = 0 + READSTAT_SEEK_CUR + READSTAT_SEEK_END +end + +@enum ReadStatError::Cint begin + READSTAT_OK = 0 + READSTAT_ERROR_OPEN + READSTAT_ERROR_READ + READSTAT_ERROR_MALLOC + READSTAT_ERROR_USER_ABORT + READSTAT_ERROR_PARSE + READSTAT_ERROR_UNSUPPORTED_COMPRESSION + READSTAT_ERROR_UNSUPPORTED_CHARSET + READSTAT_ERROR_COLUMN_COUNT_MISMATCH + READSTAT_ERROR_ROW_COUNT_MISMATCH + READSTAT_ERROR_ROW_WIDTH_MISMATCH + READSTAT_ERROR_BAD_FORMAT_STRING + READSTAT_ERROR_VALUE_TYPE_MISMATCH + READSTAT_ERROR_WRITE + READSTAT_ERROR_WRITER_NOT_INITIALIZED + READSTAT_ERROR_SEEK + READSTAT_ERROR_CONVERT + READSTAT_ERROR_CONVERT_BAD_STRING + READSTAT_ERROR_CONVERT_SHORT_STRING + READSTAT_ERROR_CONVERT_LONG_STRING + READSTAT_ERROR_NUMERIC_VALUE_IS_OUT_OF_RANGE + READSTAT_ERROR_TAGGED_VALUE_IS_OUT_OF_RANGE + READSTAT_ERROR_STRING_VALUE_IS_TOO_LONG + READSTAT_ERROR_TAGGED_VALUES_NOT_SUPPORTED + READSTAT_ERROR_UNSUPPORTED_FILE_FORMAT_VERSION + READSTAT_ERROR_NAME_BEGINS_WITH_ILLEGAL_CHARACTER + READSTAT_ERROR_NAME_CONTAINS_ILLEGAL_CHARACTER + READSTAT_ERROR_NAME_IS_RESERVED_WORD + READSTAT_ERROR_NAME_IS_TOO_LONG + READSTAT_ERROR_BAD_TIMESTAMP_STRING + READSTAT_ERROR_BAD_FREQUENCY_WEIGHT + READSTAT_ERROR_TOO_MANY_MISSING_VALUE_DEFINITIONS + READSTAT_ERROR_NOTE_IS_TOO_LONG + READSTAT_ERROR_STRING_REFS_NOT_SUPPORTED + READSTAT_ERROR_STRING_REF_IS_REQUIRED + READSTAT_ERROR_ROW_IS_TOO_WIDE_FOR_PAGE + READSTAT_ERROR_TOO_FEW_COLUMNS + READSTAT_ERROR_TOO_MANY_COLUMNS + READSTAT_ERROR_NAME_IS_ZERO_LENGTH + READSTAT_ERROR_BAD_TIMESTAMP_VALUE +end + +function readstat_error_message(error_code::ReadStatError) + unsafe_string(@ccall libreadstat.readstat_error_message(error_code::ReadStatError)::Ptr{Cchar}) +end + +export READSTAT_HANDLER_OK, READSTAT_HANDLER_ABORT, READSTAT_HANDLER_SKIP_VARIABLE, + readstat_error_message + +for enum_type in (ReadStatType, ReadStatTypeClass, ReadStatMeasure, ReadStatAlignment, + ReadStatCompress, ReadStatEndian, ReadStatIOFlags, ReadStatError) + @eval export $(nameof(enum_type)) + for instance in instances(enum_type) + @eval export $(Symbol(instance)) + end +end diff --git a/src/capi/parser.jl b/src/capi/parser.jl new file mode 100644 index 0000000..1aea736 --- /dev/null +++ b/src/capi/parser.jl @@ -0,0 +1,195 @@ +# Parser lifecycle, handler registration, parser configuration, metadata and +# variable accessors, and the parse entry points from readstat.h (v1.1.9). + +export readstat_parser_init, readstat_parser_free, + readstat_set_metadata_handler, readstat_set_note_handler, readstat_set_variable_handler, + readstat_set_fweight_handler, readstat_set_value_handler, readstat_set_value_label_handler, + readstat_set_error_handler, readstat_set_progress_handler, + readstat_set_open_handler, readstat_set_close_handler, readstat_set_seek_handler, + readstat_set_read_handler, readstat_set_update_handler, readstat_set_io_ctx, + readstat_set_file_character_encoding, readstat_set_handler_character_encoding, + readstat_set_row_limit, readstat_set_row_offset, + readstat_parse_dta, readstat_parse_sav, readstat_parse_por, + readstat_parse_sas7bdat, readstat_parse_sas7bcat, readstat_parse_xport, + readstat_get_row_count, readstat_get_var_count, readstat_get_creation_time, + readstat_get_modified_time, readstat_get_file_format_version, + readstat_get_file_format_is_64bit, readstat_get_compression, readstat_get_endianness, + readstat_get_table_name, readstat_get_file_label, readstat_get_file_encoding, + readstat_variable_get_index, readstat_variable_get_index_after_skipping, + readstat_variable_get_name, readstat_variable_get_label, readstat_variable_get_format, + readstat_variable_get_type, readstat_variable_get_type_class, + readstat_variable_get_storage_width, readstat_variable_get_display_width, + readstat_variable_get_measure, readstat_variable_get_alignment, + readstat_variable_get_missing_ranges_count, + readstat_variable_get_missing_range_lo, readstat_variable_get_missing_range_hi + +_string_or_empty(ptr::Cstring) = ptr == C_NULL ? "" : unsafe_string(ptr) +_string_or_empty(ptr::Ptr{Cchar}) = ptr == C_NULL ? "" : unsafe_string(ptr) + +# --------------------------------------------------------------------------- +# Parser lifecycle + +readstat_parser_init() = @ccall libreadstat.readstat_parser_init()::ParserPtr + +function readstat_parser_free(parser::ParserPtr) + @ccall libreadstat.readstat_parser_free(parser::ParserPtr)::Cvoid +end + +# --------------------------------------------------------------------------- +# Handler registration. Each takes a C function pointer (from @cfunction). + +for setter in (:readstat_set_metadata_handler, :readstat_set_note_handler, + :readstat_set_variable_handler, :readstat_set_fweight_handler, + :readstat_set_value_handler, :readstat_set_value_label_handler, + :readstat_set_error_handler, :readstat_set_progress_handler, + :readstat_set_open_handler, :readstat_set_close_handler, + :readstat_set_seek_handler, :readstat_set_read_handler, + :readstat_set_update_handler) + @eval function $setter(parser::ParserPtr, handler::Ptr{Cvoid}) + @ccall libreadstat.$setter(parser::ParserPtr, handler::Ptr{Cvoid})::ReadStatError + end +end + +function readstat_set_io_ctx(parser::ParserPtr, io_ctx::Ptr{Cvoid}) + @ccall libreadstat.readstat_set_io_ctx(parser::ParserPtr, io_ctx::Ptr{Cvoid})::ReadStatError +end + +# --------------------------------------------------------------------------- +# Parser configuration + +# Override the encoding declared in the file (iconv-compatible name); useful +# e.g. for pre-14 Stata files written in a system codepage. +function readstat_set_file_character_encoding(parser::ParserPtr, encoding::AbstractString) + @ccall libreadstat.readstat_set_file_character_encoding(parser::ParserPtr, encoding::Cstring)::ReadStatError +end + +# Encoding delivered to the handlers; defaults to UTF-8. C_NULL disables +# transliteration entirely. +function readstat_set_handler_character_encoding(parser::ParserPtr, encoding::Union{AbstractString,Ptr{Nothing}}) + @ccall libreadstat.readstat_set_handler_character_encoding(parser::ParserPtr, encoding::Cstring)::ReadStatError +end + +function readstat_set_row_limit(parser::ParserPtr, row_limit::Integer) + @ccall libreadstat.readstat_set_row_limit(parser::ParserPtr, row_limit::Clong)::ReadStatError +end + +function readstat_set_row_offset(parser::ParserPtr, row_offset::Integer) + @ccall libreadstat.readstat_set_row_offset(parser::ParserPtr, row_offset::Clong)::ReadStatError +end + +# --------------------------------------------------------------------------- +# Parse entry points. `user_ctx` is passed as `Any` so the ccall roots the +# context object for the duration of the parse. + +for parse_fn in (:readstat_parse_dta, :readstat_parse_sav, :readstat_parse_por, + :readstat_parse_sas7bdat, :readstat_parse_sas7bcat, :readstat_parse_xport) + @eval function $parse_fn(parser::ParserPtr, path::AbstractString, user_ctx) + @ccall libreadstat.$parse_fn(parser::ParserPtr, path::Cstring, user_ctx::Any)::ReadStatError + end +end + +# --------------------------------------------------------------------------- +# Metadata accessors (valid inside the metadata handler) + +function readstat_get_row_count(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_row_count(metadata::MetadataPtr)::Cint +end + +function readstat_get_var_count(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_var_count(metadata::MetadataPtr)::Cint +end + +function readstat_get_creation_time(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_creation_time(metadata::MetadataPtr)::Int64 +end + +function readstat_get_modified_time(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_modified_time(metadata::MetadataPtr)::Int64 +end + +function readstat_get_file_format_version(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_file_format_version(metadata::MetadataPtr)::Cint +end + +function readstat_get_file_format_is_64bit(metadata::MetadataPtr) + (@ccall libreadstat.readstat_get_file_format_is_64bit(metadata::MetadataPtr)::Cint) != 0 +end + +function readstat_get_compression(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_compression(metadata::MetadataPtr)::ReadStatCompress +end + +function readstat_get_endianness(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_endianness(metadata::MetadataPtr)::ReadStatEndian +end + +function readstat_get_table_name(metadata::MetadataPtr) + _string_or_empty(@ccall libreadstat.readstat_get_table_name(metadata::MetadataPtr)::Cstring) +end + +function readstat_get_file_label(metadata::MetadataPtr) + _string_or_empty(@ccall libreadstat.readstat_get_file_label(metadata::MetadataPtr)::Cstring) +end + +function readstat_get_file_encoding(metadata::MetadataPtr) + _string_or_empty(@ccall libreadstat.readstat_get_file_encoding(metadata::MetadataPtr)::Cstring) +end + +# --------------------------------------------------------------------------- +# Variable accessors (valid inside the variable/value handlers) + +function readstat_variable_get_index(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_index(variable::VariablePtr)::Cint +end + +function readstat_variable_get_index_after_skipping(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_index_after_skipping(variable::VariablePtr)::Cint +end + +function readstat_variable_get_name(variable::VariablePtr) + _string_or_empty(@ccall libreadstat.readstat_variable_get_name(variable::VariablePtr)::Cstring) +end + +function readstat_variable_get_label(variable::VariablePtr) + _string_or_empty(@ccall libreadstat.readstat_variable_get_label(variable::VariablePtr)::Cstring) +end + +function readstat_variable_get_format(variable::VariablePtr) + _string_or_empty(@ccall libreadstat.readstat_variable_get_format(variable::VariablePtr)::Cstring) +end + +function readstat_variable_get_type(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_type(variable::VariablePtr)::ReadStatType +end + +function readstat_variable_get_type_class(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_type_class(variable::VariablePtr)::ReadStatTypeClass +end + +function readstat_variable_get_storage_width(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_storage_width(variable::VariablePtr)::Csize_t +end + +function readstat_variable_get_display_width(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_display_width(variable::VariablePtr)::Cint +end + +function readstat_variable_get_measure(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_measure(variable::VariablePtr)::ReadStatMeasure +end + +function readstat_variable_get_alignment(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_alignment(variable::VariablePtr)::ReadStatAlignment +end + +function readstat_variable_get_missing_ranges_count(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_missing_ranges_count(variable::VariablePtr)::Cint +end + +function readstat_variable_get_missing_range_lo(variable::VariablePtr, i::Integer) + @ccall libreadstat.readstat_variable_get_missing_range_lo(variable::VariablePtr, i::Cint)::ReadStatValue +end + +function readstat_variable_get_missing_range_hi(variable::VariablePtr, i::Integer) + @ccall libreadstat.readstat_variable_get_missing_range_hi(variable::VariablePtr, i::Cint)::ReadStatValue +end diff --git a/src/capi/value.jl b/src/capi/value.jl new file mode 100644 index 0000000..90493be --- /dev/null +++ b/src/capi/value.jl @@ -0,0 +1,109 @@ +# The readstat_value_t mirror and its accessors. +# +# readstat_value_t is the only C struct this package mirrors: the C API passes +# and returns it by value, so an opaque pointer is not an option. Every other +# C struct is handled through an opaque pointer plus the C getter functions. + +export ReadStatValue, + ParserPtr, MetadataPtr, VariablePtr, LabelSetPtr, SchemaPtr, WriterPtr, StringRefPtr, + readstat_value_type, readstat_value_type_class, readstat_type_class, + readstat_value_is_missing, readstat_value_is_system_missing, + readstat_value_is_tagged_missing, readstat_value_is_defined_missing, + readstat_value_tag, readstat_int8_value, readstat_int16_value, readstat_int32_value, + readstat_float_value, readstat_double_value, readstat_string_value + +# Opaque C struct tags; only ever used as Ptr{...} type parameters. +abstract type readstat_parser_s end +abstract type readstat_metadata_s end +abstract type readstat_variable_s end +abstract type readstat_label_set_s end +abstract type readstat_schema_s end +abstract type readstat_writer_s end +abstract type readstat_string_ref_s end + +const ParserPtr = Ptr{readstat_parser_s} +const MetadataPtr = Ptr{readstat_metadata_s} +const VariablePtr = Ptr{readstat_variable_s} +const LabelSetPtr = Ptr{readstat_label_set_s} +const SchemaPtr = Ptr{readstat_schema_s} +const WriterPtr = Ptr{readstat_writer_s} +const StringRefPtr = Ptr{readstat_string_ref_s} + +# Layout must match readstat.h's readstat_value_t under the jll's compiler: +# 8-byte value union, Cint type at offset 8, char tag at 12, then two +# `unsigned int:1` bitfields. Where the bitfields land depends on the +# platform's bitfield ABI: Itanium/SysV GCC packs them into byte 13 (struct +# size 16), while MinGW GCC defaults to MS-compatible bitfield layout, which +# starts a fresh 4-byte-aligned unsigned int unit at offset 16 (struct size +# 24). The `bits` field is never decoded in Julia — the is_missing/tag +# accessors below are the only sanctioned way in. test/test_abi.jl checks +# this layout behaviorally against the C library. +struct ReadStatValue + v::Int64 + type::ReadStatType + tag::Cchar + @static if Sys.iswindows() + bits::Cuint + else + bits::UInt8 + end +end + +function readstat_value_type(value::ReadStatValue) + @ccall libreadstat.readstat_value_type(value::ReadStatValue)::ReadStatType +end + +function readstat_value_type_class(value::ReadStatValue) + @ccall libreadstat.readstat_value_type_class(value::ReadStatValue)::ReadStatTypeClass +end + +function readstat_type_class(type::ReadStatType) + @ccall libreadstat.readstat_type_class(type::ReadStatType)::ReadStatTypeClass +end + +# `variable` may be a VariablePtr or C_NULL; with C_NULL only system/tagged +# missingness is considered. +function readstat_value_is_missing(value::ReadStatValue, variable::Ptr) + (@ccall libreadstat.readstat_value_is_missing(value::ReadStatValue, variable::VariablePtr)::Cint) != 0 +end + +function readstat_value_is_system_missing(value::ReadStatValue) + (@ccall libreadstat.readstat_value_is_system_missing(value::ReadStatValue)::Cint) != 0 +end + +function readstat_value_is_tagged_missing(value::ReadStatValue) + (@ccall libreadstat.readstat_value_is_tagged_missing(value::ReadStatValue)::Cint) != 0 +end + +function readstat_value_is_defined_missing(value::ReadStatValue, variable::Ptr) + (@ccall libreadstat.readstat_value_is_defined_missing(value::ReadStatValue, variable::VariablePtr)::Cint) != 0 +end + +function readstat_value_tag(value::ReadStatValue) + Char(@ccall libreadstat.readstat_value_tag(value::ReadStatValue)::Cchar) +end + +function readstat_int8_value(value::ReadStatValue) + @ccall libreadstat.readstat_int8_value(value::ReadStatValue)::Int8 +end + +function readstat_int16_value(value::ReadStatValue) + @ccall libreadstat.readstat_int16_value(value::ReadStatValue)::Int16 +end + +function readstat_int32_value(value::ReadStatValue) + @ccall libreadstat.readstat_int32_value(value::ReadStatValue)::Int32 +end + +function readstat_float_value(value::ReadStatValue) + @ccall libreadstat.readstat_float_value(value::ReadStatValue)::Float32 +end + +function readstat_double_value(value::ReadStatValue) + @ccall libreadstat.readstat_double_value(value::ReadStatValue)::Float64 +end + +# Returns C_NULL-able pointer; callers decide how to treat NULL vs "". +function readstat_string_value(value::ReadStatValue) + @ccall libreadstat.readstat_string_value(value::ReadStatValue)::Cstring +end diff --git a/test/runtests.jl b/test/runtests.jl index 391ebaa..c9b0edc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using TestItemRunner +include("test_abi.jl") include("test_readstat.jl") @run_package_tests diff --git a/test/test_abi.jl b/test/test_abi.jl new file mode 100644 index 0000000..21d9c3e --- /dev/null +++ b/test/test_abi.jl @@ -0,0 +1,56 @@ +@testitem "C ABI: ReadStatValue layout" begin + using ReadStat: CAPI + using ReadStat_jll: libreadstat + + # readstat_value_t: 8-byte value union, Cint type at 8, char tag at 12, + # then two 1-bit bitfields. SysV GCC packs the bitfields into byte 13 + # (size 16); MinGW GCC uses MS bitfield layout, placing a fresh unsigned + # int unit at offset 16 (size 24). + @test sizeof(CAPI.ReadStatValue) == (Sys.iswindows() ? 24 : 16) + @test fieldoffset(CAPI.ReadStatValue, findfirst(==(:type), fieldnames(CAPI.ReadStatValue))) == 8 + @test fieldoffset(CAPI.ReadStatValue, findfirst(==(:tag), fieldnames(CAPI.ReadStatValue))) == 12 + @test fieldoffset(CAPI.ReadStatValue, findfirst(==(:bits), fieldnames(CAPI.ReadStatValue))) == + (Sys.iswindows() ? 16 : 13) + + # Behavioral check of the by-value ABI in both directions. Missing ranges + # declared on a writer variable come back as readstat_value_t BY VALUE from + # readstat_variable_get_missing_range_lo/_hi, and are then passed BY VALUE + # into the accessor functions. If the struct layout disagreed with the C + # library, these reads would come back as garbage rather than the exact + # doubles inserted below. + writer = @ccall libreadstat.readstat_writer_init()::CAPI.WriterPtr + @test writer != C_NULL + try + var = @ccall libreadstat.readstat_add_variable( + writer::CAPI.WriterPtr, "x"::Cstring, + CAPI.READSTAT_TYPE_DOUBLE::CAPI.ReadStatType, 8::Csize_t)::CAPI.VariablePtr + @test var != C_NULL + + @test (@ccall libreadstat.readstat_variable_add_missing_double_range( + var::CAPI.VariablePtr, (-99.5)::Cdouble, (-90.25)::Cdouble)::CAPI.ReadStatError) == + CAPI.READSTAT_OK + @test (@ccall libreadstat.readstat_variable_add_missing_double_value( + var::CAPI.VariablePtr, 999.0::Cdouble)::CAPI.ReadStatError) == CAPI.READSTAT_OK + + @test CAPI.readstat_variable_get_missing_ranges_count(var) == 2 + + lo = CAPI.readstat_variable_get_missing_range_lo(var, 0) + hi = CAPI.readstat_variable_get_missing_range_hi(var, 0) + @test CAPI.readstat_value_type(lo) == CAPI.READSTAT_TYPE_DOUBLE + @test CAPI.readstat_double_value(lo) == -99.5 + @test CAPI.readstat_double_value(hi) == -90.25 + @test CAPI.readstat_value_tag(lo) == '\0' + @test !CAPI.readstat_value_is_system_missing(lo) + @test !CAPI.readstat_value_is_tagged_missing(lo) + + # A singleton "range" (lo == hi) carrying the flag bits through the + # by-value path: 999.0 is defined missing for this variable. + v = CAPI.readstat_variable_get_missing_range_lo(var, 1) + @test CAPI.readstat_double_value(v) == 999.0 + @test CAPI.readstat_value_is_defined_missing(v, var) + @test CAPI.readstat_value_is_missing(v, var) + @test !CAPI.readstat_value_is_missing(v, C_NULL) + finally + @ccall libreadstat.readstat_writer_free(writer::CAPI.WriterPtr)::Cvoid + end +end From 8589410ab2389ccc3f424ac2498da860f133f526 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 22:59:23 -0700 Subject: [PATCH 02/14] Replace ReadStatDataFrame with the metadata-rich ReadStatTable Reading now produces a ReadStatTable: concretely typed DataValueVector columns reached by index or name, a ReadStatMeta with the full file-level metadata the C library exposes (creation and modified time, format version, 64-bit flag, compression, endianness, table name, file label, encoding, notes, fweight, and all value-label sets), and a ReadStatVarMeta per column (label, format, raw type and type class, value-label set name, widths, measure, alignment, and SPSS missing-value ranges). Value-label sets are parsed into typed dictionaries keyed by Int32/Float64/String, with labeled tagged missing values keyed by their tag Char. Parsing goes through typed column buffers (raw Vector plus Bool NA mask, grouped by element type and dispatched via a small typecode branch), so the value path is type-stable without touching any DataValues internals; the buffers are wrapped into DataValueVectors only at the end. The NA mask starts all-missing over defined backing storage, so cells the parser never delivers surface as NA instead of undefined memory. Handler @cfunction pointers are created once in __init__ and shared; every handler captures Julia exceptions and aborts the parse instead of unwinding through C. ReadStatDataFrame remains as a deprecated alias whose 1.x field access is reconstructed through getproperty shims with deprecation warnings, so existing consumers such as StatFiles.jl keep working until their follow-up. Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 217 +++------------------------------------- src/deprecated.jl | 90 +++++++++++++++++ src/reader/columns.jl | 124 +++++++++++++++++++++++ src/reader/context.jl | 14 +++ src/reader/handlers.jl | 173 ++++++++++++++++++++++++++++++++ src/reader/metadata.jl | 61 +++++++++++ src/reader/read.jl | 92 +++++++++++++++++ src/reader/table.jl | 129 ++++++++++++++++++++++++ test/runtests.jl | 3 +- test/test_deprecated.jl | 32 ++++++ test/test_read.jl | 59 +++++++++++ test/test_readstat.jl | 35 ------- 12 files changed, 790 insertions(+), 239 deletions(-) create mode 100644 src/deprecated.jl create mode 100644 src/reader/columns.jl create mode 100644 src/reader/context.jl create mode 100644 src/reader/handlers.jl create mode 100644 src/reader/metadata.jl create mode 100644 src/reader/read.jl create mode 100644 src/reader/table.jl create mode 100644 test/test_deprecated.jl create mode 100644 test/test_read.jl delete mode 100644 test/test_readstat.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index edb09d4..8a49990 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -1,10 +1,11 @@ module ReadStat -using DataValues: DataValueVector -import DataValues +using DataValues: DataValues, DataValueVector using Dates -export ReadStatDataFrame, read_dta, read_sav, read_por, read_sas7bdat, read_xport +export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, + read_dta, read_sav, read_por, read_sas7bdat, read_xport, + filemetadata, varmetadata, valuelabels public CAPI @@ -37,207 +38,17 @@ const READSTAT_TYPES = (String, Int8, Int16, Int32, Float32, Float64, String) jltype(t::ReadStatType) = READSTAT_TYPES[Int(t) + 1] -############################################################################## -## -## Result type -## -############################################################################## +include("reader/metadata.jl") +include("reader/columns.jl") +include("reader/context.jl") +include("reader/handlers.jl") +include("reader/table.jl") +include("reader/read.jl") +include("deprecated.jl") -mutable struct ReadStatDataFrame - data::Vector{Any} - headers::Vector{Symbol} - types::Vector{DataType} - labels::Vector{String} - formats::Vector{String} - storagewidths::Vector{Csize_t} - measures::Vector{Cint} - alignments::Vector{Cint} - val_label_keys::Vector{String} - val_label_dict::Dict{String, Dict{Any,String}} - rows::Int - columns::Int - filelabel::String - timestamp::DateTime - format::Clong - types_as_int::Vector{Cint} - hasmissings::Vector{Bool} - - ReadStatDataFrame() = - new(Any[], Symbol[], DataType[], String[], String[], Csize_t[], Cint[], Cint[], - String[], Dict{String, Dict{Any,String}}(), 0, 0, "", Dates.unix2datetime(0), 0, Cint[], Bool[]) -end - -############################################################################## -## -## Parse handlers -## -############################################################################## - -function handle_metadata!(metadata::MetadataPtr, ds_ptr::Ptr{ReadStatDataFrame}) - ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame - ds.filelabel = readstat_get_file_label(metadata) - ds.timestamp = Dates.unix2datetime(readstat_get_modified_time(metadata)) - ds.format = readstat_get_file_format_version(metadata) - ds.rows = readstat_get_row_count(metadata) - ds.columns = readstat_get_var_count(metadata) - return READSTAT_HANDLER_OK -end - -function handle_variable!(var_index::Cint, variable::VariablePtr, - val_label::Cstring, ds_ptr::Ptr{ReadStatDataFrame}) - ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame - missing_count = readstat_variable_get_missing_ranges_count(variable) - - push!(ds.val_label_keys, val_label == C_NULL ? "" : unsafe_string(val_label)) - push!(ds.headers, Symbol(readstat_variable_get_name(variable))) - push!(ds.labels, readstat_variable_get_label(variable)) - push!(ds.formats, readstat_variable_get_format(variable)) - ctype = readstat_variable_get_type(variable) - T = jltype(ctype) - push!(ds.types, T) - push!(ds.types_as_int, Cint(Int(ctype))) - push!(ds.hasmissings, missing_count > 0) - # XPORT and POR report an unknown row count as -1. - if ds.rows >= 0 - push!(ds.data, DataValueVector{T}(Vector{T}(undef, ds.rows), fill(false, ds.rows))) - else - push!(ds.data, DataValueVector{T}(Vector{T}(undef, 0), fill(false, 0))) - end - push!(ds.storagewidths, readstat_variable_get_storage_width(variable)) - push!(ds.measures, Cint(Int(readstat_variable_get_measure(variable)))) - push!(ds.alignments, Cint(Int(readstat_variable_get_alignment(variable)))) - - return READSTAT_HANDLER_OK -end - -function as_native(val::ReadStatValue) - t = readstat_value_type(val) - if t == READSTAT_TYPE_INT8 - return readstat_int8_value(val) - elseif t == READSTAT_TYPE_INT16 - return readstat_int16_value(val) - elseif t == READSTAT_TYPE_INT32 - return readstat_int32_value(val) - elseif t == READSTAT_TYPE_FLOAT - return readstat_float_value(val) - elseif t == READSTAT_TYPE_DOUBLE - return readstat_double_value(val) - else - ptr = readstat_string_value(val) - return ptr == C_NULL ? "" : unsafe_string(ptr) - end -end - -function handle_value!(obs_index::Cint, variable::VariablePtr, - value::ReadStatValue, ds_ptr::Ptr{ReadStatDataFrame}) - ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame - var_index = readstat_variable_get_index(variable) + 1 - data = ds.data - - ismissing = if @inbounds(ds.hasmissings[var_index]) - readstat_value_is_missing(value, variable) - else - readstat_value_is_missing(value, C_NULL) - end - - col = data[var_index] - - if ismissing - if obs_index < length(col) - DataValues.unsafe_setindex_isna!(col, true, obs_index + 1) - else - push!(col, DataValues.NA) - end - else - readfield!(col, obs_index + 1, value) - end - - return READSTAT_HANDLER_OK +function __init__() + _init_cfunctions() + return nothing end -function readfield!(dest::DataValueVector{String}, row, val::ReadStatValue) - ptr = readstat_string_value(val) - - if row <= length(dest) - if ptr != C_NULL - @inbounds DataValues.unsafe_setindex_value!(dest, unsafe_string(ptr), row) - end - elseif row == length(dest) + 1 - push!(dest, ptr != C_NULL ? unsafe_string(ptr) : "") - else - throw(ArgumentError("illegal row index: $row")) - end -end - -for (T, value_fn) in ((Int8, :readstat_int8_value), - (Int16, :readstat_int16_value), - (Int32, :readstat_int32_value), - (Float32, :readstat_float_value), - (Float64, :readstat_double_value)) - @eval function readfield!(dest::DataValueVector{$T}, row, val::ReadStatValue) - _val = $value_fn(val) - if row <= length(dest) - @inbounds DataValues.unsafe_setindex_value!(dest, _val, row) - elseif row == length(dest) + 1 - push!(dest, _val) - else - throw(ArgumentError("illegal row index: $row")) - end - end -end - -function handle_value_label!(val_labels::Cstring, value::ReadStatValue, label::Cstring, - ds_ptr::Ptr{ReadStatDataFrame}) - val_labels != C_NULL || return READSTAT_HANDLER_OK - ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame - dict = get!(ds.val_label_dict, unsafe_string(val_labels), Dict{Any,String}()) - dict[as_native(value)] = unsafe_string(label) - - return READSTAT_HANDLER_OK -end - -############################################################################## -## -## Parse driver -## -############################################################################## - -function Parser() - parser = readstat_parser_init() - meta_fxn = @cfunction(handle_metadata!, Cint, (MetadataPtr, Ptr{ReadStatDataFrame})) - var_fxn = @cfunction(handle_variable!, Cint, (Cint, VariablePtr, Cstring, Ptr{ReadStatDataFrame})) - val_fxn = @cfunction(handle_value!, Cint, (Cint, VariablePtr, ReadStatValue, Ptr{ReadStatDataFrame})) - label_fxn = @cfunction(handle_value_label!, Cint, (Cstring, ReadStatValue, Cstring, Ptr{ReadStatDataFrame})) - readstat_set_metadata_handler(parser, meta_fxn) - readstat_set_variable_handler(parser, var_fxn) - readstat_set_value_handler(parser, val_fxn) - readstat_set_value_label_handler(parser, label_fxn) - return parser -end - -readstat_parse(parser, path, ::Val{:dta}, ctx) = CAPI.readstat_parse_dta(parser, path, ctx) -readstat_parse(parser, path, ::Val{:sav}, ctx) = CAPI.readstat_parse_sav(parser, path, ctx) -readstat_parse(parser, path, ::Val{:por}, ctx) = CAPI.readstat_parse_por(parser, path, ctx) -readstat_parse(parser, path, ::Val{:sas7bdat}, ctx) = CAPI.readstat_parse_sas7bdat(parser, path, ctx) -readstat_parse(parser, path, ::Val{:xport}, ctx) = CAPI.readstat_parse_xport(parser, path, ctx) - -function read_data_file(filename::AbstractString, filetype::Val) - ds = ReadStatDataFrame() - parser = Parser() - local retval - try - retval = readstat_parse(parser, filename, filetype, ds) - finally - readstat_parser_free(parser) - end - retval == READSTAT_OK || error("Error parsing $filename: $(readstat_error_message(retval))") - return ds -end - -read_dta(filename::AbstractString) = read_data_file(filename, Val(:dta)) -read_sav(filename::AbstractString) = read_data_file(filename, Val(:sav)) -read_por(filename::AbstractString) = read_data_file(filename, Val(:por)) -read_sas7bdat(filename::AbstractString) = read_data_file(filename, Val(:sas7bdat)) -read_xport(filename::AbstractString) = read_data_file(filename, Val(:xport)) - end # module ReadStat diff --git a/src/deprecated.jl b/src/deprecated.jl new file mode 100644 index 0000000..44e4f0d --- /dev/null +++ b/src/deprecated.jl @@ -0,0 +1,90 @@ +# 1.x compatibility shims, to be removed in 3.0. +# +# ReadStat 1.x returned a `ReadStatDataFrame` whose data and metadata were +# reached through 15 public fields. The old name aliases the new table type, +# and property access reconstructs the old field values with a deprecation +# warning, so existing consumers (StatFiles.jl in particular uses `.data` and +# `.headers`) keep working against 2.0. + +""" + ReadStatDataFrame + +Deprecated alias for [`ReadStatTable`](@ref). The 1.x field access +(`df.data`, `df.headers`, ...) still works but emits deprecation warnings; +use the 2.0 accessors instead. +""" +const ReadStatDataFrame = ReadStatTable + +function _dep(old::Symbol, instead::AbstractString) + Base.depwarn("`ReadStatDataFrame.$old` is deprecated; use $instead instead.", old) +end + +function Base.getproperty(tbl::ReadStatTable, s::Symbol) + if s === :cols || s === :names || s === :lookup || s === :meta || s === :colmeta || s === :tags + return getfield(tbl, s) + elseif s === :data + _dep(s, "`tbl[i]` / `tbl[:name]`") + return Any[c for c in getfield(tbl, :cols)] + elseif s === :headers + _dep(s, "`names(tbl)`") + return copy(getfield(tbl, :names)) + elseif s === :types + _dep(s, "`eltype(eltype(tbl[i]))`") + return DataType[jltype(m.type) for m in getfield(tbl, :colmeta)] + elseif s === :labels + _dep(s, "`varmetadata(tbl, i).label`") + return String[m.label for m in getfield(tbl, :colmeta)] + elseif s === :formats + _dep(s, "`varmetadata(tbl, i).format`") + return String[m.format for m in getfield(tbl, :colmeta)] + elseif s === :storagewidths + _dep(s, "`varmetadata(tbl, i).storage_width`") + return Csize_t[m.storage_width for m in getfield(tbl, :colmeta)] + elseif s === :measures + _dep(s, "`varmetadata(tbl, i).measure`") + return Cint[Cint(Int(m.measure)) for m in getfield(tbl, :colmeta)] + elseif s === :alignments + _dep(s, "`varmetadata(tbl, i).alignment`") + return Cint[Cint(Int(m.alignment)) for m in getfield(tbl, :colmeta)] + elseif s === :val_label_keys + _dep(s, "`varmetadata(tbl, i).vallabel`") + return String[string(m.vallabel) for m in getfield(tbl, :colmeta)] + elseif s === :val_label_dict + _dep(s, "`valuelabels(tbl, col)` / `filemetadata(tbl).value_labels`") + return Dict{String,Dict{Any,String}}( + string(k) => Dict{Any,String}(kk => vv for (kk, vv) in v) + for (k, v) in getfield(tbl, :meta).value_labels) + elseif s === :rows + _dep(s, "`size(tbl, 1)` or `filemetadata(tbl).row_count`") + return getfield(tbl, :meta).row_count + elseif s === :columns + _dep(s, "`size(tbl, 2)`") + return getfield(tbl, :meta).var_count + elseif s === :filelabel + _dep(s, "`filemetadata(tbl).file_label`") + return getfield(tbl, :meta).file_label + elseif s === :timestamp + _dep(s, "`filemetadata(tbl).modified_time`") + return getfield(tbl, :meta).modified_time + elseif s === :format + _dep(s, "`filemetadata(tbl).file_format_version`") + return Clong(getfield(tbl, :meta).file_format_version) + elseif s === :types_as_int + _dep(s, "`varmetadata(tbl, i).type`") + return Cint[Cint(Int(m.type)) for m in getfield(tbl, :colmeta)] + elseif s === :hasmissings + _dep(s, "`varmetadata(tbl, i).missing_ranges`") + return Bool[!isempty(m.missing_ranges) for m in getfield(tbl, :colmeta)] + else + # Fall through for reflection etc. + return getfield(tbl, s) + end +end + +function Base.propertynames(::ReadStatTable, private::Bool=false) + private ? (fieldnames(ReadStatTable)..., DEPRECATED_PROPERTIES...) : DEPRECATED_PROPERTIES +end + +const DEPRECATED_PROPERTIES = (:data, :headers, :types, :labels, :formats, :storagewidths, + :measures, :alignments, :val_label_keys, :val_label_dict, :rows, :columns, :filelabel, + :timestamp, :format, :types_as_int, :hasmissings) diff --git a/src/reader/columns.jl b/src/reader/columns.jl new file mode 100644 index 0000000..fd5d7c6 --- /dev/null +++ b/src/reader/columns.jl @@ -0,0 +1,124 @@ +# Typed column buffers filled by the value handler. +# +# Columns are parsed into raw `Vector{T}` + `Vector{Bool}` NA-mask pairs and +# only wrapped into `DataValueVector`s once parsing is done. This keeps the +# hot path free of any DataValues internals (whose representation may change) +# and lets a future multi-parser run fill disjoint row regions of shared +# buffers from several threads — plain `Vector` writes to disjoint indices +# are thread-safe, which would not hold for a shared bit-packed mask. +# +# The mask starts all-`true` (every cell missing until the C library delivers +# it), so cells the parser never emits surface as NA rather than as undefined +# memory. String buffers are backed by `""` so every slot is always a defined +# object. + +struct ColumnBuf{T} + values::Vector{T} + isna::Vector{Bool} +end + +newbuf(::Type{T}, n::Int) where {T} = ColumnBuf{T}(Vector{T}(undef, n), fill(true, n)) +newbuf(::Type{String}, n::Int) = ColumnBuf{String}(fill("", n), fill(true, n)) + +# Column storage grouped by element type, so every access from the value +# handler goes through a small typecode branch into fully type-stable code +# instead of a `Vector{Any}` of columns. `slots[i]` maps the i-th kept column +# (C `index_after_skipping` plus one) to `(typecode, index)` within the +# matching typed vector. +struct TypedColumns + strings::Vector{ColumnBuf{String}} + int8s::Vector{ColumnBuf{Int8}} + int16s::Vector{ColumnBuf{Int16}} + int32s::Vector{ColumnBuf{Int32}} + floats::Vector{ColumnBuf{Float32}} + doubles::Vector{ColumnBuf{Float64}} + slots::Vector{Tuple{UInt8,Int}} +end + +TypedColumns() = TypedColumns(ColumnBuf{String}[], ColumnBuf{Int8}[], ColumnBuf{Int16}[], + ColumnBuf{Int32}[], ColumnBuf{Float32}[], ColumnBuf{Float64}[], Tuple{UInt8,Int}[]) + +const CODE_STRING = 0x01 +const CODE_INT8 = 0x02 +const CODE_INT16 = 0x03 +const CODE_INT32 = 0x04 +const CODE_FLOAT = 0x05 +const CODE_DOUBLE = 0x06 + +# `n` is the preallocated length; pass 0 when the row count is unknown and +# the buffers grow row by row instead. +function addcolumn!(cols::TypedColumns, T::DataType, n::Int) + if T === String + push!(cols.strings, newbuf(String, n)) + push!(cols.slots, (CODE_STRING, length(cols.strings))) + elseif T === Int8 + push!(cols.int8s, newbuf(Int8, n)) + push!(cols.slots, (CODE_INT8, length(cols.int8s))) + elseif T === Int16 + push!(cols.int16s, newbuf(Int16, n)) + push!(cols.slots, (CODE_INT16, length(cols.int16s))) + elseif T === Int32 + push!(cols.int32s, newbuf(Int32, n)) + push!(cols.slots, (CODE_INT32, length(cols.int32s))) + elseif T === Float32 + push!(cols.floats, newbuf(Float32, n)) + push!(cols.slots, (CODE_FLOAT, length(cols.floats))) + elseif T === Float64 + push!(cols.doubles, newbuf(Float64, n)) + push!(cols.slots, (CODE_DOUBLE, length(cols.doubles))) + else + throw(ArgumentError("unsupported column type $T")) + end + return cols +end + +@inline function setvalue!(buf::ColumnBuf{T}, row::Int, v::T) where {T} + n = length(buf.values) + if row <= n + @inbounds buf.values[row] = v + @inbounds buf.isna[row] = false + elseif row == n + 1 + push!(buf.values, v) + push!(buf.isna, false) + else + throw(ArgumentError("out-of-order row index $row for column of length $n")) + end + return buf +end + +@inline function setmissing!(buf::ColumnBuf{T}, row::Int) where {T} + n = length(buf.values) + if row <= n + @inbounds buf.isna[row] = true + elseif row == n + 1 + push!(buf.values, _navalue(T)) + push!(buf.isna, true) + else + throw(ArgumentError("out-of-order row index $row for column of length $n")) + end + return buf +end + +_navalue(::Type{T}) where {T<:Number} = zero(T) +_navalue(::Type{String}) = "" + +# Wrap a finished buffer without copying. The constructor takes ownership of +# both vectors; nothing may touch the ColumnBuf afterwards. +finalize_column(buf::ColumnBuf{T}) where {T} = DataValueVector{T}(buf.values, buf.isna) + +function finalize_column(cols::TypedColumns, i::Int) + code, slot = cols.slots[i] + if code == CODE_STRING + finalize_column(cols.strings[slot]) + elseif code == CODE_INT8 + finalize_column(cols.int8s[slot]) + elseif code == CODE_INT16 + finalize_column(cols.int16s[slot]) + elseif code == CODE_INT32 + finalize_column(cols.int32s[slot]) + elseif code == CODE_FLOAT + finalize_column(cols.floats[slot]) + else + finalize_column(cols.doubles[slot]) + end +end diff --git a/src/reader/context.jl b/src/reader/context.jl new file mode 100644 index 0000000..76a2526 --- /dev/null +++ b/src/reader/context.jl @@ -0,0 +1,14 @@ +# Per-parse state. One ParseContext per parser; nothing here is shared, so a +# future multi-parser run needs no synchronization beyond disjoint buffers. + +mutable struct ParseContext + meta::ReadStatMeta + names::Vector{Symbol} + varmeta::Vector{ReadStatVarMeta} + cols::TypedColumns + # Exception thrown by a handler, captured so it never unwinds through C; + # rethrown by the parse driver after readstat_parse returns. + err::Union{Nothing,Tuple{Any,Any}} +end + +ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), nothing) diff --git a/src/reader/handlers.jl b/src/reader/handlers.jl new file mode 100644 index 0000000..0fae0e5 --- /dev/null +++ b/src/reader/handlers.jl @@ -0,0 +1,173 @@ +# Parse handler bodies. All of them are top-level functions whose @cfunction +# pointers are created once in __init__ and shared by every parse — per-parse +# state lives exclusively in the ParseContext passed through user_ctx. +# +# Every handler body is wrapped in try/catch: a Julia exception must never +# unwind through the C parser, so it is captured on the context and the +# handler returns READSTAT_HANDLER_ABORT; the parse driver rethrows it once +# readstat_parse has returned (with READSTAT_ERROR_USER_ABORT). + +function handle_metadata(metadata::MetadataPtr, ctx::Any)::Cint + pc = ctx::ParseContext + try + m = pc.meta + m.row_count = readstat_get_row_count(metadata) + m.var_count = readstat_get_var_count(metadata) + m.creation_time = Dates.unix2datetime(readstat_get_creation_time(metadata)) + m.modified_time = Dates.unix2datetime(readstat_get_modified_time(metadata)) + m.file_format_version = readstat_get_file_format_version(metadata) + m.is_64bit = readstat_get_file_format_is_64bit(metadata) + m.compression = readstat_get_compression(metadata) + m.endianness = readstat_get_endianness(metadata) + m.table_name = readstat_get_table_name(metadata) + m.file_label = readstat_get_file_label(metadata) + m.file_encoding = readstat_get_file_encoding(metadata) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function _missing_ranges(variable::VariablePtr) + n = readstat_variable_get_missing_ranges_count(variable) + ranges = Vector{Tuple{Any,Any}}(undef, n) + for i in 0:(n - 1) + lo = readstat_variable_get_missing_range_lo(variable, i) + hi = readstat_variable_get_missing_range_hi(variable, i) + ranges[i + 1] = (_native_value(lo), _native_value(hi)) + end + return ranges +end + +function _native_value(value::ReadStatValue) + t = readstat_value_type(value) + if t == READSTAT_TYPE_INT8 + readstat_int8_value(value) + elseif t == READSTAT_TYPE_INT16 + readstat_int16_value(value) + elseif t == READSTAT_TYPE_INT32 + readstat_int32_value(value) + elseif t == READSTAT_TYPE_FLOAT + readstat_float_value(value) + elseif t == READSTAT_TYPE_DOUBLE + readstat_double_value(value) + else + ptr = readstat_string_value(value) + ptr == C_NULL ? "" : unsafe_string(ptr) + end +end + +function handle_variable(index::Cint, variable::VariablePtr, val_labels::Cstring, ctx::Any)::Cint + pc = ctx::ParseContext + try + t = readstat_variable_get_type(variable) + vm = ReadStatVarMeta( + Symbol(readstat_variable_get_name(variable)), + readstat_variable_get_label(variable), + readstat_variable_get_format(variable), + t, + readstat_variable_get_type_class(variable), + val_labels == C_NULL ? Symbol("") : Symbol(unsafe_string(val_labels)), + Int(readstat_variable_get_storage_width(variable)), + Int(readstat_variable_get_display_width(variable)), + readstat_variable_get_measure(variable), + readstat_variable_get_alignment(variable), + _missing_ranges(variable)) + push!(pc.varmeta, vm) + push!(pc.names, vm.name) + addcolumn!(pc.cols, jltype(t), max(pc.meta.row_count, 0)) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function handle_value(obs_index::Cint, variable::VariablePtr, value::ReadStatValue, ctx::Any)::Cint + pc = ctx::ParseContext + try + idx = readstat_variable_get_index_after_skipping(variable) + 1 + row = Int(obs_index) + 1 + cols = pc.cols + code, slot = @inbounds cols.slots[idx] + miss = readstat_value_is_missing(value, variable) + + if code == CODE_STRING + buf = @inbounds cols.strings[slot] + if miss + setmissing!(buf, row) + else + ptr = readstat_string_value(value) + setvalue!(buf, row, ptr == C_NULL ? "" : unsafe_string(ptr)) + end + elseif code == CODE_INT8 + buf = @inbounds cols.int8s[slot] + miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_int8_value(value)) + elseif code == CODE_INT16 + buf = @inbounds cols.int16s[slot] + miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_int16_value(value)) + elseif code == CODE_INT32 + buf = @inbounds cols.int32s[slot] + miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_int32_value(value)) + elseif code == CODE_FLOAT + buf = @inbounds cols.floats[slot] + miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_float_value(value)) + else + buf = @inbounds cols.doubles[slot] + miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_double_value(value)) + end + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function handle_value_label(val_labels::Cstring, value::ReadStatValue, label::Cstring, ctx::Any)::Cint + pc = ctx::ParseContext + try + val_labels == C_NULL && return READSTAT_HANDLER_OK + name = Symbol(unsafe_string(val_labels)) + dict = get!(ValueLabelDict, pc.meta.value_labels, name) + key = if readstat_value_is_tagged_missing(value) + readstat_value_tag(value) + else + t = readstat_value_type(value) + if t == READSTAT_TYPE_INT8 + Int32(readstat_int8_value(value)) + elseif t == READSTAT_TYPE_INT16 + Int32(readstat_int16_value(value)) + elseif t == READSTAT_TYPE_INT32 + readstat_int32_value(value) + elseif t == READSTAT_TYPE_FLOAT + Float64(readstat_float_value(value)) + elseif t == READSTAT_TYPE_DOUBLE + readstat_double_value(value) + else + ptr = readstat_string_value(value) + ptr == C_NULL ? "" : unsafe_string(ptr) + end + end + dict[key] = label == C_NULL ? "" : unsafe_string(label) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +# @cfunction pointers are runtime values, so they are created in __init__ and +# cached here rather than serialized into the precompile image. +const CF_METADATA = Ref(C_NULL) +const CF_VARIABLE = Ref(C_NULL) +const CF_VALUE = Ref(C_NULL) +const CF_VALUE_LABEL = Ref(C_NULL) + +function _init_cfunctions() + CF_METADATA[] = @cfunction(handle_metadata, Cint, (MetadataPtr, Any)) + CF_VARIABLE[] = @cfunction(handle_variable, Cint, (Cint, VariablePtr, Cstring, Any)) + CF_VALUE[] = @cfunction(handle_value, Cint, (Cint, VariablePtr, ReadStatValue, Any)) + CF_VALUE_LABEL[] = @cfunction(handle_value_label, Cint, (Cstring, ReadStatValue, Cstring, Any)) + return nothing +end diff --git a/src/reader/metadata.jl b/src/reader/metadata.jl new file mode 100644 index 0000000..1ce600d --- /dev/null +++ b/src/reader/metadata.jl @@ -0,0 +1,61 @@ +# File-level and per-variable metadata captured during a parse. + +# One value-label set: maps a raw value to its display label. Keys are +# normalized to Int32 (all integer types), Float64 (all floats), String, or — +# for labeled tagged missing values like Stata's `.a` — the tag Char. +const ValueLabelDict = Dict{Union{Char,Int32,Float64,String},String} + +""" + ReadStatVarMeta + +Per-variable metadata from a stat-package file: `name`, `label`, `format` +(the producer's display format string), `type`/`type_class` (the raw C +storage type), `vallabel` (name of the value-label set, `Symbol("")` when +none), `storage_width`, `display_width`, `measure`, `alignment`, and +`missing_ranges` (SPSS user-defined missing values as `(lo, hi)` pairs, where +a single missing value has `lo == hi`). +""" +struct ReadStatVarMeta + name::Symbol + label::String + format::String + type::ReadStatType + type_class::ReadStatTypeClass + vallabel::Symbol + storage_width::Int + display_width::Int + measure::ReadStatMeasure + alignment::ReadStatAlignment + missing_ranges::Vector{Tuple{Any,Any}} +end + +""" + ReadStatMeta + +File-level metadata: `row_count` (`-1` when the format does not record it — +XPORT, POR, and some non-conforming SAV files), `var_count`, `creation_time` +and `modified_time`, `file_format_version`, `is_64bit` (SAS), `compression`, +`endianness`, `table_name` (XPORT), `file_label`, `file_encoding`, `notes`, +`fweight` (frequency-weight variable, `Symbol("")` when none), and +`value_labels` (label-set name => [`ValueLabelDict`](@ref)). +""" +mutable struct ReadStatMeta + row_count::Int + var_count::Int + creation_time::DateTime + modified_time::DateTime + file_format_version::Int + is_64bit::Bool + compression::ReadStatCompress + endianness::ReadStatEndian + table_name::String + file_label::String + file_encoding::String + notes::Vector{String} + fweight::Symbol + value_labels::Dict{Symbol,ValueLabelDict} +end + +ReadStatMeta() = ReadStatMeta(-1, 0, Dates.unix2datetime(0), Dates.unix2datetime(0), 0, + false, READSTAT_COMPRESS_NONE, READSTAT_ENDIAN_NONE, "", "", "", String[], + Symbol(""), Dict{Symbol,ValueLabelDict}()) diff --git a/src/reader/read.jl b/src/reader/read.jl new file mode 100644 index 0000000..7765f47 --- /dev/null +++ b/src/reader/read.jl @@ -0,0 +1,92 @@ +# Parse driver and the public read entry points. + +function _parse_format(parser::ParserPtr, path::AbstractString, format::Symbol, ctx) + if format === :dta + CAPI.readstat_parse_dta(parser, path, ctx) + elseif format === :sav + CAPI.readstat_parse_sav(parser, path, ctx) + elseif format === :por + CAPI.readstat_parse_por(parser, path, ctx) + elseif format === :sas7bdat + CAPI.readstat_parse_sas7bdat(parser, path, ctx) + elseif format === :xport + CAPI.readstat_parse_xport(parser, path, ctx) + else + throw(ArgumentError("unknown format $format")) + end +end + +function parse_file!(pc::ParseContext, path::AbstractString, format::Symbol) + isfile(path) || throw(ArgumentError("file not found: $path")) + parser = readstat_parser_init() + local retval + try + readstat_set_metadata_handler(parser, CF_METADATA[]) + readstat_set_variable_handler(parser, CF_VARIABLE[]) + readstat_set_value_handler(parser, CF_VALUE[]) + readstat_set_value_label_handler(parser, CF_VALUE_LABEL[]) + retval = _parse_format(parser, path, format, pc) + finally + readstat_parser_free(parser) + end + if pc.err !== nothing + e, _ = pc.err + throw(e) + end + retval == READSTAT_OK || + error("Error parsing $path: $(readstat_error_message(retval))") + return pc +end + +function build_table(pc::ParseContext) + n = length(pc.names) + columns = Vector{AbstractVector}(undef, n) + for i in 1:n + columns[i] = finalize_column(pc.cols, i) + end + return ReadStatTable(columns, pc.names, pc.meta, pc.varmeta) +end + +function read_data_file(path::AbstractString, format::Symbol) + pc = ParseContext() + parse_file!(pc, path, format) + return build_table(pc) +end + +""" + read_dta(path) -> ReadStatTable + +Read a Stata `.dta` file. See [`ReadStatTable`](@ref) for how to access the +data and metadata. +""" +read_dta(path::AbstractString) = read_data_file(path, :dta) + +""" + read_sav(path) -> ReadStatTable + +Read an SPSS `.sav` (or `.zsav`) file. +""" +read_sav(path::AbstractString) = read_data_file(path, :sav) + +""" + read_por(path) -> ReadStatTable + +Read an SPSS portable `.por` file. The format does not record a row count, +so `filemetadata(tbl).row_count` is `-1`. +""" +read_por(path::AbstractString) = read_data_file(path, :por) + +""" + read_sas7bdat(path) -> ReadStatTable + +Read a SAS `.sas7bdat` data file. +""" +read_sas7bdat(path::AbstractString) = read_data_file(path, :sas7bdat) + +""" + read_xport(path) -> ReadStatTable + +Read a SAS transport (XPORT) `.xpt` file. The format does not record a row +count, so `filemetadata(tbl).row_count` is `-1`. +""" +read_xport(path::AbstractString) = read_data_file(path, :xport) diff --git a/src/reader/table.jl b/src/reader/table.jl new file mode 100644 index 0000000..0563158 --- /dev/null +++ b/src/reader/table.jl @@ -0,0 +1,129 @@ +""" + ReadStatTable + +The result of reading a stat-package file: a set of named `DataValueVector` +columns plus the file's complete metadata. + +Columns are accessed by position (`tbl[1]`) or name (`tbl[:price]`); +`names(tbl)` lists the column names and `size(tbl)` returns +`(rows, columns)`. File-level metadata is available via +[`filemetadata`](@ref), per-variable metadata via [`varmetadata`](@ref), and +value-label sets via [`valuelabels`](@ref). +""" +struct ReadStatTable + cols::Vector{AbstractVector} + names::Vector{Symbol} + lookup::Dict{Symbol,Int} + meta::ReadStatMeta + colmeta::Vector{ReadStatVarMeta} + # Per-column tags of tagged missing values ('a'-'z'; '\0' where untagged), + # allocated lazily only for columns that contain any. + tags::Vector{Union{Nothing,Vector{Char}}} +end + +function ReadStatTable(cols::Vector{AbstractVector}, names::Vector{Symbol}, + meta::ReadStatMeta, colmeta::Vector{ReadStatVarMeta}) + lookup = Dict{Symbol,Int}(name => i for (i, name) in enumerate(names)) + tags = Union{Nothing,Vector{Char}}[nothing for _ in names] + return ReadStatTable(cols, names, lookup, meta, colmeta, tags) +end + +ncols(tbl::ReadStatTable) = length(getfield(tbl, :cols)) +nrows(tbl::ReadStatTable) = ncols(tbl) == 0 ? 0 : length(getfield(tbl, :cols)[1]) + +Base.names(tbl::ReadStatTable) = getfield(tbl, :names) +Base.size(tbl::ReadStatTable) = (nrows(tbl), ncols(tbl)) +function Base.size(tbl::ReadStatTable, dim::Integer) + dim == 1 ? nrows(tbl) : dim == 2 ? ncols(tbl) : + throw(ArgumentError("dimension must be 1 or 2")) +end + +function columnindex(tbl::ReadStatTable, name::Symbol) + i = get(getfield(tbl, :lookup), name, 0) + i == 0 && throw(ArgumentError("no column named $name")) + return i +end +columnindex(tbl::ReadStatTable, i::Integer) = Int(i) + +Base.getindex(tbl::ReadStatTable, i::Integer) = getfield(tbl, :cols)[i] +Base.getindex(tbl::ReadStatTable, name::Symbol) = getfield(tbl, :cols)[columnindex(tbl, name)] + +""" + filemetadata(tbl::ReadStatTable) -> ReadStatMeta + +The file-level metadata of the table: row/variable counts as recorded in the +file, timestamps, format version, compression, endianness, file label, +encoding, notes, and all value-label sets. +""" +filemetadata(tbl::ReadStatTable) = getfield(tbl, :meta) + +""" + varmetadata(tbl::ReadStatTable, col) -> ReadStatVarMeta + +Per-variable metadata for the column given by index or name: variable label, +display format, raw storage type, value-label set name, widths, measure, +alignment, and SPSS missing-value rules. +""" +varmetadata(tbl::ReadStatTable, col::Union{Integer,Symbol}) = + getfield(tbl, :colmeta)[columnindex(tbl, col)] + +""" + valuelabels(tbl::ReadStatTable, col) -> Union{Nothing, ValueLabelDict} + +The value-label dictionary attached to the column given by index or name, or +`nothing` when the column has no value labels. The returned dictionary maps +raw values (and `Char` tags of labeled tagged missing values) to their +labels. +""" +function valuelabels(tbl::ReadStatTable, col::Union{Integer,Symbol}) + vm = varmetadata(tbl, col) + vm.vallabel === Symbol("") && return nothing + return get(filemetadata(tbl).value_labels, vm.vallabel, nothing) +end + +############################################################################## +## +## Display +## +############################################################################## + +function Base.show(io::IO, tbl::ReadStatTable) + r, c = size(tbl) + print(io, "$(r)x$(c) ReadStatTable") +end + +function Base.show(io::IO, ::MIME"text/plain", tbl::ReadStatTable) + r, c = size(tbl) + meta = filemetadata(tbl) + print(io, "$(r)x$(c) ReadStatTable") + isempty(meta.file_label) || print(io, ": ", meta.file_label) + c == 0 && return + + maxrows = min(r, get(io, :limit, true) ? 10 : r) + maxcols = min(c, 20) + cells = Matrix{String}(undef, maxrows + 1, maxcols) + for j in 1:maxcols + cells[1, j] = string(names(tbl)[j]) + col = tbl[j] + for i in 1:maxrows + cells[i + 1, j] = _showcell(col[i]) + end + end + widths = [maximum(textwidth, view(cells, :, j)) for j in 1:maxcols] + for i in 1:(maxrows + 1) + print(io, "\n ") + for j in 1:maxcols + print(io, lpad(cells[i, j], widths[j])) + j < maxcols && print(io, " ") + end + c > maxcols && print(io, " …") + if i == 1 + print(io, "\n ", join([repeat("─", w) for w in widths], " ")) + end + end + maxrows < r && print(io, "\n ⋮ ($(r - maxrows) more rows)") + return +end + +_showcell(v) = sprint(print, v) +_showcell(v::DataValues.DataValue) = DataValues.isna(v) ? "NA" : sprint(print, get(v)) diff --git a/test/runtests.jl b/test/runtests.jl index c9b0edc..0761024 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,7 @@ using TestItemRunner include("test_abi.jl") -include("test_readstat.jl") +include("test_read.jl") +include("test_deprecated.jl") @run_package_tests diff --git a/test/test_deprecated.jl b/test/test_deprecated.jl new file mode 100644 index 0000000..1ecbe50 --- /dev/null +++ b/test/test_deprecated.jl @@ -0,0 +1,32 @@ +@testitem "deprecated 1.x API" begin + using DataValues + using Dates + + df = read_dta(joinpath(@__DIR__, "types.dta")) + @test df isa ReadStatDataFrame + @test ReadStatDataFrame === ReadStatTable + + data = df.data + @test data isa Vector{Any} + @test length(data) == 6 + @test data[1] == DataValueArray{Float32}([3.14, 7.0, NA]) + @test data[5] == DataValueArray{Int8}([2, 7, NA]) + + @test df.headers == [:vfloat, :vdouble, :vlong, :vint, :vbyte, :vstring] + @test df.types == [Float32, Float64, Int32, Int16, Int8, String] + @test df.types_as_int == Int32[4, 5, 3, 2, 1, 0] + @test df.labels == fill("", 6) + @test df.formats isa Vector{String} + @test df.storagewidths isa Vector{Csize_t} + @test df.measures == zeros(Int32, 6) + @test df.alignments == Int32[3, 3, 3, 3, 3, 3] + @test df.val_label_keys == fill("", 6) + @test df.val_label_dict isa Dict{String,Dict{Any,String}} + @test isempty(df.val_label_dict) + @test df.rows == 3 + @test df.columns == 6 + @test df.filelabel == "" + @test df.timestamp isa DateTime + @test df.format isa Clong + @test df.hasmissings == fill(false, 6) +end diff --git a/test/test_read.jl b/test/test_read.jl new file mode 100644 index 0000000..ed4c8b8 --- /dev/null +++ b/test/test_read.jl @@ -0,0 +1,59 @@ +@testitem "read basics" begin + using DataValues + using Dates + using ReadStat: CAPI + + # Expected alignments/measures are the raw enum values as stored in the + # fixtures; every fixture reports measure UNKNOWN (0). + @testset "types.$ext" for (reader, ext, alignments, has_row_count) in + ((read_dta, "dta", [3, 3, 3, 3, 3, 3], true), + (read_sav, "sav", [0, 0, 0, 0, 0, 0], true), + (read_sas7bdat, "sas7bdat", [0, 0, 0, 0, 0, 0], true), + (read_xport, "xpt", [3, 3, 3, 3, 3, 1], false)) + + tbl = reader(joinpath(@__DIR__, "types.$ext")) + + @test size(tbl) == (3, 6) + @test size(tbl, 1) == 3 + @test size(tbl, 2) == 6 + @test names(tbl) == [:vfloat, :vdouble, :vlong, :vint, :vbyte, :vstring] + + @test tbl[:vfloat] == DataValueArray{Float32}([3.14, 7.0, NA]) + @test tbl[:vdouble] == DataValueArray{Float64}([3.14, 7.0, NA]) + @test tbl[:vlong] == DataValueArray{Int32}([2, 7, NA]) + @test tbl[:vint] == DataValueArray{Int16}([2, 7, NA]) + @test tbl[:vbyte] == DataValueArray{Int8}([2, 7, NA]) + # None of these formats has a distinct missing state for strings — + # Stata's missing string literally is the empty string — so the C + # library delivers "" as a regular value and the cell is not NA. + @test tbl[:vstring] == DataValueArray{String}(["2", "7", ""]) + @test tbl[1] === tbl[:vfloat] + @test_throws ArgumentError tbl[:nonexistent] + + meta = filemetadata(tbl) + @test meta.var_count == 6 + @test meta.row_count == (has_row_count ? 3 : -1) + @test meta.modified_time isa DateTime + @test meta.modified_time > DateTime(2000) + @test isempty(meta.notes) + @test isempty(meta.value_labels) + + @test [Int(varmetadata(tbl, i).alignment) for i in 1:6] == alignments + @test all(varmetadata(tbl, i).measure == CAPI.READSTAT_MEASURE_UNKNOWN for i in 1:6) + @test varmetadata(tbl, :vstring).type == CAPI.READSTAT_TYPE_STRING + # Only Stata has a distinct float storage type; SPSS and SAS report + # doubles (the equality checks above compare numerically). + @test varmetadata(tbl, :vfloat).type == + (ext == "dta" ? CAPI.READSTAT_TYPE_FLOAT : CAPI.READSTAT_TYPE_DOUBLE) + @test varmetadata(tbl, :vbyte).name === :vbyte + @test all(isempty(varmetadata(tbl, i).missing_ranges) for i in 1:6) + @test valuelabels(tbl, :vfloat) === nothing + + rendered = sprint(show, MIME"text/plain"(), tbl; context=:limit => true) + @test occursin("3x6 ReadStatTable", rendered) + @test occursin("vstring", rendered) + @test occursin("NA", rendered) + end + + @test_throws ArgumentError read_dta(joinpath(@__DIR__, "no_such_file.dta")) +end diff --git a/test/test_readstat.jl b/test/test_readstat.jl deleted file mode 100644 index d9fcee6..0000000 --- a/test/test_readstat.jl +++ /dev/null @@ -1,35 +0,0 @@ -@testitem "ReadStat" begin - using DataValues - - # Expected alignments are readstat_alignment_t values: - # 0 = UNKNOWN, 1 = LEFT, 2 = CENTER, 3 = RIGHT. - @testset "ReadStat: $ext files" for (reader, ext, alignments) in - ((read_dta, "dta", Int32[3, 3, 3, 3, 3, 3]), - (read_sav, "sav", Int32[0, 0, 0, 0, 0, 0]), - (read_sas7bdat, "sas7bdat", Int32[0, 0, 0, 0, 0, 0]), - (read_xport, "xpt", Int32[3, 3, 3, 3, 3, 1])) - - dtafile = joinpath(@__DIR__, "types.$ext") - rsdf = reader(dtafile) - data = rsdf.data - - @test length(data) == 6 - @test rsdf.headers == [:vfloat, :vdouble, :vlong, :vint, :vbyte, :vstring] - @test data[1] == DataValueArray{Float32}([3.14, 7., NA]) - @test data[2] == DataValueArray{Float64}([3.14, 7., NA]) - @test data[3] == DataValueArray{Int32}([2, 7, NA]) - @test data[4] == DataValueArray{Int16}([2, 7, NA]) - @test data[5] == DataValueArray{Int8}([2, 7., NA]) - @test data[6] == DataValueArray{String}(["2", "7", ""]) - - # Alignments must come from readstat_variable_get_alignment, not from - # the measure accessor sitting next to it in the C API. Every fixture - # reports measure UNKNOWN, so reading the wrong one yields all zeros. - @test rsdf.alignments == alignments - @test rsdf.measures == Int32[0, 0, 0, 0, 0, 0] - - # Every readstat_type_t the readers emit must map to a concrete Julia - # type; a gap in that mapping used to surface as Nothing. - @test all(!=(Nothing), rsdf.types) - end -end From d73712cdf4494847e0aec20c45506ed939dd073b Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:04:34 -0700 Subject: [PATCH 03/14] Add reader configuration: usecols, row selection, encodings, handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every reader (and the new extension-sniffing `readstat` dispatcher) now accepts `usecols` (Symbol/index/vectors/Regex/predicate — skipped columns are skipped inside the C library via READSTAT_HANDLER_SKIP_VARIABLE, and the value path indexes by index_after_skipping), `row_limit`/`row_offset` (wired to the C setters; buffers are preallocated for exactly the selected range, and a zero limit is handled Julia-side because the C library treats 0 as unlimited), `file_encoding`/`handler_encoding` (iconv names), and `progress` (a callback that can stop the parse and keep the rows read so far, trimmed to complete rows). The note, fweight, and error handlers are now always registered: notes and the frequency-weight variable land in ReadStatMeta, and messages the C library reports through its error handler surface as Julia warnings after the parse. `read_meta` reads only metadata: the returned table has zero rows but full file- and variable-level metadata, value labels, and notes, with the true row count (the C library caps the reported count at any row limit in effect, so read_meta sets none). Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 2 +- src/reader/columns.jl | 17 +++++ src/reader/context.jl | 31 +++++++- src/reader/handlers.jl | 68 +++++++++++++++++- src/reader/metadata.jl | 4 +- src/reader/read.jl | 156 +++++++++++++++++++++++++++++++++++++---- test/runtests.jl | 1 + test/test_kwargs.jl | 88 +++++++++++++++++++++++ 8 files changed, 349 insertions(+), 18 deletions(-) create mode 100644 test/test_kwargs.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 8a49990..84c10f0 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -4,7 +4,7 @@ using DataValues: DataValues, DataValueVector using Dates export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, - read_dta, read_sav, read_por, read_sas7bdat, read_xport, + read_dta, read_sav, read_por, read_sas7bdat, read_xport, readstat, read_meta, filemetadata, varmetadata, valuelabels public CAPI diff --git a/src/reader/columns.jl b/src/reader/columns.jl index fd5d7c6..fe8ce0a 100644 --- a/src/reader/columns.jl +++ b/src/reader/columns.jl @@ -102,6 +102,23 @@ end _navalue(::Type{T}) where {T<:Number} = zero(T) _navalue(::Type{String}) = "" +function trim!(buf::ColumnBuf, n::Int) + if length(buf.values) > n + resize!(buf.values, n) + resize!(buf.isna, n) + end + return buf +end + +# Trim every buffer to `n` rows (used when a parse was stopped early and the +# preallocated or partially filled buffers extend past the last complete row). +function trimcolumns!(cols::TypedColumns, n::Int) + for group in (cols.strings, cols.int8s, cols.int16s, cols.int32s, cols.floats, cols.doubles) + foreach(buf -> trim!(buf, n), group) + end + return cols +end + # Wrap a finished buffer without copying. The constructor takes ownership of # both vectors; nothing may touch the ColumnBuf afterwards. finalize_column(buf::ColumnBuf{T}) where {T} = DataValueVector{T}(buf.values, buf.isna) diff --git a/src/reader/context.jl b/src/reader/context.jl index 76a2526..e2a1313 100644 --- a/src/reader/context.jl +++ b/src/reader/context.jl @@ -9,6 +9,35 @@ mutable struct ParseContext # Exception thrown by a handler, captured so it never unwinds through C; # rethrown by the parse driver after readstat_parse returns. err::Union{Nothing,Tuple{Any,Any}} + # Messages the C library reports through its error handler; surfaced as + # Julia warnings after the parse. + warnings::Vector{String} + + # Configuration for this parse: + usecols::Union{Nothing,Function} # (name::Symbol, index::Int) -> Bool + progress::Union{Nothing,Function} # (fraction::Float64) -> Bool (false aborts) + row_offset::Int # 0 = from the start + row_limit::Int # -1 = no limit + file_encoding::Union{Nothing,String} + handler_encoding::Union{Nothing,String} + collect_values::Bool # false for metadata-only parses + + # Progress-abort bookkeeping: the last fully delivered row, so a + # partially parsed table can be trimmed to complete rows. + aborted::Bool + rows_complete::Int end -ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), nothing) +ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), + nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, 0) + +# Rows to preallocate per column: what the file reports, minus the offset, +# capped by the limit; 0 when unknown (buffers then grow row by row). +function alloc_rows(pc::ParseContext) + pc.collect_values || return 0 + rc = pc.meta.row_count + rc < 0 && return 0 + n = max(rc - pc.row_offset, 0) + pc.row_limit >= 0 && (n = min(n, pc.row_limit)) + return n +end diff --git a/src/reader/handlers.jl b/src/reader/handlers.jl index 0fae0e5..17b3a7e 100644 --- a/src/reader/handlers.jl +++ b/src/reader/handlers.jl @@ -61,9 +61,14 @@ end function handle_variable(index::Cint, variable::VariablePtr, val_labels::Cstring, ctx::Any)::Cint pc = ctx::ParseContext try + name = Symbol(readstat_variable_get_name(variable)) + sel = pc.usecols + if sel !== nothing && !(sel(name, Int(index) + 1)::Bool) + return READSTAT_HANDLER_SKIP_VARIABLE + end t = readstat_variable_get_type(variable) vm = ReadStatVarMeta( - Symbol(readstat_variable_get_name(variable)), + name, readstat_variable_get_label(variable), readstat_variable_get_format(variable), t, @@ -76,7 +81,7 @@ function handle_variable(index::Cint, variable::VariablePtr, val_labels::Cstring _missing_ranges(variable)) push!(pc.varmeta, vm) push!(pc.names, vm.name) - addcolumn!(pc.cols, jltype(t), max(pc.meta.row_count, 0)) + addcolumn!(pc.cols, jltype(t), alloc_rows(pc)) return READSTAT_HANDLER_OK catch e pc.err = (e, catch_backtrace()) @@ -117,6 +122,9 @@ function handle_value(obs_index::Cint, variable::VariablePtr, value::ReadStatVal buf = @inbounds cols.doubles[slot] miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_double_value(value)) end + # Track the last fully delivered row so a parse stopped early (by the + # progress callback) can be trimmed to complete rows. + idx == length(cols.slots) && (pc.rows_complete = row) return READSTAT_HANDLER_OK catch e pc.err = (e, catch_backtrace()) @@ -157,17 +165,73 @@ function handle_value_label(val_labels::Cstring, value::ReadStatValue, label::Cs end end +function handle_note(note_index::Cint, note::Cstring, ctx::Any)::Cint + pc = ctx::ParseContext + try + note == C_NULL || push!(pc.meta.notes, unsafe_string(note)) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function handle_fweight(variable::VariablePtr, ctx::Any)::Cint + pc = ctx::ParseContext + try + pc.meta.fweight = Symbol(readstat_variable_get_name(variable)) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +# The C error handler reports warnings the parse survives; it returns void, +# so exceptions can neither abort nor propagate — swallow them. +function handle_error(message::Cstring, ctx::Any)::Cvoid + pc = ctx::ParseContext + try + message == C_NULL || push!(pc.warnings, unsafe_string(message)) + catch + end + return nothing +end + +function handle_progress(progress::Cdouble, ctx::Any)::Cint + pc = ctx::ParseContext + try + f = pc.progress + if f !== nothing && f(Float64(progress)) === false + pc.aborted = true + return READSTAT_HANDLER_ABORT + end + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + # @cfunction pointers are runtime values, so they are created in __init__ and # cached here rather than serialized into the precompile image. const CF_METADATA = Ref(C_NULL) const CF_VARIABLE = Ref(C_NULL) const CF_VALUE = Ref(C_NULL) const CF_VALUE_LABEL = Ref(C_NULL) +const CF_NOTE = Ref(C_NULL) +const CF_FWEIGHT = Ref(C_NULL) +const CF_ERROR = Ref(C_NULL) +const CF_PROGRESS = Ref(C_NULL) function _init_cfunctions() CF_METADATA[] = @cfunction(handle_metadata, Cint, (MetadataPtr, Any)) CF_VARIABLE[] = @cfunction(handle_variable, Cint, (Cint, VariablePtr, Cstring, Any)) CF_VALUE[] = @cfunction(handle_value, Cint, (Cint, VariablePtr, ReadStatValue, Any)) CF_VALUE_LABEL[] = @cfunction(handle_value_label, Cint, (Cstring, ReadStatValue, Cstring, Any)) + CF_NOTE[] = @cfunction(handle_note, Cint, (Cint, Cstring, Any)) + CF_FWEIGHT[] = @cfunction(handle_fweight, Cint, (VariablePtr, Any)) + CF_ERROR[] = @cfunction(handle_error, Cvoid, (Cstring, Any)) + CF_PROGRESS[] = @cfunction(handle_progress, Cint, (Cdouble, Any)) return nothing end diff --git a/src/reader/metadata.jl b/src/reader/metadata.jl index 1ce600d..6ef5fe3 100644 --- a/src/reader/metadata.jl +++ b/src/reader/metadata.jl @@ -33,7 +33,9 @@ end ReadStatMeta File-level metadata: `row_count` (`-1` when the format does not record it — -XPORT, POR, and some non-conforming SAV files), `var_count`, `creation_time` +XPORT, POR, and some non-conforming SAV files; when a read used a +`row_limit`, the C library caps the reported count at that limit — use +[`read_meta`](@ref) for the true count), `var_count`, `creation_time` and `modified_time`, `file_format_version`, `is_64bit` (SAS), `compression`, `endianness`, `table_name` (XPORT), `file_label`, `file_encoding`, `notes`, `fweight` (frequency-weight variable, `Symbol("")` when none), and diff --git a/src/reader/read.jl b/src/reader/read.jl index 7765f47..7a43a0e 100644 --- a/src/reader/read.jl +++ b/src/reader/read.jl @@ -1,5 +1,24 @@ # Parse driver and the public read entry points. +const FORMAT_BY_EXT = Dict( + ".dta" => :dta, + ".sav" => :sav, + ".zsav" => :sav, + ".por" => :por, + ".sas7bdat" => :sas7bdat, + ".xpt" => :xport, + ".xport" => :xport, +) + +function _sniff_format(path::AbstractString, format::Symbol) + format === :auto || return format + ext = lowercase(splitext(path)[2]) + fmt = get(FORMAT_BY_EXT, ext, nothing) + fmt === nothing && + throw(ArgumentError("cannot infer the file format from extension \"$ext\"; pass `format=...`")) + return fmt +end + function _parse_format(parser::ParserPtr, path::AbstractString, format::Symbol, ctx) if format === :dta CAPI.readstat_parse_dta(parser, path, ctx) @@ -16,6 +35,15 @@ function _parse_format(parser::ParserPtr, path::AbstractString, format::Symbol, end end +# Normalize the `usecols` kwarg into a `(name, index) -> Bool` predicate. +_colselector(::Nothing) = nothing +_colselector(s::Symbol) = (name, i) -> name === s +_colselector(idx::Integer) = let idx = Int(idx); (name, i) -> i == idx; end +_colselector(v::AbstractVector{Symbol}) = let s = Set(v); (name, i) -> name in s; end +_colselector(v::AbstractVector{<:Integer}) = let s = Set{Int}(v); (name, i) -> i in s; end +_colselector(r::Regex) = (name, i) -> occursin(r, String(name)) +_colselector(f::Function) = (name, i) -> f(name)::Bool + function parse_file!(pc::ParseContext, path::AbstractString, format::Symbol) isfile(path) || throw(ArgumentError("file not found: $path")) parser = readstat_parser_init() @@ -23,8 +51,18 @@ function parse_file!(pc::ParseContext, path::AbstractString, format::Symbol) try readstat_set_metadata_handler(parser, CF_METADATA[]) readstat_set_variable_handler(parser, CF_VARIABLE[]) - readstat_set_value_handler(parser, CF_VALUE[]) readstat_set_value_label_handler(parser, CF_VALUE_LABEL[]) + readstat_set_note_handler(parser, CF_NOTE[]) + readstat_set_fweight_handler(parser, CF_FWEIGHT[]) + readstat_set_error_handler(parser, CF_ERROR[]) + pc.collect_values && readstat_set_value_handler(parser, CF_VALUE[]) + pc.progress === nothing || readstat_set_progress_handler(parser, CF_PROGRESS[]) + pc.row_offset > 0 && readstat_set_row_offset(parser, pc.row_offset) + pc.row_limit >= 0 && readstat_set_row_limit(parser, pc.row_limit) + pc.file_encoding === nothing || + readstat_set_file_character_encoding(parser, pc.file_encoding) + pc.handler_encoding === nothing || + readstat_set_handler_character_encoding(parser, pc.handler_encoding) retval = _parse_format(parser, path, format, pc) finally readstat_parser_free(parser) @@ -33,12 +71,20 @@ function parse_file!(pc::ParseContext, path::AbstractString, format::Symbol) e, _ = pc.err throw(e) end - retval == READSTAT_OK || + for w in pc.warnings + @warn "readstat: $w" _module = ReadStat _file = String(path) + end + if retval == READSTAT_ERROR_USER_ABORT && pc.aborted + # The progress callback stopped the parse; the caller gets the rows + # delivered so far. + elseif retval != READSTAT_OK error("Error parsing $path: $(readstat_error_message(retval))") + end return pc end function build_table(pc::ParseContext) + pc.aborted && trimcolumns!(pc.cols, pc.rows_complete) n = length(pc.names) columns = Vector{AbstractVector}(undef, n) for i in 1:n @@ -47,46 +93,130 @@ function build_table(pc::ParseContext) return ReadStatTable(columns, pc.names, pc.meta, pc.varmeta) end -function read_data_file(path::AbstractString, format::Symbol) +function read_data_file(path::AbstractString, format::Symbol; + usecols=nothing, + row_limit::Union{Nothing,Integer}=nothing, + row_offset::Integer=0, + file_encoding::Union{Nothing,AbstractString}=nothing, + handler_encoding::Union{Nothing,AbstractString}=nothing, + progress=nothing) + row_offset >= 0 || throw(ArgumentError("row_offset must be non-negative")) + row_limit === nothing || row_limit >= 0 || + throw(ArgumentError("row_limit must be non-negative")) pc = ParseContext() + pc.usecols = _colselector(usecols) + pc.progress = progress + pc.row_offset = Int(row_offset) + if row_limit == 0 + # The C library treats a row limit of 0 as "no limit", so a zero-row + # read is done by not collecting values at all (with a 1-row limit as + # a hint to stop decoding the data section early). + pc.collect_values = false + pc.row_limit = 1 + else + pc.row_limit = row_limit === nothing ? -1 : Int(row_limit) + end + pc.file_encoding = file_encoding === nothing ? nothing : String(file_encoding) + pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) parse_file!(pc, path, format) return build_table(pc) end +const _READ_KWARGS_DOC = """ +All readers accept the same keyword arguments: + +- `usecols`: read only these columns — a `Symbol`, column index, vector of + either, `Regex`, or a predicate called with each column name. Skipped + columns are never parsed (the C library skips them). +- `row_limit`: read at most this many rows. +- `row_offset`: skip this many rows from the start. +- `file_encoding`: override the character encoding declared in the file + (an iconv-compatible name such as `"WINDOWS-1252"`). +- `handler_encoding`: the encoding delivered to Julia; defaults to UTF-8. +- `progress`: a function called with the parse fraction (0.0-1.0); return + `false` to stop the parse and get the rows read so far. +""" + """ - read_dta(path) -> ReadStatTable + read_dta(path; kwargs...) -> ReadStatTable Read a Stata `.dta` file. See [`ReadStatTable`](@ref) for how to access the data and metadata. + +$_READ_KWARGS_DOC """ -read_dta(path::AbstractString) = read_data_file(path, :dta) +read_dta(path::AbstractString; kwargs...) = read_data_file(path, :dta; kwargs...) """ - read_sav(path) -> ReadStatTable + read_sav(path; kwargs...) -> ReadStatTable Read an SPSS `.sav` (or `.zsav`) file. + +$_READ_KWARGS_DOC """ -read_sav(path::AbstractString) = read_data_file(path, :sav) +read_sav(path::AbstractString; kwargs...) = read_data_file(path, :sav; kwargs...) """ - read_por(path) -> ReadStatTable + read_por(path; kwargs...) -> ReadStatTable Read an SPSS portable `.por` file. The format does not record a row count, so `filemetadata(tbl).row_count` is `-1`. + +$_READ_KWARGS_DOC """ -read_por(path::AbstractString) = read_data_file(path, :por) +read_por(path::AbstractString; kwargs...) = read_data_file(path, :por; kwargs...) """ - read_sas7bdat(path) -> ReadStatTable + read_sas7bdat(path; kwargs...) -> ReadStatTable Read a SAS `.sas7bdat` data file. + +$_READ_KWARGS_DOC """ -read_sas7bdat(path::AbstractString) = read_data_file(path, :sas7bdat) +read_sas7bdat(path::AbstractString; kwargs...) = read_data_file(path, :sas7bdat; kwargs...) """ - read_xport(path) -> ReadStatTable + read_xport(path; kwargs...) -> ReadStatTable Read a SAS transport (XPORT) `.xpt` file. The format does not record a row count, so `filemetadata(tbl).row_count` is `-1`. + +$_READ_KWARGS_DOC +""" +read_xport(path::AbstractString; kwargs...) = read_data_file(path, :xport; kwargs...) + +""" + readstat(path; format=:auto, kwargs...) -> ReadStatTable + +Read a stat-package data file, inferring the format from the file extension +(`.dta`, `.sav`/`.zsav`, `.por`, `.sas7bdat`, `.xpt`/`.xport`) unless +`format` is given explicitly (`:dta`, `:sav`, `:por`, `:sas7bdat`, +`:xport`). + +$_READ_KWARGS_DOC """ -read_xport(path::AbstractString) = read_data_file(path, :xport) +readstat(path::AbstractString; format::Symbol=:auto, kwargs...) = + read_data_file(path, _sniff_format(path, format); kwargs...) + +""" + read_meta(path; format=:auto, file_encoding=nothing, handler_encoding=nothing) + -> ReadStatTable + +Read only the metadata of a stat-package data file: the returned table has +zero rows, but its [`filemetadata`](@ref), [`varmetadata`](@ref), and +[`valuelabels`](@ref) are fully populated, including the row count recorded +in the file (`filemetadata(tbl).row_count`). This is much cheaper than +reading the data. +""" +function read_meta(path::AbstractString; format::Symbol=:auto, + file_encoding::Union{Nothing,AbstractString}=nothing, + handler_encoding::Union{Nothing,AbstractString}=nothing) + pc = ParseContext() + pc.collect_values = false + # No row limit here: the C library caps the row count it reports at any + # row limit in effect, and read_meta must report the true count. + pc.file_encoding = file_encoding === nothing ? nothing : String(file_encoding) + pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) + parse_file!(pc, path, _sniff_format(path, format)) + return build_table(pc) +end diff --git a/test/runtests.jl b/test/runtests.jl index 0761024..ae4b1a5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,7 @@ using TestItemRunner include("test_abi.jl") include("test_read.jl") +include("test_kwargs.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_kwargs.jl b/test/test_kwargs.jl new file mode 100644 index 0000000..a0cdeb0 --- /dev/null +++ b/test/test_kwargs.jl @@ -0,0 +1,88 @@ +@testitem "reader kwargs" begin + using DataValues + + dta = joinpath(@__DIR__, "types.dta") + xpt = joinpath(@__DIR__, "types.xpt") + + @testset "usecols" begin + tbl = read_dta(dta; usecols=[:vint, :vstring]) + @test names(tbl) == [:vint, :vstring] + @test size(tbl) == (3, 2) + @test tbl[:vint] == DataValueArray{Int16}([2, 7, NA]) + @test varmetadata(tbl, :vstring).name === :vstring + + @test names(read_dta(dta; usecols=1)) == [:vfloat] + @test names(read_dta(dta; usecols=:vbyte)) == [:vbyte] + @test names(read_dta(dta; usecols=[2, 4])) == [:vdouble, :vint] + @test names(read_dta(dta; usecols=r"^vd")) == [:vdouble] + @test names(read_dta(dta; usecols=n -> startswith(string(n), "vs"))) == [:vstring] + @test size(read_dta(dta; usecols=Symbol[])) == (0, 0) + + # Column selection composes with row selection on the C side. + tbl = read_dta(dta; usecols=[:vlong], row_limit=2) + @test size(tbl) == (2, 1) + @test tbl[:vlong] == DataValueArray{Int32}([2, 7]) + end + + @testset "row_limit / row_offset" begin + tbl = read_dta(dta; row_limit=2) + @test size(tbl) == (2, 6) + @test tbl[:vlong] == DataValueArray{Int32}([2, 7]) + + tbl = read_dta(dta; row_offset=1) + @test size(tbl) == (2, 6) + @test tbl[:vlong] == DataValueArray{Int32}([7, NA]) + + tbl = read_dta(dta; row_offset=1, row_limit=1) + @test size(tbl) == (1, 6) + @test tbl[:vlong] == DataValueArray{Int32}([7]) + + @test size(read_dta(dta; row_limit=0)) == (0, 6) + + # Formats with unknown row counts grow their buffers row by row. + tbl = read_xport(xpt; row_limit=2) + @test size(tbl) == (2, 6) + tbl = read_xport(xpt; row_offset=1) + @test size(tbl) == (2, 6) + @test tbl[:vlong] == DataValueArray{Int32}([7, NA]) + + @test_throws ArgumentError read_dta(dta; row_offset=-1) + @test_throws ArgumentError read_dta(dta; row_limit=-1) + end + + @testset "read_meta" begin + tbl = read_meta(dta) + @test size(tbl) == (0, 6) + @test names(tbl) == [:vfloat, :vdouble, :vlong, :vint, :vbyte, :vstring] + @test filemetadata(tbl).row_count == 3 + @test varmetadata(tbl, :vint).name === :vint + + @test filemetadata(read_meta(joinpath(@__DIR__, "types.sav"))).row_count == 3 + @test filemetadata(read_meta(xpt)).row_count == -1 + end + + @testset "readstat dispatcher" begin + for file in ("types.dta", "types.sav", "types.sas7bdat", "types.xpt") + @test size(readstat(joinpath(@__DIR__, file)), 2) == 6 + end + @test size(readstat(dta; format=:dta), 1) == 3 + @test_throws ArgumentError readstat("data.unknown") + end + + @testset "progress" begin + fractions = Float64[] + tbl = read_dta(dta; progress=p -> (push!(fractions, p); true)) + @test size(tbl) == (3, 6) + @test !isempty(fractions) + @test all(0.0 .<= fractions .<= 1.0) + + # Returning false stops the parse; the rows read so far come back. + tbl = read_dta(dta; progress=p -> false) + @test size(tbl, 1) <= 3 + end + + @testset "encodings" begin + tbl = read_dta(dta; handler_encoding="UTF-8") + @test tbl[:vstring] == DataValueArray{String}(["2", "7", ""]) + end +end From b995e3bc8b1f5e2e522cd8bbee8edc838777be7c Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:10:42 -0700 Subject: [PATCH 04/14] Surface all three kinds of missing values, with fixtures Tagged missing values (Stata/SAS .a-.z) still read as NA, but their tags are now recorded per column and exposed through the new missingtags accessor; label sets that label a tagged missing value keep the tag as a Char key. SPSS user-defined missing values collapse to NA by default as before, and the new user_missing=:keep keyword keeps the codes as data instead, with the declared rules always available in varmetadata(tbl, col).missing_ranges. Test fixtures are sourced from ReadStatTables.jl (MIT) and pyreadstat (Apache-2.0), verified against the upstream blob SHAs and attributed in test/data/README.md; pyreadstat ships the expected values for both user-missing modes as CSVs, which the new tests assert against. read_por is now tested for the first time (queryverse/ReadStat.jl#96) using the known-good sample.por; StatFiles.jl's types.por fixture turns out to carry a timestamp readstat 1.1.9 rejects, which the tests pin down as a clean error (context for queryverse/StatFiles.jl#32). Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 2 +- src/reader/columns.jl | 40 +++++++++++---- src/reader/context.jl | 7 ++- src/reader/handlers.jl | 27 +++++++--- src/reader/read.jl | 12 ++++- src/reader/table.jl | 16 +++++- test/data/README.md | 22 ++++++++ test/data/alltypes.dta | Bin 0 -> 6755 bytes test/data/datetime13.xpt | Bin 0 -> 960 bytes test/data/sample.dta | Bin 0 -> 13313 bytes test/data/sample.por | 14 ++++++ test/data/sample.sas7bdat | Bin 0 -> 131072 bytes test/data/sample.sav | Bin 0 -> 1651 bytes test/data/sample.xpt | Bin 0 -> 2080 bytes test/data/sample.zsav | Bin 0 -> 1656 bytes test/data/sample_missing.csv | 8 +++ test/data/sample_missing.sav | Bin 0 -> 1779 bytes test/data/sample_missing_user.csv | 8 +++ test/data/string_labeled_value.sav | Bin 0 -> 7921 bytes test/data/stringtypes.dta | Bin 0 -> 13185 bytes test/runtests.jl | 1 + test/test_missing.jl | 78 +++++++++++++++++++++++++++++ test/types.por | 9 ++++ 23 files changed, 220 insertions(+), 24 deletions(-) create mode 100644 test/data/README.md create mode 100644 test/data/alltypes.dta create mode 100644 test/data/datetime13.xpt create mode 100644 test/data/sample.dta create mode 100644 test/data/sample.por create mode 100644 test/data/sample.sas7bdat create mode 100644 test/data/sample.sav create mode 100644 test/data/sample.xpt create mode 100644 test/data/sample.zsav create mode 100644 test/data/sample_missing.csv create mode 100644 test/data/sample_missing.sav create mode 100644 test/data/sample_missing_user.csv create mode 100644 test/data/string_labeled_value.sav create mode 100644 test/data/stringtypes.dta create mode 100644 test/test_missing.jl create mode 100644 test/types.por diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 84c10f0..987123d 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -5,7 +5,7 @@ using Dates export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, read_dta, read_sav, read_por, read_sas7bdat, read_xport, readstat, read_meta, - filemetadata, varmetadata, valuelabels + filemetadata, varmetadata, valuelabels, missingtags public CAPI diff --git a/src/reader/columns.jl b/src/reader/columns.jl index fe8ce0a..685228d 100644 --- a/src/reader/columns.jl +++ b/src/reader/columns.jl @@ -12,13 +12,16 @@ # memory. String buffers are backed by `""` so every slot is always a defined # object. -struct ColumnBuf{T} +mutable struct ColumnBuf{T} values::Vector{T} isna::Vector{Bool} + # Tags of tagged missing values ('a'-'z', '\0' where untagged), allocated + # lazily the first time a tagged missing arrives in this column. + tags::Union{Nothing,Vector{Char}} end -newbuf(::Type{T}, n::Int) where {T} = ColumnBuf{T}(Vector{T}(undef, n), fill(true, n)) -newbuf(::Type{String}, n::Int) = ColumnBuf{String}(fill("", n), fill(true, n)) +newbuf(::Type{T}, n::Int) where {T} = ColumnBuf{T}(Vector{T}(undef, n), fill(true, n), nothing) +newbuf(::Type{String}, n::Int) = ColumnBuf{String}(fill("", n), fill(true, n), nothing) # Column storage grouped by element type, so every access from the value # handler goes through a small typecode branch into fully type-stable code @@ -80,22 +83,32 @@ end elseif row == n + 1 push!(buf.values, v) push!(buf.isna, false) + buf.tags === nothing || push!(buf.tags, '\0') else throw(ArgumentError("out-of-order row index $row for column of length $n")) end return buf end -@inline function setmissing!(buf::ColumnBuf{T}, row::Int) where {T} +@inline function setmissing!(buf::ColumnBuf{T}, row::Int, tag::Char='\0') where {T} n = length(buf.values) if row <= n @inbounds buf.isna[row] = true elseif row == n + 1 push!(buf.values, _navalue(T)) push!(buf.isna, true) + buf.tags === nothing || push!(buf.tags, '\0') else throw(ArgumentError("out-of-order row index $row for column of length $n")) end + if tag != '\0' + t = buf.tags + if t === nothing + t = fill('\0', length(buf.values)) + buf.tags = t + end + @inbounds t[row] = tag + end return buf end @@ -106,6 +119,7 @@ function trim!(buf::ColumnBuf, n::Int) if length(buf.values) > n resize!(buf.values, n) resize!(buf.isna, n) + buf.tags === nothing || resize!(buf.tags, n) end return buf end @@ -123,19 +137,23 @@ end # both vectors; nothing may touch the ColumnBuf afterwards. finalize_column(buf::ColumnBuf{T}) where {T} = DataValueVector{T}(buf.values, buf.isna) -function finalize_column(cols::TypedColumns, i::Int) +function getbuf(cols::TypedColumns, i::Int) code, slot = cols.slots[i] if code == CODE_STRING - finalize_column(cols.strings[slot]) + cols.strings[slot] elseif code == CODE_INT8 - finalize_column(cols.int8s[slot]) + cols.int8s[slot] elseif code == CODE_INT16 - finalize_column(cols.int16s[slot]) + cols.int16s[slot] elseif code == CODE_INT32 - finalize_column(cols.int32s[slot]) + cols.int32s[slot] elseif code == CODE_FLOAT - finalize_column(cols.floats[slot]) + cols.floats[slot] else - finalize_column(cols.doubles[slot]) + cols.doubles[slot] end end + +finalize_column(cols::TypedColumns, i::Int) = finalize_column(getbuf(cols, i)) + +column_tags(cols::TypedColumns, i::Int) = getbuf(cols, i).tags diff --git a/src/reader/context.jl b/src/reader/context.jl index e2a1313..302ccd5 100644 --- a/src/reader/context.jl +++ b/src/reader/context.jl @@ -20,7 +20,10 @@ mutable struct ParseContext row_limit::Int # -1 = no limit file_encoding::Union{Nothing,String} handler_encoding::Union{Nothing,String} - collect_values::Bool # false for metadata-only parses + collect_values::Bool + # true: SPSS user-defined missing values are kept as data (only system + # and tagged missings become NA); false: they collapse to NA. + keep_user_missing::Bool # false for metadata-only parses # Progress-abort bookkeeping: the last fully delivered row, so a # partially parsed table can be trimmed to complete rows. @@ -29,7 +32,7 @@ mutable struct ParseContext end ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), - nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, 0) + nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, false, 0) # Rows to preallocate per column: what the file reports, minus the offset, # capped by the limit; 0 when unknown (buffers then grow row by row). diff --git a/src/reader/handlers.jl b/src/reader/handlers.jl index 17b3a7e..a1284bb 100644 --- a/src/reader/handlers.jl +++ b/src/reader/handlers.jl @@ -96,31 +96,44 @@ function handle_value(obs_index::Cint, variable::VariablePtr, value::ReadStatVal row = Int(obs_index) + 1 cols = pc.cols code, slot = @inbounds cols.slots[idx] - miss = readstat_value_is_missing(value, variable) + + # Three kinds of missing values (readstat.h): system missing and + # tagged missing (Stata/SAS .a-.z) always become NA, the latter with + # its tag recorded; SPSS user-defined missing values become NA by + # default but stay data under `user_missing=:keep`. + local miss::Bool, tag::Char + if readstat_value_is_tagged_missing(value) + miss = true + tag = readstat_value_tag(value) + else + tag = '\0' + miss = pc.keep_user_missing ? readstat_value_is_system_missing(value) : + readstat_value_is_missing(value, variable) + end if code == CODE_STRING buf = @inbounds cols.strings[slot] if miss - setmissing!(buf, row) + setmissing!(buf, row, tag) else ptr = readstat_string_value(value) setvalue!(buf, row, ptr == C_NULL ? "" : unsafe_string(ptr)) end elseif code == CODE_INT8 buf = @inbounds cols.int8s[slot] - miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_int8_value(value)) + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_int8_value(value)) elseif code == CODE_INT16 buf = @inbounds cols.int16s[slot] - miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_int16_value(value)) + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_int16_value(value)) elseif code == CODE_INT32 buf = @inbounds cols.int32s[slot] - miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_int32_value(value)) + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_int32_value(value)) elseif code == CODE_FLOAT buf = @inbounds cols.floats[slot] - miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_float_value(value)) + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_float_value(value)) else buf = @inbounds cols.doubles[slot] - miss ? setmissing!(buf, row) : setvalue!(buf, row, readstat_double_value(value)) + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_double_value(value)) end # Track the last fully delivered row so a parse stopped early (by the # progress callback) can be trimmed to complete rows. diff --git a/src/reader/read.jl b/src/reader/read.jl index 7a43a0e..08e7116 100644 --- a/src/reader/read.jl +++ b/src/reader/read.jl @@ -87,10 +87,12 @@ function build_table(pc::ParseContext) pc.aborted && trimcolumns!(pc.cols, pc.rows_complete) n = length(pc.names) columns = Vector{AbstractVector}(undef, n) + tags = Vector{Union{Nothing,Vector{Char}}}(undef, n) for i in 1:n columns[i] = finalize_column(pc.cols, i) + tags[i] = column_tags(pc.cols, i) end - return ReadStatTable(columns, pc.names, pc.meta, pc.varmeta) + return ReadStatTable(columns, pc.names, pc.meta, pc.varmeta, tags) end function read_data_file(path::AbstractString, format::Symbol; @@ -99,7 +101,10 @@ function read_data_file(path::AbstractString, format::Symbol; row_offset::Integer=0, file_encoding::Union{Nothing,AbstractString}=nothing, handler_encoding::Union{Nothing,AbstractString}=nothing, + user_missing::Symbol=:na, progress=nothing) + user_missing in (:na, :keep) || + throw(ArgumentError("user_missing must be :na or :keep")) row_offset >= 0 || throw(ArgumentError("row_offset must be non-negative")) row_limit === nothing || row_limit >= 0 || throw(ArgumentError("row_limit must be non-negative")) @@ -118,6 +123,7 @@ function read_data_file(path::AbstractString, format::Symbol; end pc.file_encoding = file_encoding === nothing ? nothing : String(file_encoding) pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) + pc.keep_user_missing = user_missing === :keep parse_file!(pc, path, format) return build_table(pc) end @@ -133,6 +139,10 @@ All readers accept the same keyword arguments: - `file_encoding`: override the character encoding declared in the file (an iconv-compatible name such as `"WINDOWS-1252"`). - `handler_encoding`: the encoding delivered to Julia; defaults to UTF-8. +- `user_missing`: `:na` (default) collapses SPSS user-defined missing values + to NA; `:keep` keeps them as data (the rules stay available in + `varmetadata(tbl, col).missing_ranges`). System missing and tagged missing + values are always NA; see [`missingtags`](@ref) for the tags. - `progress`: a function called with the parse fraction (0.0-1.0); return `false` to stop the parse and get the rows read so far. """ diff --git a/src/reader/table.jl b/src/reader/table.jl index 0563158..a3617b5 100644 --- a/src/reader/table.jl +++ b/src/reader/table.jl @@ -22,9 +22,10 @@ struct ReadStatTable end function ReadStatTable(cols::Vector{AbstractVector}, names::Vector{Symbol}, - meta::ReadStatMeta, colmeta::Vector{ReadStatVarMeta}) + meta::ReadStatMeta, colmeta::Vector{ReadStatVarMeta}, + tags::Vector{Union{Nothing,Vector{Char}}}= + Union{Nothing,Vector{Char}}[nothing for _ in names]) lookup = Dict{Symbol,Int}(name => i for (i, name) in enumerate(names)) - tags = Union{Nothing,Vector{Char}}[nothing for _ in names] return ReadStatTable(cols, names, lookup, meta, colmeta, tags) end @@ -81,6 +82,17 @@ function valuelabels(tbl::ReadStatTable, col::Union{Integer,Symbol}) return get(filemetadata(tbl).value_labels, vm.vallabel, nothing) end +""" + missingtags(tbl::ReadStatTable, col) -> Union{Nothing, Vector{Char}} + +Tags of the tagged missing values (Stata/SAS `.a`-`.z`) in the column given +by index or name: a vector with one `Char` per row, `'a'`-`'z'` where the +cell is a tagged missing value and `'\\0'` everywhere else. Returns `nothing` +when the column contains no tagged missing values. +""" +missingtags(tbl::ReadStatTable, col::Union{Integer,Symbol}) = + getfield(tbl, :tags)[columnindex(tbl, col)] + ############################################################################## ## ## Display diff --git a/test/data/README.md b/test/data/README.md new file mode 100644 index 0000000..624cb60 --- /dev/null +++ b/test/data/README.md @@ -0,0 +1,22 @@ +# Test fixtures + +Sourced fixtures (verified against the upstream git blob SHAs when copied): + +- `alltypes.dta`, `stringtypes.dta`, `sample.dta`, `sample.sav`, `sample.por`, + `sample.sas7bdat`, `sample.xpt`, `string_labeled_value.sav`, + `datetime13.xpt` are copied from + [ReadStatTables.jl](https://github.com/junyuan-chen/ReadStatTables.jl) + (`data/`, MIT license). `alltypes.dta` and `stringtypes.dta` are generated + by the `alltypes.do`/`stringtypes.do` scripts kept in that repository; + the `sample.*` files there originally come from pyreadstat (see below), + with `sample.dta` slightly modified for wider test coverage. +- `sample_missing.sav`, `sample_missing.csv`, `sample_missing_user.csv`, and + `sample.zsav` are copied from + [pyreadstat](https://github.com/Roche/pyreadstat) (`test_data/basic/`, + Apache-2.0 license). The two CSV files hold the expected values of + `sample_missing.sav` with user-defined missing values collapsed to missing + and kept as data, respectively. + +The `types.{dta,sav,sas7bdat,xpt,por}` fixtures in the parent directory are +the original ReadStat.jl fixtures (`types.por` was copied from +StatFiles.jl, which ships the same table in SPSS portable form). diff --git a/test/data/alltypes.dta b/test/data/alltypes.dta new file mode 100644 index 0000000000000000000000000000000000000000..fdad025320951813a698e798b78152e8c57ce1a9 GIT binary patch literal 6755 zcmeGhU1$_XcrGC|ij+!3qENFIQxT<`O9Rz*oHS@-F;Nd9^+ieM?lzY#d)e#mxNR)- zPWzx@{~i>dl$59sLZJ^q`qp>*5;0O=#1l+f`Xr?^rVY7%^VfLWJK`17d*}`!-+c4^ z&fGWOH#3WUMj0JUGfMD?Mbnl?uxB|I^(_*Q@4}`=aB_@UuE!br2YRu&gyKH38ey}K z;6tPl$%f5C1UocoIRpVs_As-v7TC+?$hz0##zE>CogJNBMm+vz*BjVeqGD6KlBGFN zAlnrMH^yYAU%_9RB;N9Zgxi`W{Bf;>ds`(OR`6#9KiMe9k8YAMr64xBRUA;yrb)*d zWMetYCx4XYOHWF3rQb^p2o>f5Vl%|TMMf!1kpz5oq}KZU5A0EfoI1J?IPPc$b;Rp4 zeXtq~J1%9Yjt->V6A(UiTL2XJ%tI)HCaSfSc8xpG&tu>tw$i35>krZtw&CZ^lZ89k zMl^)bS6yE}X_^1`3Q<#FL(;{D3{%9K9~B65_>N!WKi=GZ=ign9;YSb4+d-()=O zdRfYR(jM=6ts_&;_e|LTLU6Yad#h5r@I(8h68xUG`#wI;VW_o(;J;f{mIjN3rn^RN z?=+9L{!I&KVyJh1bK)I=|MnusTfTW^cE7+E?*9q+@$2m@B`kh7#qpD^FCUQdiD|$K z8)n+pbPN8iw~qmRY;D`EJp!M?rVa%tZO#d6P}K238BYvKxs!B4v-=Ox$bo1D5c=%m z_h&2YGW5lx$|sb6?Si8diTV;V)xdnVb=uT{W~W2Y?Ij` zx7CO2(9=L%iPx%cIsTV~y$wI@`%q@9<@l@8d5sGvr=`=oNLfBX7n&ZTiX~d<)ie#S zo?dL^{amAjVe^{rz3q~+UNM~f^q7!CL+1_7Gc?!da#+nin!4P>>8q9e$P>x03HOilBAE^0Mn?Rr#wIK~_H5e^mXd$BolnW9?pbCsy10veO zQ4W+vC@LaR6^@QE;$#vXV`@9shKtBkuioltjZ-7kqbX*2@VI2Be1gq?jKD)2C9#+~ z!4|q@MSI*$ldII2J~lq?9-ht5mf-5vjO61w{aM5H@UKdZpH!E*}X*9m>E z*KWcPx_5I(+$&ml&dK`)AA~lp?kRj)ylI)`_%&ZOeY|w86B-fXea_E+mL7D3W-?aV X$lAUS3&p?gfZJu4FdhO+-GuiL`07>f#MN4)U(tK}*57 zP@qK7D8BVLSGi)UgEHP*d*GJ<&`iSvQMGaaryX_c*OcA}Zj~bw115EI+0zWC89iqB zFy<32A1w_$^?s$0xy@jnUy_Cd2*snGUXSZSwMNa;3u>-DbCVrshB@vg|D0dVq|opD zl7Z2l0H6dRv_J_r#%j#JN*jtWl@w@c`z=DJXWxxMBd$~LrOEvpAMDNiKg4R|4i|sp Gm-!u4d!dN{ literal 0 HcmV?d00001 diff --git a/test/data/sample.dta b/test/data/sample.dta new file mode 100644 index 0000000000000000000000000000000000000000..0289ddf92640c5dcd1bc6691550b69bb1ae15455 GIT binary patch literal 13313 zcmeHOO>7%Q6dtF4p{by#sS4s^P$O}H&92iVjz)_Uq!5%JD*3<#nr`edS+tv2yW1eQ zf(DKt5l}A$6{%9iAyO_}dgz6umm*cPJ#a!CB7{_l3q*(rD$y`+e&V&0Ra%c>eDFl^ z?EJlX^XK*Lu%zaWm`x`%i`i>ieKbK{@Z?D?iPAnKgi3)!#tll#`8-i&)=|ken#Yf zW_UT($8&7Jta!l*TxZ53(@~{NLOX>vx8Uyq{oCBaMqM*>pL6K)Y`;>NbpkF*%cozN zJLhYeSxo9zicUl{YEXo&q;Y=aSLlxSG-}{isMTtmpCX|79xw~#7Ky|8AoAQW+NKiF zuA9PxjqN5}nh$&@3hkbk%upiNWA?vnm3RrRdw01;SN>Vvx=8!PIw$s(f@Aq-s3$5S zDLK}Ve0jtF@x+C$+jL;z@dm#A>xm2Q-z!Yj$C{DqGXDL_$Bhj9?q0OK1*(pX4NXoC z<@3qws%bg9dSf3tOK*|9F_dDxb7Y!*mrV0JTQgv{!o8)Z*5KZVbXnxrnv|s6fQXwwRNKfOEJ&%l$RA5D-2;vw!r{B?!r@_}7${Wa!_UG}+)f(@!U_#H8BWfOSo)uTqNpq8R{BRku3wrtMlHQZZVogVnP2qAcIvv07>mT zG`vI1%)nr#x33Bd_}Wr>E%S+y)It9zYs~MIcaTJ$sxp zx6SfDy3kH1R&|?je$@;}E71wJ8_j^I(uuO&WUTB!rnkCHIvJa2Mn#oPU=ffM*aQ%$ zBNOVq=zK$=olJK4CYk|hJu=Y@NXwB4r&!IXsFH~?#fo>EHEg2&g=;IZ$pK2&+5u5z z6Ig`Oca9nJ>EV>19T8PJQ9tVR_4Q}Xm=6oz>)K&a9!b556$V78^uXZvq>z^cP21GlVWUGL{j?*3hu1Y4#X48La zAR9a9aE8^qOH{c;nPJ7Y;ABGIii&7B`OcWSEspJc`30nnSK#Qqc=v}KLhs+{zI}cq z1;_ElhtY?_GZ~itU|n+VPpfFrIkoZnM>jhuaA36CowX$E?%$!I0K3rf6vW;{D)EFoF%HH4dY|qp!UU zk;j4oX{|To*}Zfi50m6!8!`f^KBi8VQ{qK)eau`ZQ-OYnm^oRN{blnnUx}DG5z{AQ z0%83}MJM>xNEiKw1p{=@=FPu{d{T7j?skV+dW8N% bg7^4R{Yl3oXq5Pn3AN)?`:#@'="000000~000000000000000000000{}\0000000000000 +00000000000000000000000000000000000000000000000000000000SPSSPORTA8/201812166/172 +8211O/IBM SPSS Statistics 25.047/5B/71/6/MYCHAR1/1/0/1/1/0/C9/character70/5/MYNU +M5/8/2/5/8/2/C7/numeric70/6/MYDATE40/A/0/40/A/0/C4/date70/5/DTIME3E/K/0/3E/K/0/C +8/datetime70/6/MYLABL5/8/2/5/8/2/C7/labeled70/5/MYORD5/8/2/5/8/2/C7/ordinal70/6/ +MYTIME3D/8/0/3D/8/0/C4/timeD1/6/MYLABL2/1/4/Male2/6/FemaleD1/5/MYORD3/1/3/low2/6 +/medium3/4/highE4/N/some test text as notesO/ (Entered 15-Aug-2018)J/some othe +r commentsO/ (Entered 15-Aug-2018)F1/a1.3/IPJ2+3/IPJ3AKA/1/1/1AKA/1/b1.6/CQCMC ++2/CQCNMKA/2/2/32KA/1/c-13A.9/G9Q+4/G9Q+4/1/3/0/1/d-1.C/8CO+2/8CO+2/2/1/24KA/1/e +13A.9/*.*.1/1/*.ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ diff --git a/test/data/sample.sas7bdat b/test/data/sample.sas7bdat new file mode 100644 index 0000000000000000000000000000000000000000..cba665fee37dafdbaaa95e77b5223577c150cd78 GIT binary patch literal 131072 zcmeIz&x=)6902h1oKdHeRG5W9(8C00VZ-Riq!9F;%%s9h3G)^S_G5H3C7*(5I1pse z!iCTe+!QTZ_y+`TB8p5GxihjwV2f(eB3c-RrP)mP-gnNtc~?g!Yf z@0Dgc+C8n-S6eUC^H#_H%7erN2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5cm%ZT;6)2ey8o7 zz815Kdc*Rp`OU3X+}G7P`drMf>P(0IH(QBk7j<@gc%v16l+u+}V^_>KO8GB;ZzWEZ z@+V_{TPeRS=4&Y*>z;Y?7k_IFD33omPk!4x`JQ?57q8!W{gjW}3y~qb_ikQ(c{0zR zxb$@`JBPD}uf%6`waACis%zhUUz>;v3>M{D(O*08?YR)nEq~_YA3l%W**Zlj>gsd9 z)^^9lzPxOC^IQx|Rk~hJEp_aPjbh%u>z@_jrI26e#E*ST&d2UrtB(6T_;YPCcEszo z`ln}Rigc>ueWm*gS$=Pvry@Xr009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly@J|G$|ClLCd5zxC74oB9J_y}e@P2V_ zXXl&0EE`g756fC_6*EuW^ZXOV%vTpUKfiQe$Mp|}o-Cd$&h3u*_J4miBtP1v5td}Z zXmM_5=bL}~{nv_}Ke535SF%gwN4pFZ@w!rWy7l;GHinC@+u9o7yQXIpgid&hf z%_i4G^^IY1ksFTjt{5MU@f$HdU3y9({23QLGrML8)lwO*H|pb!ksZycJ^Lq)gs>(X zn;*@oL&us~y=7#)ksr;e$%&(TL)f`bNaAT}F zb#P+$!MXDfADQe}uQf5BlOI`EyTk``CT6;0ot|ua;J?#a78g$MLwX1EcRw%XIJu^L z+Fv$8V0xmh#PjjRN$*LjXNPOtRp9d0 z3-!AWSYT&*D(Q=#;s@jW6sPZfihE1_#s9zpule_e_00JP?VbG}3Sn8%et1siZtHeu XU$V+={}Rpg`HKG|9V?_;Es}o$^x}4% literal 0 HcmV?d00001 diff --git a/test/data/sample.sav b/test/data/sample.sav new file mode 100644 index 0000000000000000000000000000000000000000..20d0c5ce6689a60adfa329a17b4347274e9a863b GIT binary patch literal 1651 zcmb_cOKTHR6uy1Mq&^U-LN{{3Vp=fTOie9?(nl>OreQidTONKNb zn5Hx<#Y)+%lvhkSe>=D31ky0&Bd2NmJ8h{IZqD*26B!mloPiEODPxhrv#RIkWwEm@ z^9%Vqi<-7LHlm9hbtOvkvu@dmI*^*4&8 zji|nBZ7EmT1ND!v_cvE#u71mQnwC3&73s@@YE&Z<sQFhx(T zD~}Q4c}BcH?x99Ti5qGBSAOhI-UZK+ zeKiS9hj&gG9FA8eoTDY1?TZ!8305#9E<8*E983Y;+39eI zW2uW!(jXmDrzD>vu3v&O{t_B-Se()~eOFCqXS3row?ZoYArd+s3wa$6MeO=xAlV-x z)nhU8nuvLwgWZX~v$?!h(7y8QAzg@4-L4h)`2K6k_hl@-ncUBm@Xsf%h8FYu>9f8S z$73#(AH?tOJs%?e@KtDqkWYW=wHRNgl=eQL+O~~-z`x7hNC6*N`)^$R*L%dp d+sxx)w+?dMUN7-!b@!Xj&kQBbt1A^k#2@(7G6nzu literal 0 HcmV?d00001 diff --git a/test/data/sample.xpt b/test/data/sample.xpt new file mode 100644 index 0000000000000000000000000000000000000000..da15bd3ecda8b7482693d10a64df4debe08aecd2 GIT binary patch literal 2080 zcmd5-J4-_`6i!>;I9M!#TM7;;QhKqi+M$r!Tx{W{t@qvz862z+a8Pg%Iyt#JIq2Z# z-w;Fvg}V3`{3GUZ)5mSGiIZo@$#-&c&Ua2iQgIM=Jm@*)x`)@LxK-Oj9)*?uwX~G% zuL7Zue8{Cwvv3gZWOBmpo}OQ}0RRGw7_1y>?poVLD_hLxi@8D+kJw^l8mK9(mpH^` zJK($arY&y!W(JHWtvd4LsH%#1rj`81g{x!2%QaDG2}Yx zeXiT`PZ?O#{ErS$@{Y?r_Dh0Z@3&v=D>g$= zz^-;u`@(YgQo8PNk;}bNMMl}4jn9srmrjJdr60t~$Hv|#pCHJ`9yR>k68N|rX2=K# RfYRLzVJ*r&)YZQmegGQy4vhc+ literal 0 HcmV?d00001 diff --git a/test/data/sample.zsav b/test/data/sample.zsav new file mode 100644 index 0000000000000000000000000000000000000000..092c78ec128d14a060d7dc96f94e391e6352aa82 GIT binary patch literal 1656 zcmb_cOK1~e5Z;=$Ce=Q$4=XDCVzKQZn#A-$AtdP|Hkz#^8>F^o2Hb9Q(_t-TMV%@`$&f=HDIXmg^@ka=Q7D!avtmq@vQ)?9 zg*}iX)*4Ou=-3UO*1~`oVAsM@==`xxT3y=tjU+- zw!PK7n3gCPg(1G$fFvUVJkALBhdoeTw!N0LGDBOK5%AT@Qam*+8@|<7olGT?v~K38 zB_I@jPLU<1N;#DtvnmsFETfQ%bcdC$kqq4}otLx6PW@$fIh_67hfSvOQ`96iOH7ev zDWm=qH_-U6{Lr7c3!EkT>J+Idt8}rrGbrx(JAr^VvDmk($(C+*x8_QQ58N>WOP!FRO)z}b_$HvYsYW{Sk28*a90 zXXp3U`r6v(x#m_j@h#N!C;4@G*>NS?_NLjAf3R@o#odXi+3nSHws#E$yS;w2J-?Dz zEIumT4lEWMpRTwq`N5wj=i`N3E%e}XIY6^{o`;R|1oF4Z)9P{6N{!8|bFNsx& zXB+G8R~x0$ZsSHiIlt7`@b=5*%i`IvuyVDXyY?Z_wq$ge!401p?XXSqpNv1TN`NP% Fe*xZGI)?xN literal 0 HcmV?d00001 diff --git a/test/data/sample_missing.csv b/test/data/sample_missing.csv new file mode 100644 index 0000000..54eec2f --- /dev/null +++ b/test/data/sample_missing.csv @@ -0,0 +1,8 @@ +mychar,mynum,mydate,dtime,mylabl,myord,mytime +a,1.10,2018-05-06,2018-05-06T10:10:10.000000,1.00,1.00,10:10:10.000000 +b,1.20,1880-05-06,1880-05-06T10:10:10.000000,2.00,2.00,23:10:10.000000 +c,-1000.30,1960-01-01,1960-01-01T00:00:00.000000,1.00,3.00,0:00:00.000000 +d,-1.40,1583-01-01,1583-01-01T00:00:00.000000,2.00,1.00,16:10:10.000000 +e,1000.30,#NULL!,#NULL!,1.00,1.00,#NULL! +Z,#NULL!,#NULL!,#NULL!,#NULL!,#NULL!,#NULL! +,#NULL!,#NULL!,#NULL!,#NULL!,#NULL!,#NULL! diff --git a/test/data/sample_missing.sav b/test/data/sample_missing.sav new file mode 100644 index 0000000000000000000000000000000000000000..b2c1243eba2ad9796dce7689762f5aa9da6874a4 GIT binary patch literal 1779 zcmb_cO=}ZT6n*_llUhZj3f;&9i*3PZlcu#4N+;*?$UV9%*2|stLiDY_uc!>yYIZoyKiiDEv`-6n3U<1 zE;A1@8JWo@v*}DWy_Au&iRhN)N!Ep}I~ zo34z_#S-(0>G{NLd(nA43Ufd1)!t9^U_;+hxat&d zl@3#St`}Mv9Da&MhO`Zw2-axR^1Nbu-0F)iY3gqYlc9iF|Ji? zC0$i>@o&eKMol^uge-SnOsqIKC{7uhi6-mCXnZO*H+kM-p1z7(!eulowyhlZpR`fN zf7QqP-~N4gKSyD0e8gqpZ%E_4`GOBlKEFG;T!)qdj(pcTdT|`YHAH9jisNtLx4|WOs^EJ95-D66hL+-}c|}Xe>Sx|H^L%-ai9z7Vjn57d)YX(~HO*Gb-1O7ZY#sB(H^>*e K?=-IfhQ9%wOIZ>C literal 0 HcmV?d00001 diff --git a/test/data/sample_missing_user.csv b/test/data/sample_missing_user.csv new file mode 100644 index 0000000..3544782 --- /dev/null +++ b/test/data/sample_missing_user.csv @@ -0,0 +1,8 @@ +mychar,mynum,mydate,dtime,mylabl,myord,mytime +a,1.10,2018-05-06,2018-05-06T10:10:10.000000,1.00,1.00,10:10:10.000000 +b,1.20,1880-05-06,1880-05-06T10:10:10.000000,2.00,2.00,23:10:10.000000 +c,-1000.30,1960-01-01,1960-01-01T00:00:00.000000,1.00,3.00,0:00:00.000000 +d,-1.40,1583-01-01,1583-01-01T00:00:00.000000,2.00,1.00,16:10:10.000000 +e,1000.30,#NULL!,#NULL!,1.00,1.00,#NULL! +Z,-1.00,#NULL!,#NULL!,-1.00,-1.00,#NULL! +,2500.00,#NULL!,#NULL!,#NULL!,-3.00,#NULL! diff --git a/test/data/string_labeled_value.sav b/test/data/string_labeled_value.sav new file mode 100644 index 0000000000000000000000000000000000000000..a5bb89e1994240bb81c6909b990df1416f5ae598 GIT binary patch literal 7921 zcmbtZS#ul55msW0;9-drCEtg}v1E}>U`eKA*{(`eaFK)ok|-=lm8J5-5IY8jSnMEs zKn7m&b5nWpABgRI-8~B)@>{kFB>I@%nV#;idj|J*+h6@->$7bYbb>(DE4_-^ZMGY# ztcF=ONxyseaA31xe)?tKjUPU@bDe|-y8p0ibQokhQ-8j8?b@IG2mh`SEzb}Cc=WY8 zbW`=!)5kmCJ=yv0=@a#TJ}^x7-^7RB$ z-UzO}0se3JG#WC8QX|b=Y<07sn~8~Z93J#Ue*Hj4nO3TOEc9wS|JjUBU7ie>QTsvo zn_V->Pb1q`?k827O_DriLA}A#QJYmAnz;NtH%XSkfbrw&jjvd5sj6otiB-L^<7tyb zmT2+TS~bC}OnmBJTyH$}dWjhu()*g}(!L&(PQCF=v?;M!^C=xs66PHZf zcr5SzmgP4NtqDz%>i&?1!ES~5w+~$sCPQPf29st#Y1kk^y-{WPdY5G6xpD`>dA(6% z{&JvoLTsj#I&KO74(w$^)zE3CUN+(Zsr(3?W>4g+^uF@Z6z}4Z^+uilmjapDaZ6}# zhvsy}j>wnU4mR1pzwS5wrnghB#6#>b<|P5}-VZJrr9+z-MU(CgjoQy?yZrcOPxO{* zCe9Lv_3yG|Gw2T^lfJTp^1h4G49;NwaZBXy@=6bENDEERr`qg^{EZ-I7u6ekJZ`l& zM5x$!Aa2I|g#RwD^pEU_{CYEWks&|tvz>K*FJjXB__)<%{!-h;p^KsSJ=1prm;cdz z9Jg8*<*)hqxYcI)_2-BPg#2;qfa&F+@3Kq#e#ra{-gk;PpJ>cgs@iz`h`)Bt;7b)6 zRZWm+5p0%gdSWwJR}WQnUskEqZddegG$DZcF4Ciqb?+Yw|6a?@&7^#w2Q!_xB%SMl z+TuOa5q2~l=w!By7w&d<_`MWIA)un8({gC=&PZFz;5{#2uVk4WS+zy~`+B<($hg}z zH_l@l>X2q}pu@!SJ~o19YRvTZj-_|>cn|`S(arVk2 z6B7rX|JPuQEknkRO{knsV)&$B5j#U7lUtE|wC(VR*joj}B=!s#84(-)5dM3>eK$cS zvriU;=E2bG9*N#neA(>^|5k0N#}hq>O}Ua_k7KNXo;@$$XikhxTnb-JEQ&>8w-NYx zbtwK$blP`3uF@0!ot8%OWC)tA8bUbEhL-2Fe=PF*68H5y)^V0c=P}7zd~_w>WttbVo#yw1*L~yAQGqGejeQq>`MEXG=zQldkQxBA zml%zysE(sh!V?4!q1-sp6 zY=5mgbQJs1gsK9~_Z#NlY3mq36}o=8>+&?k(mCFMi`)pU1ZSFKjZu_wk@@#t=!p*D zz90f{#+bO;qGCjqgjj$b&I8~g>)#C!e_?D@4N*8dPlKu%nlV&?ldQLOggAOFzKdf0 zT6iOVWZccUsE_42U&BjG69d*?vU-7AYGi?H-Sr;Z%Tc*=6=8*;)?rXRGZkGKS* z>U_3nu?0d2H^qJjO+nwn7mjwll4dzq7vLz%Kb8LNQpQN|6d5D?1UzN^P09bZ<*re^ ztTw0%qx={7`@t++q`mYASMpu-R`*PVzLBzm#{);jeo%z)h2Vt-@RQ{w${Vz2z)hx? zJE*Qn2dg-8-w^-oBl2~QCK9;F{Ck9m=#z0aAcW*~Z+Sp081!P`q1X|; zL2oE70xmNDt&R&vI+}Xyb z4tTx7SAvTbSct*`xEaM%2Q5wk+k}fUpG724R8u_^KJu?WbVw*b;{q?(bqlOuTfjrM z^QLbZfPYLcRU(`z4B;NrcNVx;R&C@Tx;L~j^uDJo-#A(nU;2j5MjqfE%a^N($*8<~ zKJd=lUG!robVdO%G4=uXSpNBQos2Ev2WrJU_rS+Y;2!do(g`a8?y>$2?5QnxW{HUn zAsKLw`5V%SanqM43;4shXDnZ?xI^`ShTfQaFF%i{@xDVoUjw=KrQ|EVa9AkN@j+LP~0v<8_ zX?@0h?xLv^n8gMaNJO(CKZyPc`rAyO!fC<>4zZmZD4gg~PVhIT3uS1!q-rPaf9{O3S2elm+$~LnE%Pq z6g9d)h@zV*=xxFa)_dzkZbLdsdPD_YF#l$N;*z;hGM{nHM*cJZdJo+sT3GZ$@*bLR zk@Yg$OIeS+X1!Zz#&X|d&^NUH{2*^xeuEEIL!5}*nELR&g5OFkuxYD?G$aoquUUVs z@iPkVScka)GxC-B8y#memzU8NQ@}}6fHxs;S^i0r&U&hfoYhH+POC}1kEbC({<7XX z7uuDJ$b)=k`7-beeT6@LbKk-ITQxK}sK~L)a72i+Fa0`^Z+yYBhCI*CW`(;c-4B5QF|JpF@!!BA2+K)EKAd z?J00Y=-)7#j#E4uvJhd^+zuFgY|sA z9_q0GG3Lt`YVQz!7PI9%lG$ROEHCbXFb}3zs&qBz7QhcXauHYbqX+QU6q{dc6w|&* zfoEZt^_SWGd(~mZ&&wrn1B5Q^$>Vn-FqT>f%^%u z{L)|2{D|><5WRPlQvZF19$EVJ3rm(3Ue*k|tbg;w;4;Cf9**vA>c~FxOC5|4?G$=^ z(7S%ZXM4zZ_yDy;_XYrp-R9#e9WXZbamuvu$d-?R0%Kj^IOH{we}?gp<8WOLayJHo zEvlm0l?zM`Md;T_|Kmo6JSYbma}rffe2{cH5`R)*UvKF=cIVzCKN9~I@&+5uj;zzp z`#>julDu@_P(Nc${v>+C`;*cWjlPj?9Q6Z!CHV(?4)}S-_T(gtYXEd*kvB~KWV?GP z%5Wt4k?DP$B$?jf#;{a(DBArY`H|#HL1fXFkRNMAf7GJ7UdC-QK(mK=qZ8E%v%=w< zmO%a?y}KPZJagnt_7DT!MGsND5dhzAq1&9`Fmom1+9x6}wen<+n{4u%=sVnwQ21ee zc5%%?`_A&E6Wo)fn&Xi4*Qycl%}fUgIvqUZFXG>VzbxCKFY`sBk)KHZW;aJ!;CNTcSU18C20Cri$b~k0|mh>D|S> z&Kb@h;#el=Zm96^Hu+WbykA>7Im69X(Yk{~qUde%D*-Uho4CHhRTFyTi>@UfuTO~o7JfZKr|<%s#|;DdllTuL?`$X~&@t1+pmaiH> z;2}Q|e+7<2w4otjHpdh3-@CB#OA8h?&QsF!__T-;Y4wUIkvt{6&m}%DSS^-w**e7) z6|&91>KuV)&moQ9OnxHyn_W8~Y$ZPt{ZTN%>F~@98Kg1b~GH+vf8X>cj_S&n&Ic*bA;iXjuo)mpsL^>b?%&}RETf&s4l(c zPL4a0mrz#={39Aythkq=#68cwCXgag1>P@G5ZdAo`UP*}7{PZHWJwNxkQ{y8nBtrE zxdM9g#74-Y*BXK1w^jOmwS*7-Zc4mmeD2^=Jip^9e)px{WdHN4=oh*)j&y$i{TKQ7 z`!E0c&C_Ie4Ik1az2aT?%J014c`Nu&zw7>eia#`H10R|Qy^sdcL*ME7H(A1idGLb< PkMLCe`ridj`Jny>f+L_^ literal 0 HcmV?d00001 diff --git a/test/data/stringtypes.dta b/test/data/stringtypes.dta new file mode 100644 index 0000000000000000000000000000000000000000..fbb31c37211085422ac7a9efa01baec41921c22e GIT binary patch literal 13185 zcmeHOTW=Fb6duQhI4QvnO-s2I7i{jZUMDdDjTR6oDgv2{N#dro)_6TpmO8PuvqYE& z7=*Y;yz+*GDgx~zzX7TIfL`8s;hiVmkzi(LHVzxWPQYEDvr2O2?77UD&pYSLj5m-t z$U()jgA6!{Q5o9?v~dL^f(>0Cf||#`Qr*E-n=2g8AAwqfg~yB-fZ8zw=8P~97u0eF zte_IE7=Xz{3p&;mX8ldoj9z^pF^tVbCY{M7^z{Br7HSPC)VxrpP>ned*Ih~1ghZnt z>5Hh~^)5m0tPu26pP+}g3OXU_(SE@fBt0+b6-lASdxbN|wos{pi%z|Ui4g)}5CNSa z3SuA*Hi85=01kr7fP$9*grMeA=jPLh4cC|eJODitU?va&1eghU0GYu66M+aIz(n9yYEhXXsoGA(B96Pnc*2$k z&mxZ^k0T4`DAeXu;Y8J*LJl!f`-rRPuj77xfj7@^fuPxqPsvap6zDV?lV6_Fvr9DW z*OzLTS(4$wRJQ+JhCg|0VLrrCjX!7U^oic^?oui{KtMYEZ>S?2S+=~hyO5m0HnFPH zKPHiFPFjpRlXR-JLb6&qhfRm90-@`_72~lKOI`$&4@srJqFp10Y#Y^S=wW=_7b{k8 z+MG;n*}84d(7ye{-yb=7>U93+Uy9bbUq_hX5CJ_aV_%Qw@_VG$(WF^Hg!~@wUG-(( zSG0q6(;gb9eRMTl`_DH$E7xrJZY;%;1p#Ds?Yi|eOe1vJ?6(^ebUE#$QQ9?rw2Y@6 zYr?|z$e2}jCP#m;%Iww|%d9d(`9Vhi-o9h!u5r;BpLhk)#N=!)KWv!}8#uD-_jc|s zgXf#Xc-}Su1)xC61KN8a z6oBdp6e&PWVU+__5pnaQDk6(pRj4AOiij#AY9Y;&r`n0Aort^DP+K{*7k=|$fI3Y0 zpPa_bd(%rEa6xkt*~oOT&A#HZOkzN7!JRUD$<9FPQV1vn6aoqXg@8ifvqS&@gxHV! z+aMn64Vi1qzx@1s8EZEYHOsRFG$2rO Z7x$38SzF<2e@W+kGtTe&*v5a|yaEuZJ;wk5 literal 0 HcmV?d00001 diff --git a/test/runtests.jl b/test/runtests.jl index ae4b1a5..4b6e281 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,6 +3,7 @@ using TestItemRunner include("test_abi.jl") include("test_read.jl") include("test_kwargs.jl") +include("test_missing.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_missing.jl b/test/test_missing.jl new file mode 100644 index 0000000..9ca0b8d --- /dev/null +++ b/test/test_missing.jl @@ -0,0 +1,78 @@ +@testitem "missing values" begin + using DataValues + + @testset "tagged missings (alltypes.dta)" begin + tbl = read_dta(joinpath(@__DIR__, "data", "alltypes.dta")) + @test size(tbl, 1) == 3 + + # Rows: 1 = the value 1, 2 = .a (tagged missing), 3 = system missing. + for col in (:vbyte, :vint, :vlong, :vfloat, :vdouble) + c = tbl[col] + @test !isna(c[1]) && get(c[1]) == 1 + @test isna(c[2]) + @test isna(c[3]) + @test missingtags(tbl, col) == ['\0', 'a', '\0'] + end + @test missingtags(tbl, :vstr) === nothing + + # The label set covers a value and a tagged missing value. + d = valuelabels(tbl, :vbyte) + @test d !== nothing + @test d[Int32(1)] == "A" + @test d['a'] == "Tagged missing" + + # strL values surface as Strings. + @test occursin("long string!", get(tbl[:vstrL][1])) + end + + @testset "SPSS user-defined missings (sample_missing.sav)" begin + path = joinpath(@__DIR__, "data", "sample_missing.sav") + + # Default: user-defined missing values collapse to NA + # (expected values from pyreadstat's sample_missing.csv). + tbl = read_sav(path) + @test tbl[:mychar] == DataValueArray{String}(["a", "b", "c", "d", "e", "Z", ""]) + @test tbl[:mynum] == DataValueArray{Float64}([1.1, 1.2, -1000.3, -1.4, 1000.3, NA, NA]) + @test isna.(tbl[:mydate]) == [false, false, false, false, true, true, true] + @test !isempty(varmetadata(tbl, :mynum).missing_ranges) + + # :keep returns the user-missing codes as data + # (expected values from pyreadstat's sample_missing_user.csv). + kept = read_sav(path; user_missing=:keep) + @test kept[:mynum] == + DataValueArray{Float64}([1.1, 1.2, -1000.3, -1.4, 1000.3, -1.0, 2500.0]) + @test kept[:mylabl] == DataValueArray{Float64}([1, 2, 1, 2, 1, -1, NA]) + @test kept[:myord] == DataValueArray{Float64}([1, 2, 3, 1, 1, -1, -3]) + # System-missing cells stay NA either way. + @test isna.(kept[:mydate]) == [false, false, false, false, true, true, true] + + @test_throws ArgumentError read_sav(path; user_missing=:bogus) + end +end + +@testitem "read_por" begin + using DataValues + + tbl = read_por(joinpath(@__DIR__, "data", "sample.por")) + @test size(tbl) == (5, 7) + @test filemetadata(tbl).row_count == -1 + @test names(tbl) == [:MYCHAR, :MYNUM, :MYDATE, :DTIME, :MYLABL, :MYORD, :MYTIME] + @test tbl[:MYCHAR] == DataValueArray{String}(["a", "b", "c", "d", "e"]) + @test tbl[:MYNUM] == DataValueArray{Float64}([1.1, 1.2, -1000.3, -1.4, 1000.3]) + @test isna(tbl[:MYDATE][5]) + @test varmetadata(tbl, :MYDATE).format == "EDATE10" + d = valuelabels(tbl, :MYLABL) + @test d !== nothing && d[1.0] == "Male" && d[2.0] == "Female" + + # The types.por fixture inherited from StatFiles.jl has an invalid + # timestamp that readstat 1.1.9 rejects; the error must surface cleanly + # (queryverse/ReadStat.jl#96, queryverse/StatFiles.jl#32). + err = try + read_por(joinpath(@__DIR__, "types.por")) + nothing + catch e + e + end + @test err isa ErrorException + @test occursin("timestamp", err.msg) +end diff --git a/test/types.por b/test/types.por new file mode 100644 index 0000000..cd4b8b3 --- /dev/null +++ b/test/types.por @@ -0,0 +1,9 @@ +ASCII SPSS PORT FILE ASCII SPSS PORT FILE +ASCII SPSS PORT FILE ASCII SPSS PORT FILE +ASCII SPSS PORT FILE 0000000000000000000000000000000000000000 +0000000000000000000000000123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrst +uvwxyz .<(+0&[]!$*);^-/|,%_>?`:#@'="000000~000000000000000000000{}\0000000000000 +00000000000000000000000000000000000000000000000000000000SPSSPORTA8/201705106/ + 1D/STAT/TRANSFER46/5A/70/6/VFLOAT5/9/2/5/9/2/70/7/VDOUBLE5/9/2/5/9/2/70/5/VLO +NG5/9/2/5/9/2/70/4/VINT5/9/2/5/9/2/70/5/VBYTE5/9/2/5/9/2/73/7/VSTRING1/3/0/1/3/0 +/F3.46002GE7J/3.46/2/2/2/1/27/7/7/7/7/1/7*1*1*1*1*10/ZZZZZZZZZZZZZZZZZZZZZZZZZZZ \ No newline at end of file From 240a484879f3d18c4ba266fddf32cc6c38d6fedc Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:15:47 -0700 Subject: [PATCH 05/14] Add LabeledValue/LabeledArray value-label views and sas7bcat support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Value-labeled columns can now be viewed through a LabeledArray: a zero-copy wrapper around the raw code column plus the shared label dictionary, whose elements (DataValue-wrapped LabeledValues, so NA propagates in the usual DataValues way) display as their labels but compute as their codes — ==, isless, and hash act on the code, while comparison against a string compares the label. Unlabeled codes display as string(code), so partial label sets and labels on continuous variables lose nothing. Obtain the view per column with labeled(tbl, col) or table-wide with the new apply_value_labels=true keyword; rawvalues/getvaluelabels/unwrap/valuelabel take it apart again. This addresses the long-standing value-label requests (queryverse/ReadStat.jl#74, #47). SAS value labels are now reachable too: read_sas7bcat parses a .sas7bcat catalog into label sets keyed by format name, and read_sas7bdat gained a catalog keyword that merges those labels into the table, where they line up with the vallabel names the variables already carry. Fixtures for the catalog pair and a string-keyed label set are sourced from pyreadstat and ReadStatTables.jl like the rest of test/data. Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 7 +- src/reader/context.jl | 6 +- src/reader/read.jl | 38 +++++++++- src/reader/table.jl | 18 +++++ src/values/labeled.jl | 104 ++++++++++++++++++++++++++ test/data/sas_formatted.csv | 4 + test/data/test_data_linux.sas7bdat | Bin 0 -> 131072 bytes test/data/test_formats_linux.sas7bcat | Bin 0 -> 24576 bytes test/runtests.jl | 1 + test/test_labels.jl | 92 +++++++++++++++++++++++ 10 files changed, 265 insertions(+), 5 deletions(-) create mode 100644 src/values/labeled.jl create mode 100644 test/data/sas_formatted.csv create mode 100644 test/data/test_data_linux.sas7bdat create mode 100644 test/data/test_formats_linux.sas7bcat create mode 100644 test/test_labels.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 987123d..205354e 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -4,8 +4,10 @@ using DataValues: DataValues, DataValueVector using Dates export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, - read_dta, read_sav, read_por, read_sas7bdat, read_xport, readstat, read_meta, - filemetadata, varmetadata, valuelabels, missingtags + read_dta, read_sav, read_por, read_sas7bdat, read_xport, read_sas7bcat, + readstat, read_meta, + filemetadata, varmetadata, valuelabels, missingtags, + LabeledValue, LabeledArray, labeled, unwrap, valuelabel, rawvalues, getvaluelabels public CAPI @@ -39,6 +41,7 @@ const READSTAT_TYPES = (String, Int8, Int16, Int32, Float32, Float64, String) jltype(t::ReadStatType) = READSTAT_TYPES[Int(t) + 1] include("reader/metadata.jl") +include("values/labeled.jl") include("reader/columns.jl") include("reader/context.jl") include("reader/handlers.jl") diff --git a/src/reader/context.jl b/src/reader/context.jl index 302ccd5..d8d4c41 100644 --- a/src/reader/context.jl +++ b/src/reader/context.jl @@ -23,7 +23,9 @@ mutable struct ParseContext collect_values::Bool # true: SPSS user-defined missing values are kept as data (only system # and tagged missings become NA); false: they collapse to NA. - keep_user_missing::Bool # false for metadata-only parses + keep_user_missing::Bool + # true: wrap value-labeled columns in a LabeledArray when building the table. + apply_value_labels::Bool # false for metadata-only parses # Progress-abort bookkeeping: the last fully delivered row, so a # partially parsed table can be trimmed to complete rows. @@ -32,7 +34,7 @@ mutable struct ParseContext end ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), - nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, false, 0) + nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, false, false, 0) # Rows to preallocate per column: what the file reports, minus the offset, # capped by the limit; 0 when unknown (buffers then grow row by row). diff --git a/src/reader/read.jl b/src/reader/read.jl index 08e7116..76ba3cf 100644 --- a/src/reader/read.jl +++ b/src/reader/read.jl @@ -30,6 +30,8 @@ function _parse_format(parser::ParserPtr, path::AbstractString, format::Symbol, CAPI.readstat_parse_sas7bdat(parser, path, ctx) elseif format === :xport CAPI.readstat_parse_xport(parser, path, ctx) + elseif format === :sas7bcat + CAPI.readstat_parse_sas7bcat(parser, path, ctx) else throw(ArgumentError("unknown format $format")) end @@ -89,7 +91,15 @@ function build_table(pc::ParseContext) columns = Vector{AbstractVector}(undef, n) tags = Vector{Union{Nothing,Vector{Char}}}(undef, n) for i in 1:n - columns[i] = finalize_column(pc.cols, i) + col = finalize_column(pc.cols, i) + if pc.apply_value_labels + vallabel = pc.varmeta[i].vallabel + if vallabel !== Symbol("") + d = get(pc.meta.value_labels, vallabel, nothing) + d === nothing || (col = LabeledArray(col, d)) + end + end + columns[i] = col tags[i] = column_tags(pc.cols, i) end return ReadStatTable(columns, pc.names, pc.meta, pc.varmeta, tags) @@ -102,6 +112,8 @@ function read_data_file(path::AbstractString, format::Symbol; file_encoding::Union{Nothing,AbstractString}=nothing, handler_encoding::Union{Nothing,AbstractString}=nothing, user_missing::Symbol=:na, + apply_value_labels::Bool=false, + catalog::Union{Nothing,AbstractString}=nothing, progress=nothing) user_missing in (:na, :keep) || throw(ArgumentError("user_missing must be :na or :keep")) @@ -124,7 +136,11 @@ function read_data_file(path::AbstractString, format::Symbol; pc.file_encoding = file_encoding === nothing ? nothing : String(file_encoding) pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) pc.keep_user_missing = user_missing === :keep + pc.apply_value_labels = apply_value_labels + catalog === nothing || format === :sas7bdat || + throw(ArgumentError("`catalog` is only supported when reading sas7bdat files")) parse_file!(pc, path, format) + catalog === nothing || merge!(pc.meta.value_labels, read_sas7bcat(catalog)) return build_table(pc) end @@ -143,6 +159,12 @@ All readers accept the same keyword arguments: to NA; `:keep` keeps them as data (the rules stay available in `varmetadata(tbl, col).missing_ranges`). System missing and tagged missing values are always NA; see [`missingtags`](@ref) for the tags. +- `apply_value_labels`: when `true`, every value-labeled column is wrapped + in a [`LabeledArray`](@ref) (labels for display, codes for computation). + The raw label dictionaries are always available via [`valuelabels`](@ref) + regardless. +- `catalog` (sas7bdat only): path to the `.sas7bcat` catalog holding the + file's value labels; they are merged into the table's value labels. - `progress`: a function called with the parse fraction (0.0-1.0); return `false` to stop the parse and get the rows read so far. """ @@ -208,6 +230,20 @@ $_READ_KWARGS_DOC readstat(path::AbstractString; format::Symbol=:auto, kwargs...) = read_data_file(path, _sniff_format(path, format); kwargs...) +""" + read_sas7bcat(path) -> Dict{Symbol, ValueLabelDict} + +Read the value-label sets from a SAS `.sas7bcat` catalog file, keyed by +format name. Usually not called directly — pass the catalog path to +`read_sas7bdat(...; catalog=...)` to attach the labels to a data file. +""" +function read_sas7bcat(path::AbstractString) + pc = ParseContext() + pc.collect_values = false + parse_file!(pc, path, :sas7bcat) + return pc.meta.value_labels +end + """ read_meta(path; format=:auto, file_encoding=nothing, handler_encoding=nothing) -> ReadStatTable diff --git a/src/reader/table.jl b/src/reader/table.jl index a3617b5..a1066e3 100644 --- a/src/reader/table.jl +++ b/src/reader/table.jl @@ -82,6 +82,24 @@ function valuelabels(tbl::ReadStatTable, col::Union{Integer,Symbol}) return get(filemetadata(tbl).value_labels, vm.vallabel, nothing) end +""" + labeled(tbl::ReadStatTable, col) -> LabeledArray + +A lazy labeled view of the value-labeled column given by index or name: the +elements display as their labels but compute as their raw codes (see +[`LabeledValue`](@ref)). Throws when the column has no value labels; reading +with `apply_value_labels=true` wraps every labeled column this way up front. +""" +function labeled(tbl::ReadStatTable, col::Union{Integer,Symbol}) + i = columnindex(tbl, col) + c = getfield(tbl, :cols)[i] + c isa LabeledArray && return c + d = valuelabels(tbl, i) + d === nothing && + throw(ArgumentError("column $(getfield(tbl, :names)[i]) has no value labels")) + return LabeledArray(c, d) +end + """ missingtags(tbl::ReadStatTable, col) -> Union{Nothing, Vector{Char}} diff --git a/src/values/labeled.jl b/src/values/labeled.jl new file mode 100644 index 0000000..19d590e --- /dev/null +++ b/src/values/labeled.jl @@ -0,0 +1,104 @@ +# A lazily labeled view of a value-labeled column. +# +# Stat packages store "categorical" columns as raw codes plus a (possibly +# partial) label set. A LabeledArray keeps the raw code column untouched and +# looks labels up only on access, so nothing is lost (codes, gaps, unlabeled +# values, ordering) and nothing is paid when the view is not used. This is +# deliberately a partial dictionary encoding, not a categorical/factor type: +# unlabeled codes remain first-class values that merely display as +# `string(code)`. + +""" + LabeledValue{T} + +A single value from a value-labeled column: the raw code of type `T` plus a +reference to the column's shared label dictionary. It displays as its label +(falling back to `string(code)` when the code has no label) but computes as +its code: `==`, `isless`, and `hash` all act on the code, so predicates, +joins, and grouping behave exactly as on the raw column. Comparing against +an `AbstractString` compares the label instead. + +Access the parts with [`unwrap`](@ref) (the code) and [`valuelabel`](@ref) +(the label). +""" +struct LabeledValue{T} + value::T + labels::ValueLabelDict +end + +""" + unwrap(lv::LabeledValue{T}) -> T + +The raw code stored in a labeled value. +""" +unwrap(lv::LabeledValue) = lv.value + +_labelkey(v::Integer) = Int32(v) +_labelkey(v::AbstractFloat) = Float64(v) +_labelkey(v::AbstractString) = String(v) + +""" + valuelabel(lv::LabeledValue) -> String + +The label of a labeled value, or `string(unwrap(lv))` when its code has no +label in the column's label set. +""" +valuelabel(lv::LabeledValue) = get(() -> string(lv.value), lv.labels, _labelkey(lv.value)) + +Base.:(==)(a::LabeledValue, b::LabeledValue) = a.value == b.value +Base.:(==)(a::LabeledValue, b::Number) = a.value == b +Base.:(==)(a::Number, b::LabeledValue) = a == b.value +Base.:(==)(a::LabeledValue, b::AbstractString) = valuelabel(a) == b +Base.:(==)(a::AbstractString, b::LabeledValue) = a == valuelabel(b) +Base.isequal(a::LabeledValue, b::LabeledValue) = isequal(a.value, b.value) +Base.isless(a::LabeledValue, b::LabeledValue) = isless(a.value, b.value) +Base.isless(a::LabeledValue, b::Number) = isless(a.value, b) +Base.isless(a::Number, b::LabeledValue) = isless(a, b.value) +Base.hash(lv::LabeledValue, h::UInt) = hash(lv.value, h) + +Base.show(io::IO, lv::LabeledValue) = print(io, valuelabel(lv)) +function Base.show(io::IO, ::MIME"text/plain", lv::LabeledValue) + print(io, valuelabel(lv), " (", lv.value, ")") +end + +""" + LabeledArray{T} <: AbstractVector{DataValue{LabeledValue{T}}} + +A lazy labeled view of a value-labeled column: wraps the raw +`DataValueVector` of codes plus the shared label dictionary, constructing +[`LabeledValue`](@ref)s only on access. NA cells stay NA. Obtain one with +[`labeled`](@ref) or by reading with `apply_value_labels=true`; get the raw +column back with [`rawvalues`](@ref) (zero-copy) and the dictionary with +`getvaluelabels`. +""" +struct LabeledArray{T} <: AbstractVector{DataValues.DataValue{LabeledValue{T}}} + values::DataValueVector{T} + labels::ValueLabelDict +end + +""" + rawvalues(a::LabeledArray) -> DataValueVector + +The raw code column underlying a labeled view (zero-copy). +""" +rawvalues(a::LabeledArray) = a.values + +""" + getvaluelabels(a::LabeledArray) -> ValueLabelDict + +The label dictionary shared by the elements of a labeled view. +""" +getvaluelabels(a::LabeledArray) = a.labels + +Base.size(a::LabeledArray) = size(a.values) +Base.IndexStyle(::Type{<:LabeledArray}) = IndexLinear() + +function Base.getindex(a::LabeledArray{T}, i::Int) where {T} + v = a.values[i] + DataValues.isna(v) ? DataValues.DataValue{LabeledValue{T}}() : + DataValues.DataValue(LabeledValue{T}(get(v), a.labels)) +end + +Base.setindex!(a::LabeledArray{T}, v, i::Int) where {T} = (a.values[i] = v; a) +Base.setindex!(a::LabeledArray{T}, v::LabeledValue, i::Int) where {T} = + (a.values[i] = unwrap(v); a) diff --git a/test/data/sas_formatted.csv b/test/data/sas_formatted.csv new file mode 100644 index 0000000..2f382c7 --- /dev/null +++ b/test/data/sas_formatted.csv @@ -0,0 +1,4 @@ +ID,SEXA,SEXB +ID1,Male,Male +ID2,Female,Female +ID3,Male,Male \ No newline at end of file diff --git a/test/data/test_data_linux.sas7bdat b/test/data/test_data_linux.sas7bdat new file mode 100644 index 0000000000000000000000000000000000000000..5d88454d227a7b9186b32587a1059d0377f27d70 GIT binary patch literal 131072 zcmeI!%WEA~901@mH&Lrr`!H&RiWlv|MGZ|-VuVO@YpO(3DI|;fpbxR2&`LoCg@C$u zp(4^h!G$}w3L=8;Tq^3)m5UM;pN+lFQv%6%@W{lXHH*Gv_hi%$!UJz2nm# z-Z=i@$c`^RerHR#{`2uSNAKFcWAg4WIhnP_v+A^w5QegK`N`~_$uKe!+C%Y_5EeU& z?T6|c)Y#<2eNzuijg3!_-9OgXR90V^o;o(Qua_<6{pQ6d_N1R)OI161 z=3H4>x4!yGVgdvR5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5*BhXszEIk(~JzWODmTQ`Rr^Xk{D zRebNz`qMv&`O)?1@Z6tO;z(8z@J_eWLktL5K{`Gs2kNX$>y@}n_7oAR;z z`uD0*&!4?aJ`EF>8RqB4+r?4+?v?X%6J@^o-EOCHer|HZ^Ot|#cs(-%@~b~>eEvrR z^2-DAGXwIgKMojwKz?~Zer7;^b!EW#1M+82w zCqRGz0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5cppSTsYq?kIgN*-MS-o_-FlxTemdo|9y6AqyArHI~w(WKfA3_|97*S8uee!wl(Vi zY<6>_{&QKg`@g5*{JXLn8m)i0;r(9;&F23uG`s(wg=Y8fgNFUjHthd;*xBg*9S^rR z>OUJAKEL6{G=H^yc8`QFUMo_EWB%4~XUIa@rHb;}HM4DDIGpdc!gb-d?l0Yy?#?ij zw~ypM-R>=6YuJ+4c9c(8dP?#{eypasvMRo-Dnh#`hg)HPjLYUco~^~57++hIdHePl z7qf@qETla-#c7XAaWYGB>|U<7=>JGew94^XRX(p5*Hg657iE5=_3f&{^6HEBM`N7Y zAFIU+wfInsOQrmJo~*S$Rg0gFakZXSc(xYDP;r$6Hv4Tb8_FkVZT8LfpKjQHZQ@wD z5Z|}m@!jq{uS5AWT)qYQncePCxX|tIo8Hsu{$yTH&s|ye#pyjv?fu>261wfRi^ z_ugH$jK=nQ|CqjL-1Hu&Z@Au*RQ2e@hV-1IGL5Iaw7{`5=PrH16}G1LEd5nTak_6Q z-WtbCar!n+aoTUMaN13rwx!&Acjw=1#rIopp6VNJS6}8*gAV6AS$5GbzK|~SqUSds JD_vSS?BA&yB`N>_ literal 0 HcmV?d00001 diff --git a/test/data/test_formats_linux.sas7bcat b/test/data/test_formats_linux.sas7bcat new file mode 100644 index 0000000000000000000000000000000000000000..d40588026ac29b8977f452b3895f58729653cdcd GIT binary patch literal 24576 zcmeI3?Q0xG7{KRpmuq_M?Ovp`rKV&wm8L?DdkJUG7wqQ9U4y&c_4b-3DR`6;YQ$F1 zXkv@Dsoz8)Z9$@f{sp2`K}7{o#1B>!{AQsRKSYrVD)yVAyED&oo87%~dq<>ceJ13d zndfb0e>1nq?(8z=l+S)yJo4_A-fuoR*TXh`nSN<|f1)>gC(C9-UFlH6EgW*`v!Q{k zZR6pFi>aBsIIfS?#>`s1Sgsmc&Ey@Ff>zV?a*5mNKUjJod3foPcI}1X@kh>H(rg{$ zAMb5QZq*WuMTEln$R|_7skD*RXOBO<;HX(9nM-CfL%Dn|nI1{rlU!vI)rFB>Y z5g-CYfCvx)B0vO)01+SpMBu+9@EYt}d;4f?<|iU7%m!7ouc%+E7u03-v^uL+)%(01+SpM1Tko0U|&IhyW2F0&7O#pMYiC zM%iV>)Wzf2TG6NuP4wlW2&sP*^~V^s zf-vrHV$(~eZX18-_Q&m;kU8g?*h*&Aw6m_j^WdwGxb*P22t3uyrX3Wvzr+6iiiRNq zmzvlLWn-sHZBCihF|*1mrP1-4p;cr$|7RDdehViAYWcW)fsguKT^fu_MVYu%$2j?5 zyvx)WjC=Ju*Z&wasTcNP{r@icY5>-|26pkjZ-DnrkaGpT4AR^AL_fv`g%bTy zI-XP`0&7QLQa7u1HaJ-^4yhM@m+#lyZ=}WhHCsU6Sf^M)Db~OLr5*_LqZ)ZOy|#!me!I>mkMUL zo7gQ%=@R5K!72rv7oRvj$Bmh}Cufg5g?XDvTt5n==8Y~sowWIESa9<>1@^n2hYAj` z1s}l6w&DV5^wYY+rHhLXra{>{3wm_M=QarEn1)F{KYI9_0ejCPR08tZ3ho?aI|5>E zD4%UGU>J+ll6>BO*1dja!7uqJR08tZ0qz`RI|5={eDM5)GI+ku*3a$G0gT0JNj~RJ zxcQs}zi&@KB_N-EaOWV~5fJ0zgWoHZ!Si*-2gblqBA=rt-F$HU47Ry`2H*gk{GbRR zAjZW9*A2=5X~3m3KD!_q#$vao?DKQ}IX9mv@OxttDgoCIo_{vSAg&F;rSA>l)5b7w z3fE=p`hV$}ojDxrP-Y}MobS#(IB;f1?O0|S^^8Tu%sST%SHv>`kHNnVJ&q&QhyW2F z0z`la5CI}U1c(3;AOb{y2oQmFM&M2B`ad!s$88lB&&1wa)O#m*i4Ub0WA{44jfv}j zy#K`6UvNJ_hVIhj-G1qR$3eLpw8`367-UiFM;6@OM(Je7yi&u#uot9Pgsi;c!3VtC z>Y5g-CLA%TAZ$NS${ literal 0 HcmV?d00001 diff --git a/test/runtests.jl b/test/runtests.jl index 4b6e281..58db9bf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ include("test_abi.jl") include("test_read.jl") include("test_kwargs.jl") include("test_missing.jl") +include("test_labels.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_labels.jl b/test/test_labels.jl new file mode 100644 index 0000000..f617f8c --- /dev/null +++ b/test/test_labels.jl @@ -0,0 +1,92 @@ +@testitem "value labels" begin + using DataValues + + dir = joinpath(@__DIR__, "data") + + @testset "LabeledValue / LabeledArray semantics" begin + tbl = read_sav(joinpath(dir, "sample.sav")) + d = valuelabels(tbl, :mylabl) + @test d == Dict(1.0 => "Male", 2.0 => "Female") + + la = labeled(tbl, :mylabl) + @test la isa LabeledArray + @test length(la) == 5 + @test rawvalues(la) === tbl[:mylabl] + @test getvaluelabels(la) === d + + x = la[1] + @test x isa DataValue{LabeledValue{Float64}} + lv = get(x) + @test unwrap(lv) == 1.0 + @test valuelabel(lv) == "Male" + + # Computes as the code, compares against strings as the label. + @test lv == 1.0 + @test 1.0 == lv + @test lv == "Male" + @test lv != "Female" + @test hash(lv) == hash(1.0) + lv2 = get(la[2]) + @test lv != lv2 + @test isless(lv, lv2) + @test sort([lv2, lv]) == [lv, lv2] + + # Displays as the label. + @test sprint(print, lv) == "Male" + @test sprint(show, MIME"text/plain"(), lv) == "Male (1.0)" + + # Partial label sets: unlabeled codes fall back to string(code). + @test valuelabel(LabeledValue(9.0, d)) == "9.0" + + @test_throws ArgumentError labeled(tbl, :mychar) + end + + @testset "apply_value_labels" begin + tbl = read_sav(joinpath(dir, "sample.sav"); apply_value_labels=true) + @test tbl[:mylabl] isa LabeledArray + @test tbl[:myord] isa LabeledArray + @test !(tbl[:mynum] isa LabeledArray) + @test labeled(tbl, :mylabl) === tbl[:mylabl] + @test [sprint(print, get(x)) for x in tbl[:mylabl]] == + ["Male", "Female", "Male", "Female", "Male"] + @test rawvalues(tbl[:mylabl]) == DataValueArray{Float64}([1, 2, 1, 2, 1]) + @test occursin("Male", sprint(show, MIME"text/plain"(), tbl)) + end + + @testset "string-keyed labels" begin + tbl = read_sav(joinpath(dir, "string_labeled_value.sav"); apply_value_labels=true) + la = tbl[:v611] + @test la isa LabeledArray + @test unwrap(get(la[1])) == "BE33" + @test valuelabel(get(la[1])) == "Prov. Liege" + end + + @testset "sas7bcat catalogs" begin + catpath = joinpath(dir, "test_formats_linux.sas7bcat") + cat = read_sas7bcat(catpath) + @test cat[Symbol("\$A")]["1"] == "Male" + @test cat[Symbol("\$A")]["2"] == "Female" + + tbl = read_sas7bdat(joinpath(dir, "test_data_linux.sas7bdat"); + catalog=catpath, apply_value_labels=true) + @test varmetadata(tbl, :SEXA).vallabel == Symbol("\$A") + @test tbl[:SEXA] isa LabeledArray + # Expected labels from pyreadstat's sas_formatted.csv. + @test [valuelabel(get(x)) for x in tbl[:SEXA]] == ["Male", "Female", "Male"] + @test [valuelabel(get(x)) for x in tbl[:SEXB]] == ["Male", "Female", "Male"] + + # Without the catalog the label sets are simply absent. + plain = read_sas7bdat(joinpath(dir, "test_data_linux.sas7bdat")) + @test valuelabels(plain, :SEXA) === nothing + + @test_throws ArgumentError read_dta(joinpath(@__DIR__, "types.dta"); + catalog=catpath) + end + + @testset "labeled view over tagged missings (alltypes.dta)" begin + tbl = read_dta(joinpath(dir, "alltypes.dta")) + la = labeled(tbl, :vbyte) + @test valuelabel(get(la[1])) == "A" + @test isna(la[2]) + end +end From dc75eb7b6f5a0ac876f125778c3e251763f0ad99 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:19:50 -0700 Subject: [PATCH 06/14] Decode date/time columns from display formats, on by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Columns whose display format is a date/time format now come back as DataValueVector{Date}, {DateTime}, or {HMS} columns; convert_datetime=false keeps the raw numbers. The format tables cover Stata (%tc/%tC milliseconds, %td/%d days, %tw/%tm/%tq/%th counts with Stata week semantics, %ty calendar years; epoch 1960-01-01), SAS (DATETIME*/E8601* seconds, DATE/MMDDYY/ DDMMYY*/YYMMDD*/... days, TIME/HHMM/HOUR/E8601TM times; epoch 1960-01-01), and SPSS (DATETIME/YMDHMS, DATE/ADATE/EDATE/JDATE/SDATE, TIME/DTIME; seconds since 1582-10-14), with width and decimal suffixes normalized away. Value-labeled columns are never converted — they hold codes. The new HMS type carries times of day as raw seconds with unbounded hours, since SAS and SPSS time values can exceed 24 hours or be negative, which Dates.Time cannot represent; it displays as H:MM:SS[.fff] and converts to Time when within a calendar day. Tested against the pyreadstat-derived sample fixtures in all five formats plus DATETIME13 and Stata %tc/%td/%tw/%tm/%tq/%th/%ty edge cases. Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 4 +- src/reader/context.jl | 10 ++- src/reader/read.jl | 24 +++++-- src/values/datetime.jl | 152 +++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/test_datetime.jl | 95 ++++++++++++++++++++++++++ 6 files changed, 278 insertions(+), 8 deletions(-) create mode 100644 src/values/datetime.jl create mode 100644 test/test_datetime.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 205354e..994d160 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -7,7 +7,8 @@ export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, read_dta, read_sav, read_por, read_sas7bdat, read_xport, read_sas7bcat, readstat, read_meta, filemetadata, varmetadata, valuelabels, missingtags, - LabeledValue, LabeledArray, labeled, unwrap, valuelabel, rawvalues, getvaluelabels + LabeledValue, LabeledArray, labeled, unwrap, valuelabel, rawvalues, getvaluelabels, + HMS public CAPI @@ -42,6 +43,7 @@ jltype(t::ReadStatType) = READSTAT_TYPES[Int(t) + 1] include("reader/metadata.jl") include("values/labeled.jl") +include("values/datetime.jl") include("reader/columns.jl") include("reader/context.jl") include("reader/handlers.jl") diff --git a/src/reader/context.jl b/src/reader/context.jl index d8d4c41..7bb54dc 100644 --- a/src/reader/context.jl +++ b/src/reader/context.jl @@ -25,7 +25,12 @@ mutable struct ParseContext # and tagged missings become NA); false: they collapse to NA. keep_user_missing::Bool # true: wrap value-labeled columns in a LabeledArray when building the table. - apply_value_labels::Bool # false for metadata-only parses + apply_value_labels::Bool + # true: decode columns whose display format is a date/time format. + convert_datetime::Bool + # the format being parsed (:dta, :sav, ...); selects the producer's + # date/time format tables. + file_format::Symbol # false for metadata-only parses # Progress-abort bookkeeping: the last fully delivered row, so a # partially parsed table can be trimmed to complete rows. @@ -34,7 +39,8 @@ mutable struct ParseContext end ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), - nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, false, false, 0) + nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, false, true, + :none, false, 0) # Rows to preallocate per column: what the file reports, minus the offset, # capped by the limit; 0 when unknown (buffers then grow row by row). diff --git a/src/reader/read.jl b/src/reader/read.jl index 76ba3cf..1907aff 100644 --- a/src/reader/read.jl +++ b/src/reader/read.jl @@ -92,13 +92,21 @@ function build_table(pc::ParseContext) tags = Vector{Union{Nothing,Vector{Char}}}(undef, n) for i in 1:n col = finalize_column(pc.cols, i) - if pc.apply_value_labels - vallabel = pc.varmeta[i].vallabel - if vallabel !== Symbol("") - d = get(pc.meta.value_labels, vallabel, nothing) - d === nothing || (col = LabeledArray(col, d)) + vm = pc.varmeta[i] + labels = vm.vallabel === Symbol("") ? nothing : + get(pc.meta.value_labels, vm.vallabel, nothing) + # Value labels take precedence over a date/time display format: a + # labeled column holds codes, not calendar values. + if labels === nothing && pc.convert_datetime && eltype(eltype(col)) <: Number + rule = datetime_rule(vm.format, pc.file_format) + if rule !== nothing + f, target = rule + col = convert_datetime_column(col, f, target) end end + if pc.apply_value_labels && labels !== nothing + col = LabeledArray(col, labels) + end columns[i] = col tags[i] = column_tags(pc.cols, i) end @@ -112,6 +120,7 @@ function read_data_file(path::AbstractString, format::Symbol; file_encoding::Union{Nothing,AbstractString}=nothing, handler_encoding::Union{Nothing,AbstractString}=nothing, user_missing::Symbol=:na, + convert_datetime::Bool=true, apply_value_labels::Bool=false, catalog::Union{Nothing,AbstractString}=nothing, progress=nothing) @@ -137,6 +146,8 @@ function read_data_file(path::AbstractString, format::Symbol; pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) pc.keep_user_missing = user_missing === :keep pc.apply_value_labels = apply_value_labels + pc.convert_datetime = convert_datetime + pc.file_format = format catalog === nothing || format === :sas7bdat || throw(ArgumentError("`catalog` is only supported when reading sas7bdat files")) parse_file!(pc, path, format) @@ -159,6 +170,9 @@ All readers accept the same keyword arguments: to NA; `:keep` keeps them as data (the rules stay available in `varmetadata(tbl, col).missing_ranges`). System missing and tagged missing values are always NA; see [`missingtags`](@ref) for the tags. +- `convert_datetime`: decode columns whose display format is a date/time + format into `Date`/`DateTime`/[`HMS`](@ref) columns (default `true`); + `false` keeps the raw numbers. Value-labeled columns are never converted. - `apply_value_labels`: when `true`, every value-labeled column is wrapped in a [`LabeledArray`](@ref) (labels for display, codes for computation). The raw label dictionaries are always available via [`valuelabels`](@ref) diff --git a/src/values/datetime.jl b/src/values/datetime.jl new file mode 100644 index 0000000..f4eeb37 --- /dev/null +++ b/src/values/datetime.jl @@ -0,0 +1,152 @@ +# Date/time decoding driven by the producers' display-format strings. +# +# Stat packages store dates and times as plain numbers plus a display format; +# the format string is the only record of which numbers are calendar values. +# The tables below map format names to (kind, epoch, unit) rules per +# producer: Stata counts from 1960-01-01 (milliseconds for %tc, days for %td, +# plus week/month/quarter/half-year/year counts), SAS counts seconds or days +# from 1960-01-01, and SPSS counts seconds from 1582-10-14. Times of day are +# plain second counts that may exceed 24 hours (durations), which +# `Dates.Time` cannot represent — they decode into `HMS`. + +""" + HMS + +A time of day or duration as a number of seconds, with unbounded hours: SAS +and SPSS time values can exceed 24 hours or be negative, which `Dates.Time` +cannot represent. Displays as `H:MM:SS[.fff]`; access the raw seconds with +[`unwrap`](@ref), the parts with `Dates.hour`/`minute`/`second`, and convert +to `Dates.Time` via `Time(hms)` when the value is within a calendar day. +""" +struct HMS + seconds::Float64 +end + +""" + unwrap(t::HMS) -> Float64 + +The raw number of seconds in an `HMS` value. +""" +unwrap(t::HMS) = t.seconds + +Base.:(==)(a::HMS, b::HMS) = a.seconds == b.seconds +Base.isequal(a::HMS, b::HMS) = isequal(a.seconds, b.seconds) +Base.isless(a::HMS, b::HMS) = isless(a.seconds, b.seconds) +Base.hash(t::HMS, h::UInt) = hash(t.seconds, hash(:HMS, h)) + +Dates.hour(t::HMS) = + t.seconds < 0 ? -Int(fld(-t.seconds, 3600)) : Int(fld(t.seconds, 3600)) +Dates.minute(t::HMS) = Int(fld(mod(abs(t.seconds), 3600), 60)) +Dates.second(t::HMS) = floor(Int, mod(abs(t.seconds), 60)) +Dates.millisecond(t::HMS) = round(Int, mod(abs(t.seconds), 1) * 1000) + +function Dates.Time(t::HMS) + 0 <= t.seconds < 86400 || + throw(ArgumentError("HMS value $(t) is outside a calendar day")) + return Time(0) + Millisecond(round(Int64, t.seconds * 1000)) +end + +function Base.show(io::IO, t::HMS) + s = abs(t.seconds) + t.seconds < 0 && print(io, '-') + print(io, Int(fld(s, 3600)), ':', lpad(Int(fld(mod(s, 3600), 60)), 2, '0'), ':') + sec = mod(s, 60) + isec = floor(Int, sec) + print(io, lpad(isec, 2, '0')) + frac = sec - isec + frac > 0 && print(io, '.', lpad(round(Int, frac * 1000), 3, '0')) + return +end + +############################################################################## +## +## Format classification +## +############################################################################## + +const STATA_EPOCH_DATETIME = DateTime(1960, 1, 1) +const STATA_EPOCH_DATE = Date(1960, 1, 1) +const SAS_EPOCH_DATETIME = DateTime(1960, 1, 1) +const SAS_EPOCH_DATE = Date(1960, 1, 1) +const SPSS_EPOCH_DATETIME = DateTime(1582, 10, 14) +const SPSS_EPOCH_DATE = Date(1582, 10, 14) + +# SAS format base names (with trailing width digits and decimals stripped). +const SAS_DATETIME_FORMATS = Set(["DATETIME", "E8601DT", "E8601DX", "E8601DZ", + "E8601LX", "E8601DN", "E8601LZ", "E8601TX", "E8601TZ"]) +const SAS_DATE_FORMATS = Set(["DATE", "WEEKDATE", "WEEKDATX", "WEEKDAY", "MMDDYY", + "DDMMYY", "YYMMDD", "DDMMYYB", "DDMMYYC", "DDMMYYD", "DDMMYYN", "DDMMYYP", + "DDMMYYS", "MMDDYYB", "MMDDYYC", "MMDDYYD", "MMDDYYN", "MMDDYYP", "MMDDYYS", + "YYMMDDB", "YYMMDDD", "YYMMDDN", "YYMMDDP", "YYMMDDS", "MONNAME", "MONTH", + "MONYY", "QTR", "QTRR", "YEAR", "DAY", "DOWNAME", "JULIAN", "E8601DA"]) +const SAS_TIME_FORMATS = Set(["TIME", "HHMM", "HOUR", "MMSS", "E8601TM"]) + +# SPSS format base names. +const SPSS_DATETIME_FORMATS = Set(["DATETIME", "YMDHMS"]) +const SPSS_DATE_FORMATS = Set(["DATE", "ADATE", "EDATE", "JDATE", "SDATE"]) +const SPSS_TIME_FORMATS = Set(["TIME", "DTIME"]) + +# "DATETIME22.3" -> "DATETIME", "EDATE10" -> "EDATE", "E8601DA" -> "E8601DA". +_format_base(fmt::AbstractString) = + replace(replace(uppercase(fmt), r"\.\d*$" => ""), r"\d+$" => "") + +# Classify a format string into a converter function (applied to non-missing +# numeric cells) and a target element type, or `nothing` when the column is +# not a date/time. +function datetime_rule(fmt::AbstractString, producer::Symbol) + isempty(fmt) && return nothing + if producer === :dta + if startswith(fmt, "%tc") || startswith(fmt, "%tC") || + startswith(fmt, "%-tc") || startswith(fmt, "%-tC") + return (v -> STATA_EPOCH_DATETIME + Millisecond(round(Int64, Float64(v)))), DateTime + elseif startswith(fmt, "%td") || startswith(fmt, "%-td") || startswith(fmt, "%d") || + startswith(fmt, "%-d") + return (v -> STATA_EPOCH_DATE + Day(round(Int64, Float64(v)))), Date + elseif startswith(fmt, "%tw") + # Stata weeks reset every year: 52 fixed-length weeks per year. + return (v -> (i = round(Int64, Float64(v)); + Date(1960 + fld(i, 52), 1, 1) + Week(mod(i, 52)))), Date + elseif startswith(fmt, "%tm") + return (v -> STATA_EPOCH_DATE + Month(round(Int64, Float64(v)))), Date + elseif startswith(fmt, "%tq") + return (v -> STATA_EPOCH_DATE + Month(3 * round(Int64, Float64(v)))), Date + elseif startswith(fmt, "%th") + return (v -> STATA_EPOCH_DATE + Month(6 * round(Int64, Float64(v)))), Date + elseif startswith(fmt, "%ty") + return (v -> Date(round(Int64, Float64(v)), 1, 1)), Date + end + return nothing + elseif producer === :sas7bdat || producer === :xport + base = _format_base(fmt) + if base in SAS_DATETIME_FORMATS + return (v -> SAS_EPOCH_DATETIME + Millisecond(round(Int64, Float64(v) * 1000))), DateTime + elseif base in SAS_DATE_FORMATS + return (v -> SAS_EPOCH_DATE + Day(round(Int64, Float64(v)))), Date + elseif base in SAS_TIME_FORMATS + return (v -> HMS(Float64(v))), HMS + end + return nothing + elseif producer === :sav || producer === :por + base = _format_base(fmt) + if base in SPSS_DATETIME_FORMATS + return (v -> SPSS_EPOCH_DATETIME + Millisecond(round(Int64, Float64(v) * 1000))), DateTime + elseif base in SPSS_DATE_FORMATS + return (v -> SPSS_EPOCH_DATE + Day(fld(round(Int64, Float64(v)), 86400))), Date + elseif base in SPSS_TIME_FORMATS + return (v -> HMS(Float64(v))), HMS + end + return nothing + end + return nothing +end + +# Convert a numeric column to Date/DateTime/HMS behind a function barrier. +function convert_datetime_column(col::DataValueVector{T}, f::F, ::Type{S}) where {T<:Number,F,S} + n = length(col) + out = DataValueVector{S}(Vector{S}(undef, n), fill(true, n)) + @inbounds for i in 1:n + v = col[i] + DataValues.isna(v) || (out[i] = f(get(v))) + end + return out +end diff --git a/test/runtests.jl b/test/runtests.jl index 58db9bf..49d58ad 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,6 +5,7 @@ include("test_read.jl") include("test_kwargs.jl") include("test_missing.jl") include("test_labels.jl") +include("test_datetime.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_datetime.jl b/test/test_datetime.jl new file mode 100644 index 0000000..68228bd --- /dev/null +++ b/test/test_datetime.jl @@ -0,0 +1,95 @@ +@testitem "date/time conversion" begin + using DataValues + using Dates + + dir = joinpath(@__DIR__, "data") + + expected_dates = DataValueArray{Date}( + [Date(2018, 5, 6), Date(1880, 5, 6), Date(1960, 1, 1), Date(1583, 1, 1), NA]) + expected_dtimes = DataValueArray{DateTime}( + [DateTime(2018, 5, 6, 10, 10, 10), DateTime(1880, 5, 6, 10, 10, 10), + DateTime(1960, 1, 1), DateTime(1583, 1, 1), NA]) + expected_times = [36610.0, 83410.0, 0.0, 58210.0] # seconds of day, row 5 NA + + @testset "$file" for (file, datecol, dtimecol, timecol) in + (("sample.dta", :mydate, :dtime, nothing), # mytime in .dta is %tc, tested below + ("sample.sav", :mydate, :dtime, :mytime), + ("sample.por", :MYDATE, :DTIME, :MYTIME), + ("sample.sas7bdat", :mydate, :dtime, :mytime), + ("sample.xpt", :MYDATE, :DTIME, :MYTIME)) + + tbl = readstat(joinpath(dir, file)) + @test tbl[datecol] == expected_dates + @test tbl[dtimecol] == expected_dtimes + if timecol !== nothing + col = tbl[timecol] + @test eltype(eltype(col)) === HMS + @test [unwrap(get(x)) for x in col[1:4]] == expected_times + @test isna(col[5]) + end + + # Raw numbers when conversion is off. + raw = readstat(joinpath(dir, file); convert_datetime=false) + @test eltype(eltype(raw[datecol])) <: Number + end + + @testset "Stata %tc time-of-day and DATETIME13" begin + tbl = read_dta(joinpath(dir, "sample.dta")) + # mytime uses %tcHH:MM:SS — a datetime anchored at the epoch day. + @test get(tbl[:mytime][1]) == DateTime(1960, 1, 1, 10, 10, 10) + + x = readstat(joinpath(dir, "datetime13.xpt")) + @test get(x[:DTTEST][1]) == DateTime(1960, 1, 2, 10, 17, 36) + + a = read_dta(joinpath(dir, "alltypes.dta")) + @test get(a[:vdate][1]) == Date(1960, 1, 2) # %td, value 1 + @test get(a[:vtime][1]) == DateTime(1960, 1, 1, 0, 0, 0, 1) # %tc, value 1 + @test isna(a[:vdate][2]) && isna(a[:vdate][3]) + end + + @testset "HMS" begin + t = HMS(36610.0) + @test unwrap(t) == 36610.0 + @test Dates.hour(t) == 10 + @test Dates.minute(t) == 10 + @test Dates.second(t) == 10 + @test sprint(show, t) == "10:10:10" + @test Time(t) == Time(10, 10, 10) + + # Durations beyond a day and negative durations are representable. + long = HMS(30 * 3600.0) + @test Dates.hour(long) == 30 + @test sprint(show, long) == "30:00:00" + @test_throws ArgumentError Time(long) + @test sprint(show, HMS(-3661.0)) == "-1:01:01" + @test Dates.hour(HMS(-3661.0)) == -1 + + @test sprint(show, HMS(0.5)) == "0:00:00.500" + @test HMS(1.0) < HMS(2.0) + @test HMS(1.0) == HMS(1.0) + @test hash(HMS(1.0)) == hash(HMS(1.0)) + end + + @testset "format classification" begin + @test ReadStat.datetime_rule("%tm", :dta) !== nothing + f, T = ReadStat.datetime_rule("%tm", :dta) + @test T === Date && f(1) == Date(1960, 2, 1) + f, T = ReadStat.datetime_rule("%tq", :dta) + @test f(1) == Date(1960, 4, 1) + f, T = ReadStat.datetime_rule("%th", :dta) + @test f(1) == Date(1960, 7, 1) + f, T = ReadStat.datetime_rule("%ty", :dta) + @test f(1987) == Date(1987, 1, 1) + f, T = ReadStat.datetime_rule("%tw", :dta) + @test f(0) == Date(1960, 1, 1) + @test f(52) == Date(1961, 1, 1) + + @test ReadStat.datetime_rule("F8.2", :sav) === nothing + @test ReadStat.datetime_rule("BEST12", :sas7bdat) === nothing + @test ReadStat.datetime_rule("", :dta) === nothing + @test ReadStat.datetime_rule("DATETIME22.3", :sas7bdat) !== nothing + @test ReadStat.datetime_rule("MMDDYYS10", :xport) !== nothing + @test ReadStat.datetime_rule("SDATE10", :por) !== nothing + @test ReadStat.datetime_rule("DTIME10", :sav) !== nothing + end +end From 0fa84b35753f0dd46d8f0a04af6c5ae981508f6f Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:24:13 -0700 Subject: [PATCH 07/14] Read from Julia IO streams via the C custom-IO handlers Every reader, the readstat dispatcher (with an explicit format), and read_meta now accept an IO containing the complete file in addition to a path. Julia implementations of the C library open/close/seek/read handler quartet drive the parse over the stream; non-seekable streams are buffered into an IOBuffer first, since the parsers seek backwards routinely. Exceptions raised by the stream are captured on the IO context and rethrown after the parse instead of unwinding through C. Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 2 + src/reader/io.jl | 79 +++++++++++++++++++++++++++++ src/reader/read.jl | 120 +++++++++++++++++++++++++++++++-------------- test/runtests.jl | 1 + test/test_io.jl | 47 ++++++++++++++++++ 5 files changed, 212 insertions(+), 37 deletions(-) create mode 100644 src/reader/io.jl create mode 100644 test/test_io.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 994d160..15c24f3 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -48,11 +48,13 @@ include("reader/columns.jl") include("reader/context.jl") include("reader/handlers.jl") include("reader/table.jl") +include("reader/io.jl") include("reader/read.jl") include("deprecated.jl") function __init__() _init_cfunctions() + _init_io_cfunctions() return nothing end diff --git a/src/reader/io.jl b/src/reader/io.jl new file mode 100644 index 0000000..d24a2a0 --- /dev/null +++ b/src/reader/io.jl @@ -0,0 +1,79 @@ +# Parsing from Julia IO streams via the C library's custom IO handlers. +# +# The C parsers drive all reading through an open/close/seek/read handler +# quartet; installing Julia implementations lets any seekable IO (an open +# file, an IOBuffer over bytes from anywhere) be parsed without touching the +# file system. Non-seekable streams are slurped into an IOBuffer first — the +# parsers seek backwards routinely, so streaming through them directly is +# not possible. +# +# The stream must contain the complete file starting at position 0: seek +# offsets from the C side are absolute. + +mutable struct IOSource + io::IO + size::Int64 + err::Union{Nothing,Tuple{Any,Any}} +end + +function IOSource(io::IO) + seekable = try + seek(io, position(io)) + true + catch + false + end + seekable || (io = IOBuffer(read(io))) + pos = position(io) + seekend(io) + size = Int64(position(io)) + seek(io, pos) + return IOSource(io, size, nothing) +end + +# The C side treats an open-handler return of -1 as failure; the stream is +# already open, so both open and close are no-ops. +handle_io_open(path::Cstring, io_ctx::Ptr{Cvoid})::Cint = Cint(0) +handle_io_close(io_ctx::Ptr{Cvoid})::Cint = Cint(0) + +function handle_io_seek(offset::Int64, whence::ReadStatIOFlags, io_ctx::Ptr{Cvoid})::Int64 + src = unsafe_pointer_to_objref(io_ctx)::IOSource + try + pos = whence == READSTAT_SEEK_SET ? offset : + whence == READSTAT_SEEK_CUR ? Int64(position(src.io)) + offset : + src.size + offset + (pos < 0 || pos > src.size) && return Int64(-1) + seek(src.io, pos) + return Int64(position(src.io)) + catch e + src.err = (e, catch_backtrace()) + return Int64(-1) + end +end + +function handle_io_read(buf::Ptr{Cvoid}, nbyte::Csize_t, io_ctx::Ptr{Cvoid})::Cssize_t + src = unsafe_pointer_to_objref(io_ctx)::IOSource + try + io = src.io + n = min(Int(nbyte), max(src.size - Int64(position(io)), 0)) + n <= 0 && return Cssize_t(0) + unsafe_read(io, Ptr{UInt8}(buf), UInt(n)) + return Cssize_t(n) + catch e + src.err = (e, catch_backtrace()) + return Cssize_t(-1) + end +end + +const CF_IO_OPEN = Ref(C_NULL) +const CF_IO_CLOSE = Ref(C_NULL) +const CF_IO_SEEK = Ref(C_NULL) +const CF_IO_READ = Ref(C_NULL) + +function _init_io_cfunctions() + CF_IO_OPEN[] = @cfunction(handle_io_open, Cint, (Cstring, Ptr{Cvoid})) + CF_IO_CLOSE[] = @cfunction(handle_io_close, Cint, (Ptr{Cvoid},)) + CF_IO_SEEK[] = @cfunction(handle_io_seek, Int64, (Int64, ReadStatIOFlags, Ptr{Cvoid})) + CF_IO_READ[] = @cfunction(handle_io_read, Cssize_t, (Ptr{Cvoid}, Csize_t, Ptr{Cvoid})) + return nothing +end diff --git a/src/reader/read.jl b/src/reader/read.jl index 1907aff..799b2ee 100644 --- a/src/reader/read.jl +++ b/src/reader/read.jl @@ -19,6 +19,12 @@ function _sniff_format(path::AbstractString, format::Symbol) return fmt end +function _sniff_format(io::IO, format::Symbol) + format === :auto && + throw(ArgumentError("the file format cannot be inferred from an IO stream; pass `format=...`")) + return format +end + function _parse_format(parser::ParserPtr, path::AbstractString, format::Symbol, ctx) if format === :dta CAPI.readstat_parse_dta(parser, path, ctx) @@ -46,45 +52,78 @@ _colselector(v::AbstractVector{<:Integer}) = let s = Set{Int}(v); (name, i) -> i _colselector(r::Regex) = (name, i) -> occursin(r, String(name)) _colselector(f::Function) = (name, i) -> f(name)::Bool -function parse_file!(pc::ParseContext, path::AbstractString, format::Symbol) - isfile(path) || throw(ArgumentError("file not found: $path")) - parser = readstat_parser_init() - local retval - try - readstat_set_metadata_handler(parser, CF_METADATA[]) - readstat_set_variable_handler(parser, CF_VARIABLE[]) - readstat_set_value_label_handler(parser, CF_VALUE_LABEL[]) - readstat_set_note_handler(parser, CF_NOTE[]) - readstat_set_fweight_handler(parser, CF_FWEIGHT[]) - readstat_set_error_handler(parser, CF_ERROR[]) - pc.collect_values && readstat_set_value_handler(parser, CF_VALUE[]) - pc.progress === nothing || readstat_set_progress_handler(parser, CF_PROGRESS[]) - pc.row_offset > 0 && readstat_set_row_offset(parser, pc.row_offset) - pc.row_limit >= 0 && readstat_set_row_limit(parser, pc.row_limit) - pc.file_encoding === nothing || - readstat_set_file_character_encoding(parser, pc.file_encoding) - pc.handler_encoding === nothing || - readstat_set_handler_character_encoding(parser, pc.handler_encoding) - retval = _parse_format(parser, path, format, pc) - finally - readstat_parser_free(parser) - end +function _set_handlers!(parser::ParserPtr, pc::ParseContext) + readstat_set_metadata_handler(parser, CF_METADATA[]) + readstat_set_variable_handler(parser, CF_VARIABLE[]) + readstat_set_value_label_handler(parser, CF_VALUE_LABEL[]) + readstat_set_note_handler(parser, CF_NOTE[]) + readstat_set_fweight_handler(parser, CF_FWEIGHT[]) + readstat_set_error_handler(parser, CF_ERROR[]) + pc.collect_values && readstat_set_value_handler(parser, CF_VALUE[]) + pc.progress === nothing || readstat_set_progress_handler(parser, CF_PROGRESS[]) + pc.row_offset > 0 && readstat_set_row_offset(parser, pc.row_offset) + pc.row_limit >= 0 && readstat_set_row_limit(parser, pc.row_limit) + pc.file_encoding === nothing || + readstat_set_file_character_encoding(parser, pc.file_encoding) + pc.handler_encoding === nothing || + readstat_set_handler_character_encoding(parser, pc.handler_encoding) + return parser +end + +function _finish_parse(pc::ParseContext, retval::ReadStatError, what::AbstractString) if pc.err !== nothing e, _ = pc.err throw(e) end for w in pc.warnings - @warn "readstat: $w" _module = ReadStat _file = String(path) + @warn "readstat: $w" _module = ReadStat _file = what end if retval == READSTAT_ERROR_USER_ABORT && pc.aborted # The progress callback stopped the parse; the caller gets the rows # delivered so far. elseif retval != READSTAT_OK - error("Error parsing $path: $(readstat_error_message(retval))") + error("Error parsing $what: $(readstat_error_message(retval))") end return pc end +function parse_file!(pc::ParseContext, path::AbstractString, format::Symbol) + isfile(path) || throw(ArgumentError("file not found: $path")) + parser = readstat_parser_init() + local retval + try + _set_handlers!(parser, pc) + retval = _parse_format(parser, path, format, pc) + finally + readstat_parser_free(parser) + end + return _finish_parse(pc, retval, path) +end + +function parse_file!(pc::ParseContext, io::IO, format::Symbol) + src = IOSource(io) + parser = readstat_parser_init() + local retval + try + _set_handlers!(parser, pc) + readstat_set_open_handler(parser, CF_IO_OPEN[]) + readstat_set_close_handler(parser, CF_IO_CLOSE[]) + readstat_set_seek_handler(parser, CF_IO_SEEK[]) + readstat_set_read_handler(parser, CF_IO_READ[]) + GC.@preserve src begin + readstat_set_io_ctx(parser, pointer_from_objref(src)) + retval = _parse_format(parser, "", format, pc) + end + finally + readstat_parser_free(parser) + end + if src.err !== nothing + e, _ = src.err + throw(e) + end + return _finish_parse(pc, retval, "IO stream") +end + function build_table(pc::ParseContext) pc.aborted && trimcolumns!(pc.cols, pc.rows_complete) n = length(pc.names) @@ -113,7 +152,7 @@ function build_table(pc::ParseContext) return ReadStatTable(columns, pc.names, pc.meta, pc.varmeta, tags) end -function read_data_file(path::AbstractString, format::Symbol; +function read_data_file(source::Union{AbstractString,IO}, format::Symbol; usecols=nothing, row_limit::Union{Nothing,Integer}=nothing, row_offset::Integer=0, @@ -150,12 +189,16 @@ function read_data_file(path::AbstractString, format::Symbol; pc.file_format = format catalog === nothing || format === :sas7bdat || throw(ArgumentError("`catalog` is only supported when reading sas7bdat files")) - parse_file!(pc, path, format) + parse_file!(pc, source, format) catalog === nothing || merge!(pc.meta.value_labels, read_sas7bcat(catalog)) return build_table(pc) end const _READ_KWARGS_DOC = """ +Instead of a path, every reader also accepts an `IO` containing the complete +file (a non-seekable stream is buffered in memory first). The `progress` +callback only fires for path input. + All readers accept the same keyword arguments: - `usecols`: read only these columns — a `Symbol`, column index, vector of @@ -191,7 +234,7 @@ data and metadata. $_READ_KWARGS_DOC """ -read_dta(path::AbstractString; kwargs...) = read_data_file(path, :dta; kwargs...) +read_dta(source::Union{AbstractString,IO}; kwargs...) = read_data_file(source, :dta; kwargs...) """ read_sav(path; kwargs...) -> ReadStatTable @@ -200,7 +243,7 @@ Read an SPSS `.sav` (or `.zsav`) file. $_READ_KWARGS_DOC """ -read_sav(path::AbstractString; kwargs...) = read_data_file(path, :sav; kwargs...) +read_sav(source::Union{AbstractString,IO}; kwargs...) = read_data_file(source, :sav; kwargs...) """ read_por(path; kwargs...) -> ReadStatTable @@ -210,7 +253,7 @@ so `filemetadata(tbl).row_count` is `-1`. $_READ_KWARGS_DOC """ -read_por(path::AbstractString; kwargs...) = read_data_file(path, :por; kwargs...) +read_por(source::Union{AbstractString,IO}; kwargs...) = read_data_file(source, :por; kwargs...) """ read_sas7bdat(path; kwargs...) -> ReadStatTable @@ -219,7 +262,8 @@ Read a SAS `.sas7bdat` data file. $_READ_KWARGS_DOC """ -read_sas7bdat(path::AbstractString; kwargs...) = read_data_file(path, :sas7bdat; kwargs...) +read_sas7bdat(source::Union{AbstractString,IO}; kwargs...) = + read_data_file(source, :sas7bdat; kwargs...) """ read_xport(path; kwargs...) -> ReadStatTable @@ -229,7 +273,8 @@ count, so `filemetadata(tbl).row_count` is `-1`. $_READ_KWARGS_DOC """ -read_xport(path::AbstractString; kwargs...) = read_data_file(path, :xport; kwargs...) +read_xport(source::Union{AbstractString,IO}; kwargs...) = + read_data_file(source, :xport; kwargs...) """ readstat(path; format=:auto, kwargs...) -> ReadStatTable @@ -237,12 +282,13 @@ read_xport(path::AbstractString; kwargs...) = read_data_file(path, :xport; kwarg Read a stat-package data file, inferring the format from the file extension (`.dta`, `.sav`/`.zsav`, `.por`, `.sas7bdat`, `.xpt`/`.xport`) unless `format` is given explicitly (`:dta`, `:sav`, `:por`, `:sas7bdat`, -`:xport`). +`:xport`). For `IO` input the format cannot be inferred, so `format` is +required. $_READ_KWARGS_DOC """ -readstat(path::AbstractString; format::Symbol=:auto, kwargs...) = - read_data_file(path, _sniff_format(path, format); kwargs...) +readstat(source::Union{AbstractString,IO}; format::Symbol=:auto, kwargs...) = + read_data_file(source, _sniff_format(source, format); kwargs...) """ read_sas7bcat(path) -> Dict{Symbol, ValueLabelDict} @@ -268,7 +314,7 @@ zero rows, but its [`filemetadata`](@ref), [`varmetadata`](@ref), and in the file (`filemetadata(tbl).row_count`). This is much cheaper than reading the data. """ -function read_meta(path::AbstractString; format::Symbol=:auto, +function read_meta(source::Union{AbstractString,IO}; format::Symbol=:auto, file_encoding::Union{Nothing,AbstractString}=nothing, handler_encoding::Union{Nothing,AbstractString}=nothing) pc = ParseContext() @@ -277,6 +323,6 @@ function read_meta(path::AbstractString; format::Symbol=:auto, # row limit in effect, and read_meta must report the true count. pc.file_encoding = file_encoding === nothing ? nothing : String(file_encoding) pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) - parse_file!(pc, path, _sniff_format(path, format)) + parse_file!(pc, source, _sniff_format(source, format)) return build_table(pc) end diff --git a/test/runtests.jl b/test/runtests.jl index 49d58ad..ce752a7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,6 +6,7 @@ include("test_kwargs.jl") include("test_missing.jl") include("test_labels.jl") include("test_datetime.jl") +include("test_io.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_io.jl b/test/test_io.jl new file mode 100644 index 0000000..9a9603d --- /dev/null +++ b/test/test_io.jl @@ -0,0 +1,47 @@ +@testitem "reading from IO" begin + using DataValues + + dir = @__DIR__ + + # A minimal read-only, non-seekable stream to exercise the buffering path. + struct NoSeekIO <: IO + inner::IOBuffer + end + Base.read(io::NoSeekIO, ::Type{UInt8}) = read(io.inner, UInt8) + Base.eof(io::NoSeekIO) = eof(io.inner) + + @testset "$file" for (reader, file) in + ((read_dta, "types.dta"), (read_sav, "types.sav"), + (read_sas7bdat, "types.sas7bdat"), (read_xport, "types.xpt")) + path = joinpath(dir, file) + expected = reader(path) + + for source in (open(path), IOBuffer(read(path)), NoSeekIO(IOBuffer(read(path)))) + tbl = reader(source) + @test names(tbl) == names(expected) + @test size(tbl) == size(expected) + for i in 1:size(tbl, 2) + @test tbl[i] == expected[i] + end + source isa IOStream && close(source) + end + end + + @testset "kwargs compose with IO input" begin + path = joinpath(dir, "types.dta") + tbl = read_dta(IOBuffer(read(path)); usecols=[:vlong], row_limit=2) + @test names(tbl) == [:vlong] + @test tbl[:vlong] == DataValueArray{Int32}([2, 7]) + end + + @testset "dispatcher and read_meta over IO" begin + path = joinpath(dir, "types.sav") + io = IOBuffer(read(path)) + @test size(readstat(io; format=:sav)) == (3, 6) + @test_throws ArgumentError readstat(IOBuffer(read(path))) + + m = read_meta(IOBuffer(read(path)); format=:sav) + @test size(m) == (0, 6) + @test filemetadata(m).row_count == 3 + end +end From 960c658cc24df0e6804a8a8f07cac18023f08216 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:30:45 -0700 Subject: [PATCH 08/14] Parse dta/sav/sas7bdat files with multiple parsers into shared buffers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new ntasks keyword (auto-enabled for large files) splits the selected row range into contiguous chunks, one C parser per Threads.@spawn task. All tasks write directly into shared, preallocated column buffers: chunk obs_index values are relative to each parser''s row_offset (verified by the row-selection tests), so every task fills a disjoint region of plain Vectors, which is thread-safe without locks — no per-task buffers, no merge copy, and no chained columns. Only the rare tagged-missing tag vectors are per-task (lazy shared allocation would race) and get combined after the join, and chunk variable handlers skip column setup entirely via a preassigned-columns mode. A metadata pre-pass without a row limit supplies the true row count, the schema for preallocation, and the value labels and notes: experimentation showed the sav parser does not reliably deliver value labels while a row limit is in effect, so chunk parses cannot be trusted for metadata. The same finding fixes row_limit=0 reads, which previously used a 1-row C limit and could lose sav value labels. Formats without a recorded row count (xport, por) and IO input fall back to a serial parse. Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 1 + src/reader/columns.jl | 15 +++++ src/reader/context.jl | 10 +++- src/reader/handlers.jl | 5 +- src/reader/read.jl | 24 ++++++-- src/reader/threaded.jl | 124 +++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/test_threads.jl | 63 +++++++++++++++++++++ 8 files changed, 236 insertions(+), 7 deletions(-) create mode 100644 src/reader/threaded.jl create mode 100644 test/test_threads.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 15c24f3..aae86ea 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -49,6 +49,7 @@ include("reader/context.jl") include("reader/handlers.jl") include("reader/table.jl") include("reader/io.jl") +include("reader/threaded.jl") include("reader/read.jl") include("deprecated.jl") diff --git a/src/reader/columns.jl b/src/reader/columns.jl index 685228d..395c641 100644 --- a/src/reader/columns.jl +++ b/src/reader/columns.jl @@ -157,3 +157,18 @@ end finalize_column(cols::TypedColumns, i::Int) = finalize_column(getbuf(cols, i)) column_tags(cols::TypedColumns, i::Int) = getbuf(cols, i).tags + +# A per-task view for multi-task reads: shares the value and NA-mask vectors +# (each task writes only its disjoint row region) but gives every task its +# own lazily allocated tag vector, since concurrent lazy allocation on a +# shared ColumnBuf would race. Tags are merged after the tasks join. +function share_buffers(cols::TypedColumns) + TypedColumns( + [ColumnBuf{String}(b.values, b.isna, nothing) for b in cols.strings], + [ColumnBuf{Int8}(b.values, b.isna, nothing) for b in cols.int8s], + [ColumnBuf{Int16}(b.values, b.isna, nothing) for b in cols.int16s], + [ColumnBuf{Int32}(b.values, b.isna, nothing) for b in cols.int32s], + [ColumnBuf{Float32}(b.values, b.isna, nothing) for b in cols.floats], + [ColumnBuf{Float64}(b.values, b.isna, nothing) for b in cols.doubles], + cols.slots) +end diff --git a/src/reader/context.jl b/src/reader/context.jl index 7bb54dc..e558cee 100644 --- a/src/reader/context.jl +++ b/src/reader/context.jl @@ -28,6 +28,14 @@ mutable struct ParseContext apply_value_labels::Bool # true: decode columns whose display format is a date/time format. convert_datetime::Bool + # Row index (0-based) of this parse's first row within the final table; + # nonzero only for the chunk contexts of a multi-task read, whose + # obs_index values are relative to their own row_offset. + row_base::Int + # true for chunk contexts whose `cols` were prepared up front (shared + # buffers): the variable handler must not allocate columns, only apply + # the usecols decision. + preassigned_cols::Bool # the format being parsed (:dta, :sav, ...); selects the producer's # date/time format tables. file_format::Symbol # false for metadata-only parses @@ -40,7 +48,7 @@ end ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, false, true, - :none, false, 0) + 0, false, :none, false, 0) # Rows to preallocate per column: what the file reports, minus the offset, # capped by the limit; 0 when unknown (buffers then grow row by row). diff --git a/src/reader/handlers.jl b/src/reader/handlers.jl index a1284bb..87352e6 100644 --- a/src/reader/handlers.jl +++ b/src/reader/handlers.jl @@ -66,6 +66,9 @@ function handle_variable(index::Cint, variable::VariablePtr, val_labels::Cstring if sel !== nothing && !(sel(name, Int(index) + 1)::Bool) return READSTAT_HANDLER_SKIP_VARIABLE end + # Chunk contexts of a multi-task read come with shared, fully + # prepared columns; only the skip decision above matters here. + pc.preassigned_cols && return READSTAT_HANDLER_OK t = readstat_variable_get_type(variable) vm = ReadStatVarMeta( name, @@ -93,7 +96,7 @@ function handle_value(obs_index::Cint, variable::VariablePtr, value::ReadStatVal pc = ctx::ParseContext try idx = readstat_variable_get_index_after_skipping(variable) + 1 - row = Int(obs_index) + 1 + row = pc.row_base + Int(obs_index) + 1 cols = pc.cols code, slot = @inbounds cols.slots[idx] diff --git a/src/reader/read.jl b/src/reader/read.jl index 799b2ee..972f003 100644 --- a/src/reader/read.jl +++ b/src/reader/read.jl @@ -162,6 +162,7 @@ function read_data_file(source::Union{AbstractString,IO}, format::Symbol; convert_datetime::Bool=true, apply_value_labels::Bool=false, catalog::Union{Nothing,AbstractString}=nothing, + ntasks::Union{Nothing,Integer}=nothing, progress=nothing) user_missing in (:na, :keep) || throw(ArgumentError("user_missing must be :na or :keep")) @@ -174,10 +175,10 @@ function read_data_file(source::Union{AbstractString,IO}, format::Symbol; pc.row_offset = Int(row_offset) if row_limit == 0 # The C library treats a row limit of 0 as "no limit", so a zero-row - # read is done by not collecting values at all (with a 1-row limit as - # a hint to stop decoding the data section early). + # read is done by not collecting values at all. No C row limit is set + # either: the sav parser does not reliably deliver value labels when + # one is in effect. pc.collect_values = false - pc.row_limit = 1 else pc.row_limit = row_limit === nothing ? -1 : Int(row_limit) end @@ -189,7 +190,14 @@ function read_data_file(source::Union{AbstractString,IO}, format::Symbol; pc.file_format = format catalog === nothing || format === :sas7bdat || throw(ArgumentError("`catalog` is only supported when reading sas7bdat files")) - parse_file!(pc, source, format) + + parsed = false + if source isa AbstractString && pc.collect_values && progress === nothing && + format in _THREADED_FORMATS + T = ntasks === nothing ? _auto_ntasks(source) : max(Int(ntasks), 1) + T > 1 && (parsed = threaded_parse!(pc, source, format, T)) + end + parsed || parse_file!(pc, source, format) catalog === nothing || merge!(pc.meta.value_labels, read_sas7bcat(catalog)) return build_table(pc) end @@ -222,8 +230,14 @@ All readers accept the same keyword arguments: regardless. - `catalog` (sas7bdat only): path to the `.sas7bcat` catalog holding the file's value labels; they are merged into the table's value labels. +- `ntasks`: number of parallel parsers for path input to formats that record + a row count (`.dta`, `.sav`, `.sas7bdat`): the row range is split into + contiguous chunks that parse concurrently into shared buffers. By default + large files use up to 8 threads and small files parse serially; pass an + integer to force a task count, or `1` to disable. - `progress`: a function called with the parse fraction (0.0-1.0); return - `false` to stop the parse and get the rows read so far. + `false` to stop the parse and get the rows read so far (disables + `ntasks`). """ """ diff --git a/src/reader/threaded.jl b/src/reader/threaded.jl new file mode 100644 index 0000000..bd85df6 --- /dev/null +++ b/src/reader/threaded.jl @@ -0,0 +1,124 @@ +# Multi-task reading: split the row range into contiguous chunks, one C +# parser per task, all writing directly into shared preallocated column +# buffers (disjoint row regions of plain Vectors, which is thread-safe). +# There are no per-task buffers to merge and no chained columns afterwards; +# the only post-join work is combining the rare tagged-missing tag vectors. +# +# A metadata pre-pass (no value handler, no row limit) supplies the true row +# count, the schema for preallocation, and the value labels and notes — the +# C parsers do not reliably deliver value labels under a row limit, so the +# chunk parses cannot be trusted for metadata. + +const _THREADED_FORMATS = (:dta, :sav, :sas7bdat) + +function _auto_ntasks(path::AbstractString) + Threads.nthreads() == 1 && return 1 + sz = try + filesize(path) + catch + 0 + end + # Small files are not worth the metadata pre-pass and task setup. + return sz < 4_000_000 ? 1 : min(Threads.nthreads(), 8) +end + +# Parse `path` into `pc` using `ntasks` chunk parsers. Returns false when a +# multi-task read is not possible (unknown row count, too few rows), in +# which case the caller falls back to a serial parse. +function threaded_parse!(pc::ParseContext, path::AbstractString, format::Symbol, ntasks::Int) + pre = ParseContext() + pre.usecols = pc.usecols + pre.file_encoding = pc.file_encoding + pre.handler_encoding = pc.handler_encoding + pre.collect_values = false + parse_file!(pre, path, format) + + total = pre.meta.row_count + total < 0 && return false + nrows = max(total - pc.row_offset, 0) + pc.row_limit >= 0 && (nrows = min(nrows, pc.row_limit)) + if nrows == 0 + # Nothing to read; keep the pre-pass metadata over empty columns. + empty = TypedColumns() + for vm in pre.varmeta + addcolumn!(empty, jltype(vm.type), 0) + end + _adopt!(pc, pre, empty) + return true + end + ntasks = min(ntasks, nrows) + if ntasks <= 1 + # Not worth splitting, but the pre-pass already paid for the + # metadata; run the single data parse and keep the pre-pass results. + chunk = _chunk_context(pc, pre, 0, nrows) + parse_file!(chunk, path, format) + _adopt!(pc, pre, chunk.cols) + return true + end + + shared = TypedColumns() + for vm in pre.varmeta + addcolumn!(shared, jltype(vm.type), nrows) + end + + bounds = [round(Int, i * nrows / ntasks) for i in 0:ntasks] + ctxs = Vector{ParseContext}(undef, ntasks) + tasks = Vector{Task}(undef, ntasks) + for i in 1:ntasks + chunk = _chunk_context(pc, pre, bounds[i], bounds[i + 1] - bounds[i]) + chunk.cols = share_buffers(shared) + chunk.preassigned_cols = true + ctxs[i] = chunk + tasks[i] = Threads.@spawn parse_file!($chunk, $path, $format) + end + firsterr = nothing + for t in tasks + try + wait(t) + catch e + firsterr === nothing && (firsterr = e) + end + end + if firsterr !== nothing + throw(firsterr isa TaskFailedException ? firsterr.task.exception : firsterr) + end + + # Merge the per-task tag vectors (each full-length, '\0' where untagged). + for i in 1:length(shared.slots) + buf = getbuf(shared, i) + for chunk in ctxs + t = column_tags(chunk.cols, i) + t === nothing && continue + dest = buf.tags + dest === nothing && (buf.tags = dest = fill('\0', nrows)) + @inbounds for j in eachindex(t) + t[j] != '\0' && (dest[j] = t[j]) + end + end + end + + _adopt!(pc, pre, shared) + return true +end + +function _chunk_context(pc::ParseContext, pre::ParseContext, base::Int, len::Int) + chunk = ParseContext() + chunk.usecols = pc.usecols + chunk.file_encoding = pc.file_encoding + chunk.handler_encoding = pc.handler_encoding + chunk.keep_user_missing = pc.keep_user_missing + chunk.row_offset = pc.row_offset + base + chunk.row_limit = len + chunk.row_base = base + return chunk +end + +# Take over the pre-pass metadata and the filled buffers as this context's +# parse result, so the ordinary build_table path applies. +function _adopt!(pc::ParseContext, pre::ParseContext, cols::TypedColumns) + pc.meta = pre.meta + pc.names = pre.names + pc.varmeta = pre.varmeta + pc.cols = cols + return pc +end diff --git a/test/runtests.jl b/test/runtests.jl index ce752a7..6a06288 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,6 +7,7 @@ include("test_missing.jl") include("test_labels.jl") include("test_datetime.jl") include("test_io.jl") +include("test_threads.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_threads.jl b/test/test_threads.jl new file mode 100644 index 0000000..89aa97c --- /dev/null +++ b/test/test_threads.jl @@ -0,0 +1,63 @@ +@testitem "multi-task reads" begin + using DataValues + + dir = joinpath(@__DIR__, "data") + + function tables_equal(a, b) + names(a) == names(b) || return false + size(a) == size(b) || return false + for i in 1:size(a, 2) + typeof(a[i]) == typeof(b[i]) || return false + a[i] isa LabeledArray ? (rawvalues(a[i]) == rawvalues(b[i]) || return false) : + (a[i] == b[i] || return false) + missingtags(a, i) == missingtags(b, i) || return false + end + fa, fb = filemetadata(a), filemetadata(b) + return fa.value_labels == fb.value_labels && fa.notes == fb.notes + end + + @testset "$file ≡ serial" for file in + ("sample.dta", "sample.sav", "sample.sas7bdat", "alltypes.dta") + path = joinpath(dir, file) + serial = readstat(path; ntasks=1) + for T in (2, 3, 8) + @test tables_equal(readstat(path; ntasks=T), serial) + end + end + + @testset "kwargs compose with ntasks" begin + path = joinpath(dir, "sample.dta") + serial = read_dta(path; row_offset=1, row_limit=3) + threaded = read_dta(path; row_offset=1, row_limit=3, ntasks=2) + @test tables_equal(threaded, serial) + # The pre-pass reports the file's true row count even under a limit. + @test filemetadata(threaded).row_count == 5 + + @test tables_equal(read_dta(path; usecols=[:mynum, :mylabl], ntasks=2), + read_dta(path; usecols=[:mynum, :mylabl])) + @test tables_equal( + read_sav(joinpath(dir, "sample_missing.sav"); user_missing=:keep, ntasks=2), + read_sav(joinpath(dir, "sample_missing.sav"); user_missing=:keep)) + @test tables_equal( + read_sav(joinpath(dir, "sample.sav"); apply_value_labels=true, ntasks=3), + read_sav(joinpath(dir, "sample.sav"); apply_value_labels=true)) + end + + @testset "tags survive the chunk merge" begin + tbl = read_dta(joinpath(dir, "alltypes.dta"); ntasks=3) + @test missingtags(tbl, :vbyte) == ['\0', 'a', '\0'] + @test valuelabels(tbl, :vbyte)['a'] == "Tagged missing" + end + + @testset "fallbacks" begin + # Unknown row count: xport cannot split and parses serially. + xpt = joinpath(dir, "sample.xpt") + @test tables_equal(readstat(xpt; ntasks=4), readstat(xpt)) + # More tasks than rows. + @test size(read_dta(joinpath(dir, "sample.dta"); ntasks=100), 1) == 5 + # Zero-row reads keep full metadata. + tbl = read_sav(joinpath(dir, "sample.sav"); row_limit=0, ntasks=2) + @test size(tbl) == (0, 7) + @test !isempty(filemetadata(tbl).value_labels) + end +end From 518589395e2a6197e5faa1cc71453f4bb787541b Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:36:22 -0700 Subject: [PATCH 09/14] Add ReadStatSource: a lazy, pushdown-capable handle with chunked streaming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReadStatSource is the interface a display layer or query engine plans against: constructing one parses nothing; schema() caches a metadata-only table; colnames/coltypes/nrows answer planning questions (coltypes reflects date/time conversion and value-label application per the source options, and nrows is missing when the format records no count); supports() reports which pushdowns the format can honor (:projection, :row_range, :row_count, :parallel). read(src; usecols=..., rows=...) pushes a column projection and a 1-based row range into the C parse, and works repeatedly off one stream for IO-backed sources. chunks(src; chunksize, usecols, rows) streams the file as complete ReadStatTables in a single pass: the parse runs on its own task and hands finished chunks through a bounded channel (backpressure blocks the parser), deliberately not one re-parse per chunk, which would be quadratic on compressed formats — so it also works for formats with unknown row counts. A metadata pre-pass supplies every chunk with the file's complete metadata, so value labels and date/time conversion apply from the first chunk on (dta stores label sets after the data). Closing the iterator early releases the parse task; already-buffered chunks still drain. This is the enabling interface for the planned StatFiles.jl follow-up (show via read(src; rows=...), iteration via chunks — StatFiles #4, #15) and the future query-execution engine (ReadStat.jl #29, #39). Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 5 +- src/reader/chunks.jl | 121 +++++++++++++++++++++++++++++++ src/reader/context.jl | 5 +- src/reader/handlers.jl | 9 ++- src/reader/io.jl | 14 ++-- src/reader/read.jl | 4 +- src/reader/source.jl | 157 +++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/test_source.jl | 99 ++++++++++++++++++++++++++ 9 files changed, 406 insertions(+), 9 deletions(-) create mode 100644 src/reader/chunks.jl create mode 100644 src/reader/source.jl create mode 100644 test/test_source.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index aae86ea..e2f23c1 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -8,7 +8,8 @@ export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, readstat, read_meta, filemetadata, varmetadata, valuelabels, missingtags, LabeledValue, LabeledArray, labeled, unwrap, valuelabel, rawvalues, getvaluelabels, - HMS + HMS, + ReadStatSource, ReadStatChunks, schema, colnames, coltypes, nrows, supports, chunks public CAPI @@ -51,6 +52,8 @@ include("reader/table.jl") include("reader/io.jl") include("reader/threaded.jl") include("reader/read.jl") +include("reader/source.jl") +include("reader/chunks.jl") include("deprecated.jl") function __init__() diff --git a/src/reader/chunks.jl b/src/reader/chunks.jl new file mode 100644 index 0000000..7c4e91e --- /dev/null +++ b/src/reader/chunks.jl @@ -0,0 +1,121 @@ +# Streaming chunked reads: a single pass over the file that hands finished +# ReadStatTables of `chunksize` rows through a bounded Channel. This is O(n) +# for a full scan — deliberately not one re-parse per chunk, which would be +# quadratic on compressed formats — and it works for formats that do not +# record a row count. + +# State threaded through the value handler while a chunked parse runs. The +# schema and metadata come from a pre-pass so every chunk (and the labels on +# it) is complete from the first chunk on, regardless of where the file +# stores its label sets. +mutable struct ChunkSink + channel::Channel{ReadStatTable} + chunksize::Int + names::Vector{Symbol} + varmeta::Vector{ReadStatVarMeta} + meta::ReadStatMeta + convert_datetime::Bool + apply_value_labels::Bool + file_format::Symbol +end + +_maybe_flush!(pc::ParseContext, sink::ChunkSink, row::Int) = + (row == sink.chunksize && _flush_chunk!(pc, sink, row); nothing) + +function _flush_chunk!(pc::ParseContext, sink::ChunkSink, nrows::Int) + trimcolumns!(pc.cols, nrows) + bt = ParseContext() + bt.names = sink.names + bt.varmeta = sink.varmeta + bt.meta = sink.meta + bt.cols = pc.cols + bt.convert_datetime = sink.convert_datetime + bt.apply_value_labels = sink.apply_value_labels + bt.file_format = sink.file_format + put!(sink.channel, build_table(bt)) + # Fresh buffers for the next chunk; rows keep arriving with absolute + # observation indices, so the base shifts back by what was emitted. + cols = TypedColumns() + for vm in sink.varmeta + addcolumn!(cols, jltype(vm.type), sink.chunksize) + end + pc.cols = cols + pc.row_base -= nrows + pc.rows_complete = 0 + return +end + +""" + ReadStatChunks + +Iterator over the chunks of a [`chunks`](@ref) read. Each element is a +complete [`ReadStatTable`](@ref) of up to `chunksize` rows sharing the +file's metadata. Iteration is single-pass and backed by a bounded channel: +the parse runs on its own task and blocks once two chunks are waiting. +Call `close(it)` if iteration stops early, so the parse task is released. +""" +struct ReadStatChunks + channel::Channel{ReadStatTable} +end + +Base.IteratorSize(::Type{ReadStatChunks}) = Base.SizeUnknown() +Base.eltype(::Type{ReadStatChunks}) = ReadStatTable +Base.close(it::ReadStatChunks) = close(it.channel) + +function Base.iterate(it::ReadStatChunks, state=nothing) + try + return (take!(it.channel), nothing) + catch e + e isa InvalidStateException && return nothing + rethrow() + end +end + +""" + chunks(src::ReadStatSource; chunksize=65_536, usecols=nothing, rows=nothing) + -> ReadStatChunks + +Stream the source as [`ReadStatTable`](@ref)s of up to `chunksize` rows, in +one pass over the file. `usecols` and `rows` push a projection and a 1-based +row range into the parse, exactly as in `read(src; ...)`. The chunks carry +the file's complete metadata (from a metadata pre-pass), so value labels and +date/time conversion apply from the first chunk on. + +Iterate the result with `for chunk in chunks(src) ... end`; when stopping +early, `close` the iterator to release the parse task. +""" +function chunks(src::ReadStatSource; chunksize::Integer=65_536, usecols=nothing, + rows::Union{Nothing,AbstractUnitRange{<:Integer}}=nothing) + chunksize >= 1 || throw(ArgumentError("chunksize must be at least 1")) + rows === nothing || first(rows) >= 1 || + throw(ArgumentError("rows must be a 1-based range")) + + # Pre-pass with the same projection, so schema and chunks line up. + pre = _source_meta(src, usecols) + + chunksize = Int(chunksize) + channel = Channel{ReadStatTable}(2; spawn=true) do ch + pc = ParseContext() + pc.usecols = _colselector(usecols) + pc.file_encoding = src.file_encoding + pc.handler_encoding = src.handler_encoding + pc.keep_user_missing = src.user_missing === :keep + if rows !== nothing + pc.row_offset = first(rows) - 1 + pc.row_limit = length(rows) + end + pc.preassigned_cols = true + cols = TypedColumns() + for vm in pre.varmeta + addcolumn!(cols, jltype(vm.type), chunksize) + end + pc.cols = cols + sink = ChunkSink(ch, chunksize, pre.names, pre.varmeta, pre.meta, + src.convert_datetime, src.apply_value_labels, src.format) + pc.chunk_sink = sink + parse_file!(pc, src.source, src.format) + pc.rows_complete > 0 && _flush_chunk!(pc, sink, pc.rows_complete) + return + end + return ReadStatChunks(channel) +end diff --git a/src/reader/context.jl b/src/reader/context.jl index e558cee..b43dc43 100644 --- a/src/reader/context.jl +++ b/src/reader/context.jl @@ -36,6 +36,9 @@ mutable struct ParseContext # buffers): the variable handler must not allocate columns, only apply # the usecols decision. preassigned_cols::Bool + # A ChunkSink (see reader/chunks.jl) when this parse streams finished + # chunks into a Channel instead of building one table; nothing otherwise. + chunk_sink::Any # the format being parsed (:dta, :sav, ...); selects the producer's # date/time format tables. file_format::Symbol # false for metadata-only parses @@ -48,7 +51,7 @@ end ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, false, true, - 0, false, :none, false, 0) + 0, false, nothing, :none, false, 0) # Rows to preallocate per column: what the file reports, minus the offset, # capped by the limit; 0 when unknown (buffers then grow row by row). diff --git a/src/reader/handlers.jl b/src/reader/handlers.jl index 87352e6..0790bdc 100644 --- a/src/reader/handlers.jl +++ b/src/reader/handlers.jl @@ -139,8 +139,13 @@ function handle_value(obs_index::Cint, variable::VariablePtr, value::ReadStatVal miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_double_value(value)) end # Track the last fully delivered row so a parse stopped early (by the - # progress callback) can be trimmed to complete rows. - idx == length(cols.slots) && (pc.rows_complete = row) + # progress callback) can be trimmed to complete rows; in chunked + # streaming mode a completed chunk is flushed to its channel here. + if idx == length(cols.slots) + pc.rows_complete = row + sink = pc.chunk_sink + sink === nothing || _maybe_flush!(pc, sink, row) + end return READSTAT_HANDLER_OK catch e pc.err = (e, catch_backtrace()) diff --git a/src/reader/io.jl b/src/reader/io.jl index d24a2a0..4463f41 100644 --- a/src/reader/io.jl +++ b/src/reader/io.jl @@ -16,18 +16,24 @@ mutable struct IOSource err::Union{Nothing,Tuple{Any,Any}} end -function IOSource(io::IO) +# Buffer a non-seekable stream in memory; the parsers seek backwards. +function _ensure_seekable(io::IO) seekable = try seek(io, position(io)) true catch false end - seekable || (io = IOBuffer(read(io))) - pos = position(io) + return seekable ? io : IOBuffer(read(io)) +end + +function IOSource(io::IO) + io = _ensure_seekable(io) seekend(io) size = Int64(position(io)) - seek(io, pos) + # Rewind so the same stream can be parsed repeatedly (ReadStatSource + # reads its schema first and the data later). + seekstart(io) return IOSource(io, size, nothing) end diff --git a/src/reader/read.jl b/src/reader/read.jl index 972f003..3769500 100644 --- a/src/reader/read.jl +++ b/src/reader/read.jl @@ -337,6 +337,8 @@ function read_meta(source::Union{AbstractString,IO}; format::Symbol=:auto, # row limit in effect, and read_meta must report the true count. pc.file_encoding = file_encoding === nothing ? nothing : String(file_encoding) pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) - parse_file!(pc, source, _sniff_format(source, format)) + fmt = _sniff_format(source, format) + pc.file_format = fmt + parse_file!(pc, source, fmt) return build_table(pc) end diff --git a/src/reader/source.jl b/src/reader/source.jl new file mode 100644 index 0000000..87f03b9 --- /dev/null +++ b/src/reader/source.jl @@ -0,0 +1,157 @@ +# A lazy handle over a stat-package file, designed for consumers that push +# work down into the reader — a display layer that wants ten rows, or a +# query engine that plans against the schema and then reads a projection of +# a row range, possibly in chunks. + +""" + ReadStatSource(path_or_io; format=:auto, kwargs...) + +A cheap, lazy handle over a stat-package data file: constructing one parses +nothing. [`schema`](@ref) returns (and caches) the metadata-only table; +[`colnames`](@ref), [`coltypes`](@ref), and [`nrows`](@ref) answer planning +questions from it; `read(src; usecols=..., rows=...)` reads a projection of +a row range; and [`chunks`](@ref) streams the data in table-sized pieces. +[`supports`](@ref) reports which pushdowns the underlying format can honor. + +The keyword arguments (`convert_datetime`, `apply_value_labels`, +`user_missing`, `file_encoding`, `handler_encoding`, `catalog`) fix how this +source presents its data; every read through the source applies them. For +`IO` input the stream must contain the complete file and `format` is +required; a non-seekable stream is buffered in memory once. +""" +mutable struct ReadStatSource + source::Union{String,IO} + format::Symbol + convert_datetime::Bool + apply_value_labels::Bool + user_missing::Symbol + file_encoding::Union{Nothing,String} + handler_encoding::Union{Nothing,String} + catalog::Union{Nothing,String} + schema::Union{Nothing,ReadStatTable} +end + +function ReadStatSource(source::Union{AbstractString,IO}; format::Symbol=:auto, + convert_datetime::Bool=true, apply_value_labels::Bool=false, + user_missing::Symbol=:na, + file_encoding::Union{Nothing,AbstractString}=nothing, + handler_encoding::Union{Nothing,AbstractString}=nothing, + catalog::Union{Nothing,AbstractString}=nothing) + fmt = _sniff_format(source, format) + user_missing in (:na, :keep) || + throw(ArgumentError("user_missing must be :na or :keep")) + src = source isa AbstractString ? String(source) : _ensure_seekable(source) + return ReadStatSource(src, fmt, convert_datetime, apply_value_labels, user_missing, + file_encoding === nothing ? nothing : String(file_encoding), + handler_encoding === nothing ? nothing : String(handler_encoding), + catalog === nothing ? nothing : String(catalog), + nothing) +end + +# Metadata-only parse honoring the source's presentation options (and an +# optional projection, used by `chunks` so its schema matches its chunks). +function _source_meta(src::ReadStatSource, usecols) + pc = ParseContext() + pc.collect_values = false + pc.usecols = _colselector(usecols) + pc.file_encoding = src.file_encoding + pc.handler_encoding = src.handler_encoding + pc.convert_datetime = src.convert_datetime + pc.apply_value_labels = src.apply_value_labels + pc.file_format = src.format + parse_file!(pc, src.source, src.format) + src.catalog === nothing || merge!(pc.meta.value_labels, read_sas7bcat(src.catalog)) + return pc +end + +""" + schema(src::ReadStatSource) -> ReadStatTable + +The metadata of the source as a zero-row table (cached after the first +call): column names and types, file- and variable-level metadata, value +labels, and notes — everything a consumer needs to plan a read without +touching the data. +""" +function schema(src::ReadStatSource) + if src.schema === nothing + src.schema = build_table(_source_meta(src, nothing)) + end + return src.schema::ReadStatTable +end + +""" + colnames(src::ReadStatSource) -> Vector{Symbol} + +The column names of the source (from the cached [`schema`](@ref)). +""" +colnames(src::ReadStatSource) = names(schema(src)) + +""" + coltypes(src::ReadStatSource) -> Vector{Type} + +The element types the source's columns will have when read — after date/time +conversion and value-label application per the source's options (e.g. +`DataValue{Float64}`, `DataValue{Date}`). +""" +coltypes(src::ReadStatSource) = Type[eltype(schema(src)[i]) for i in 1:size(schema(src), 2)] + +""" + nrows(src::ReadStatSource) -> Union{Int, Missing} + +The number of rows recorded in the file, or `missing` when the format does +not record one (XPORT, POR, some non-conforming SAV files). +""" +function nrows(src::ReadStatSource) + rc = filemetadata(schema(src)).row_count + return rc < 0 ? missing : rc +end + +""" + supports(src::ReadStatSource, feature::Symbol) -> Bool + +Whether the source can honor a pushdown feature: + +- `:projection`: column selection during the parse (always true). +- `:row_range`: `rows=` ranges via the C row offset/limit (always true). +- `:row_count`: a row count known without reading the data. +- `:parallel`: multi-task reading (path input to a format that records its + row count). +""" +function supports(src::ReadStatSource, feature::Symbol) + feature === :projection && return true + feature === :row_range && return true + feature === :row_count && return !ismissing(nrows(src)) + feature === :parallel && + return src.source isa String && src.format in _THREADED_FORMATS + return false +end + +""" + read(src::ReadStatSource; usecols=nothing, rows=nothing, ntasks=nothing, + progress=nothing) -> ReadStatTable + +Read the source, optionally pushing down a column projection (`usecols`, as +in the readers) and a 1-based row range (`rows`, a `UnitRange`); both are +applied inside the C parse. All other behavior follows the options fixed at +[`ReadStatSource`](@ref) construction. +""" +function Base.read(src::ReadStatSource; usecols=nothing, + rows::Union{Nothing,AbstractUnitRange{<:Integer}}=nothing, + ntasks::Union{Nothing,Integer}=nothing, progress=nothing) + row_offset = 0 + row_limit = nothing + if rows !== nothing + first(rows) >= 1 || throw(ArgumentError("rows must be a 1-based range")) + row_offset = first(rows) - 1 + row_limit = length(rows) + end + return read_data_file(src.source, src.format; usecols, row_limit, row_offset, + file_encoding=src.file_encoding, handler_encoding=src.handler_encoding, + user_missing=src.user_missing, convert_datetime=src.convert_datetime, + apply_value_labels=src.apply_value_labels, catalog=src.catalog, ntasks, progress) +end + +function Base.show(io::IO, src::ReadStatSource) + what = src.source isa String ? repr(src.source) : "" + print(io, "ReadStatSource(", what, ", ", src.format, ")") +end diff --git a/test/runtests.jl b/test/runtests.jl index 6a06288..a7c84b2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,6 +8,7 @@ include("test_labels.jl") include("test_datetime.jl") include("test_io.jl") include("test_threads.jl") +include("test_source.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_source.jl b/test/test_source.jl new file mode 100644 index 0000000..629e53d --- /dev/null +++ b/test/test_source.jl @@ -0,0 +1,99 @@ +@testitem "ReadStatSource and chunks" begin + using DataValues + using Dates + + dir = joinpath(@__DIR__, "data") + path = joinpath(dir, "sample.dta") + + @testset "planning surface" begin + src = ReadStatSource(path; apply_value_labels=true) + s = schema(src) + @test size(s, 1) == 0 + @test colnames(src) == [:mychar, :mynum, :mydate, :dtime, :mylabl, :myord, :mytime] + @test nrows(src) == 5 + ct = coltypes(src) + @test ct[1] == DataValue{String} + @test ct[3] == DataValue{Date} + @test ct[4] == DataValue{DateTime} + @test supports(src, :projection) + @test supports(src, :row_range) + @test supports(src, :row_count) + @test supports(src, :parallel) + @test !supports(src, :frobnicate) + + xsrc = ReadStatSource(joinpath(dir, "sample.xpt")) + @test ismissing(nrows(xsrc)) + @test !supports(xsrc, :row_count) + @test !supports(xsrc, :parallel) + + # The schema's column types match what a read produces. + full = read(src) + @test size(full) == (5, 7) + @test [eltype(full[i]) for i in 1:7] == ct + @test full[:mylabl] isa LabeledArray + + part = read(src; rows=2:4, usecols=[:mynum]) + @test size(part) == (3, 1) + @test part[:mynum] == read_dta(path)[:mynum][2:4] + @test_throws ArgumentError read(src; rows=0:3) + + # An IO-backed source serves schema and repeated reads off one stream. + io_src = ReadStatSource(open(path); format=:dta) + @test nrows(io_src) == 5 + @test size(read(io_src)) == (5, 7) + @test size(read(io_src; rows=1:2)) == (2, 7) + @test_throws ArgumentError ReadStatSource(IOBuffer(UInt8[])) + end + + @testset "chunks ≡ full read" begin + src = ReadStatSource(path; apply_value_labels=true) + full = read(src) + pieces = collect(chunks(src; chunksize=2)) + @test length(pieces) == 3 + @test [size(p, 1) for p in pieces] == [2, 2, 1] + for j in 1:size(full, 2) + @test isequal(reduce(vcat, [collect(p[j]) for p in pieces]), collect(full[j])) + end + # Metadata and labels are complete from the first chunk on. + @test pieces[1][:mylabl] isa LabeledArray + @test valuelabel(get(pieces[1][:mylabl][1])) in ("Male", "Female") + @test names(pieces[1]) == names(full) + + # A chunk size beyond the file yields one chunk. + @test length(collect(chunks(src; chunksize=100))) == 1 + end + + @testset "chunks with pushdown and unknown row counts" begin + src = ReadStatSource(path) + expected = read(src; rows=2:5, usecols=[:mynum, :mydate]) + pieces = collect(chunks(src; rows=2:5, usecols=[:mynum, :mydate], chunksize=3)) + @test [size(p, 1) for p in pieces] == [3, 1] + @test names(pieces[1]) == [:mynum, :mydate] + for j in 1:2 + @test isequal(reduce(vcat, [collect(p[j]) for p in pieces]), collect(expected[j])) + end + + # Formats without a recorded row count stream fine. + xsrc = ReadStatSource(joinpath(dir, "sample.xpt")) + pieces = collect(chunks(xsrc; chunksize=2)) + @test [size(p, 1) for p in pieces] == [2, 2, 1] + end + + @testset "early termination" begin + src = ReadStatSource(path) + it = chunks(src; chunksize=1) + step = iterate(it) + @test step !== nothing + @test size(step[1], 1) == 1 + close(it) + # Chunks already buffered when the iterator was closed still drain + # (bounded channel of 2); after that, iteration ends and the parse + # task has been released rather than blocking forever. + drained = collect(it) + @test length(drained) <= 2 + @test iterate(it) === nothing + end + + # zsav coverage rides along here. + @test size(read_sav(joinpath(dir, "sample.zsav"))) == (5, 7) +end From e8eff611596f5fd5bb44ddadae0f2ca075ab2904 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:46:32 -0700 Subject: [PATCH 10/14] Add write support: full C writer API, low-level Writer, write_* functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReadStat.jl can now write every format the C library writes. The CAPI submodule gains the complete writer surface (label sets including tagged values, variables with all attributes and SPSS missing declarations, notes, string refs, file-level metadata, compression, validation, and the row API). On top sits a low-level Writer following the C call order one-to-one and streaming output bytes into any Julia IO through the data-writer callback — public but unexported, for full control including Stata strL columns via string refs. The exported write_dta/write_sav/write_por/write_sas7bdat/write_xport take columns plus names (or a ReadStatTable, carrying its metadata across) and canonicalize onto the storage types each format supports: Bool and wide integers narrow sensibly, Date/DateTime/HMS/Time re-encode into the format''s native representation with a matching display format, LabeledArray columns bring their value labels along, and NA/tagged-missing cells write as the right kind of missing. write_sas7bcat writes value-label catalogs (numeric sets only — a readstat 1.1.9 limitation, documented). Two C quirks handled: the por writer silently writes no rows under lowercase variable names (names are uppercased for por, matching what a por read returns) and only handles doubles (numerics widen there). Round trips are tested for all five data formats plus the catalog, covering labels, tags, notes, user-missing rules, compression (zsav and row compression), IO destinations, and a 50k-row file read back with ntasks=4 and via chunks. This supersedes the stale write-support PRs #42 and #46. Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 15 +- src/capi/writer.jl | 250 +++++++++++++++++++++++++++++++ src/writer/write.jl | 346 +++++++++++++++++++++++++++++++++++++++++++ src/writer/writer.jl | 297 +++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/test_write.jl | 182 +++++++++++++++++++++++ 6 files changed, 1090 insertions(+), 1 deletion(-) create mode 100644 src/capi/writer.jl create mode 100644 src/writer/write.jl create mode 100644 src/writer/writer.jl create mode 100644 test/test_write.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index e2f23c1..e412d65 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -9,10 +9,19 @@ export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, filemetadata, varmetadata, valuelabels, missingtags, LabeledValue, LabeledArray, labeled, unwrap, valuelabel, rawvalues, getvaluelabels, HMS, - ReadStatSource, ReadStatChunks, schema, colnames, coltypes, nrows, supports, chunks + ReadStatSource, ReadStatChunks, schema, colnames, coltypes, nrows, supports, chunks, + write_dta, write_sav, write_por, write_sas7bdat, write_xport, write_sas7bcat public CAPI +# The low-level writer layer: public, not exported. +public Writer, WriterVariable, LabelSet, StringRef, + add_label_set!, label!, add_variable!, add_note!, add_string_ref!, + file_label!, timestamp!, fweight!, format_version!, table_name!, is_64bit!, + compression!, begin_writing!, validate_metadata, validate_variable, + begin_row!, end_row!, insert_value!, insert_missing!, insert_tagged_missing!, + insert_string_ref!, end_writing! + """ ReadStat.CAPI @@ -30,6 +39,7 @@ using ReadStat_jll: libreadstat include("capi/enums.jl") include("capi/value.jl") include("capi/parser.jl") +include("capi/writer.jl") end # module CAPI @@ -54,11 +64,14 @@ include("reader/threaded.jl") include("reader/read.jl") include("reader/source.jl") include("reader/chunks.jl") +include("writer/writer.jl") +include("writer/write.jl") include("deprecated.jl") function __init__() _init_cfunctions() _init_io_cfunctions() + _init_writer_cfunctions() return nothing end diff --git a/src/capi/writer.jl b/src/capi/writer.jl new file mode 100644 index 0000000..ef62b0b --- /dev/null +++ b/src/capi/writer.jl @@ -0,0 +1,250 @@ +# The full writer API from readstat.h (v1.1.9). + +export readstat_writer_init, readstat_writer_free, readstat_set_data_writer, + readstat_add_label_set, readstat_label_double_value, readstat_label_int32_value, + readstat_label_string_value, readstat_label_tagged_value, + readstat_add_variable, readstat_variable_set_label, readstat_variable_set_format, + readstat_variable_set_label_set, readstat_variable_set_measure, + readstat_variable_set_alignment, readstat_variable_set_display_width, + readstat_variable_add_missing_double_value, readstat_variable_add_missing_double_range, + readstat_variable_add_missing_string_value, readstat_variable_add_missing_string_range, + readstat_get_variable, readstat_add_note, readstat_add_string_ref, + readstat_get_string_ref, + readstat_writer_set_file_label, readstat_writer_set_file_timestamp, + readstat_writer_set_fweight_variable, readstat_writer_set_file_format_version, + readstat_writer_set_table_name, readstat_writer_set_file_format_is_64bit, + readstat_writer_set_compression, readstat_writer_set_error_handler, + readstat_begin_writing_dta, readstat_begin_writing_por, readstat_begin_writing_sas7bcat, + readstat_begin_writing_sas7bdat, readstat_begin_writing_sav, readstat_begin_writing_xport, + readstat_validate_metadata, readstat_validate_variable, + readstat_begin_row, readstat_insert_int8_value, readstat_insert_int16_value, + readstat_insert_int32_value, readstat_insert_float_value, readstat_insert_double_value, + readstat_insert_string_value, readstat_insert_string_ref, readstat_insert_missing_value, + readstat_insert_tagged_missing_value, readstat_end_row, readstat_end_writing + +readstat_writer_init() = @ccall libreadstat.readstat_writer_init()::WriterPtr + +function readstat_writer_free(writer::WriterPtr) + @ccall libreadstat.readstat_writer_free(writer::WriterPtr)::Cvoid +end + +function readstat_set_data_writer(writer::WriterPtr, data_writer::Ptr{Cvoid}) + @ccall libreadstat.readstat_set_data_writer(writer::WriterPtr, data_writer::Ptr{Cvoid})::ReadStatError +end + +# --------------------------------------------------------------------------- +# Label sets + +function readstat_add_label_set(writer::WriterPtr, type::ReadStatType, name::AbstractString) + @ccall libreadstat.readstat_add_label_set(writer::WriterPtr, type::ReadStatType, + name::Cstring)::LabelSetPtr +end + +function readstat_label_double_value(label_set::LabelSetPtr, value::Real, label::AbstractString) + @ccall libreadstat.readstat_label_double_value(label_set::LabelSetPtr, value::Cdouble, + label::Cstring)::Cvoid +end + +function readstat_label_int32_value(label_set::LabelSetPtr, value::Integer, label::AbstractString) + @ccall libreadstat.readstat_label_int32_value(label_set::LabelSetPtr, value::Int32, + label::Cstring)::Cvoid +end + +function readstat_label_string_value(label_set::LabelSetPtr, value::AbstractString, label::AbstractString) + @ccall libreadstat.readstat_label_string_value(label_set::LabelSetPtr, value::Cstring, + label::Cstring)::Cvoid +end + +function readstat_label_tagged_value(label_set::LabelSetPtr, tag::Char, label::AbstractString) + @ccall libreadstat.readstat_label_tagged_value(label_set::LabelSetPtr, tag::Cchar, + label::Cstring)::Cvoid +end + +# --------------------------------------------------------------------------- +# Variables + +function readstat_add_variable(writer::WriterPtr, name::AbstractString, type::ReadStatType, + storage_width::Integer) + @ccall libreadstat.readstat_add_variable(writer::WriterPtr, name::Cstring, + type::ReadStatType, storage_width::Csize_t)::VariablePtr +end + +function readstat_variable_set_label(variable::VariablePtr, label::AbstractString) + @ccall libreadstat.readstat_variable_set_label(variable::VariablePtr, label::Cstring)::Cvoid +end + +function readstat_variable_set_format(variable::VariablePtr, format::AbstractString) + @ccall libreadstat.readstat_variable_set_format(variable::VariablePtr, format::Cstring)::Cvoid +end + +function readstat_variable_set_label_set(variable::VariablePtr, label_set::LabelSetPtr) + @ccall libreadstat.readstat_variable_set_label_set(variable::VariablePtr, + label_set::LabelSetPtr)::Cvoid +end + +function readstat_variable_set_measure(variable::VariablePtr, measure::ReadStatMeasure) + @ccall libreadstat.readstat_variable_set_measure(variable::VariablePtr, + measure::ReadStatMeasure)::Cvoid +end + +function readstat_variable_set_alignment(variable::VariablePtr, alignment::ReadStatAlignment) + @ccall libreadstat.readstat_variable_set_alignment(variable::VariablePtr, + alignment::ReadStatAlignment)::Cvoid +end + +function readstat_variable_set_display_width(variable::VariablePtr, display_width::Integer) + @ccall libreadstat.readstat_variable_set_display_width(variable::VariablePtr, + display_width::Cint)::Cvoid +end + +function readstat_variable_add_missing_double_value(variable::VariablePtr, value::Real) + @ccall libreadstat.readstat_variable_add_missing_double_value(variable::VariablePtr, + value::Cdouble)::ReadStatError +end + +function readstat_variable_add_missing_double_range(variable::VariablePtr, lo::Real, hi::Real) + @ccall libreadstat.readstat_variable_add_missing_double_range(variable::VariablePtr, + lo::Cdouble, hi::Cdouble)::ReadStatError +end + +function readstat_variable_add_missing_string_value(variable::VariablePtr, value::AbstractString) + @ccall libreadstat.readstat_variable_add_missing_string_value(variable::VariablePtr, + value::Cstring)::ReadStatError +end + +function readstat_variable_add_missing_string_range(variable::VariablePtr, + lo::AbstractString, hi::AbstractString) + @ccall libreadstat.readstat_variable_add_missing_string_range(variable::VariablePtr, + lo::Cstring, hi::Cstring)::ReadStatError +end + +function readstat_get_variable(writer::WriterPtr, index::Integer) + @ccall libreadstat.readstat_get_variable(writer::WriterPtr, index::Cint)::VariablePtr +end + +# --------------------------------------------------------------------------- +# Notes, string refs, file-level metadata + +function readstat_add_note(writer::WriterPtr, note::AbstractString) + @ccall libreadstat.readstat_add_note(writer::WriterPtr, note::Cstring)::Cvoid +end + +function readstat_add_string_ref(writer::WriterPtr, string::AbstractString) + @ccall libreadstat.readstat_add_string_ref(writer::WriterPtr, string::Cstring)::StringRefPtr +end + +function readstat_get_string_ref(writer::WriterPtr, index::Integer) + @ccall libreadstat.readstat_get_string_ref(writer::WriterPtr, index::Cint)::StringRefPtr +end + +function readstat_writer_set_file_label(writer::WriterPtr, file_label::AbstractString) + @ccall libreadstat.readstat_writer_set_file_label(writer::WriterPtr, + file_label::Cstring)::ReadStatError +end + +function readstat_writer_set_file_timestamp(writer::WriterPtr, timestamp::Integer) + @ccall libreadstat.readstat_writer_set_file_timestamp(writer::WriterPtr, + timestamp::Int64)::ReadStatError +end + +function readstat_writer_set_fweight_variable(writer::WriterPtr, variable::VariablePtr) + @ccall libreadstat.readstat_writer_set_fweight_variable(writer::WriterPtr, + variable::VariablePtr)::ReadStatError +end + +function readstat_writer_set_file_format_version(writer::WriterPtr, version::Integer) + @ccall libreadstat.readstat_writer_set_file_format_version(writer::WriterPtr, + version::UInt8)::ReadStatError +end + +function readstat_writer_set_table_name(writer::WriterPtr, table_name::AbstractString) + @ccall libreadstat.readstat_writer_set_table_name(writer::WriterPtr, + table_name::Cstring)::ReadStatError +end + +function readstat_writer_set_file_format_is_64bit(writer::WriterPtr, is_64bit::Bool) + @ccall libreadstat.readstat_writer_set_file_format_is_64bit(writer::WriterPtr, + is_64bit::Cint)::ReadStatError +end + +function readstat_writer_set_compression(writer::WriterPtr, compression::ReadStatCompress) + @ccall libreadstat.readstat_writer_set_compression(writer::WriterPtr, + compression::ReadStatCompress)::ReadStatError +end + +function readstat_writer_set_error_handler(writer::WriterPtr, error_handler::Ptr{Cvoid}) + @ccall libreadstat.readstat_writer_set_error_handler(writer::WriterPtr, + error_handler::Ptr{Cvoid})::ReadStatError +end + +# --------------------------------------------------------------------------- +# Writing + +for begin_fn in (:readstat_begin_writing_dta, :readstat_begin_writing_por, + :readstat_begin_writing_sas7bdat, :readstat_begin_writing_sav, + :readstat_begin_writing_xport) + @eval function $begin_fn(writer::WriterPtr, user_ctx::Ptr{Cvoid}, row_count::Integer) + @ccall libreadstat.$begin_fn(writer::WriterPtr, user_ctx::Ptr{Cvoid}, + row_count::Clong)::ReadStatError + end +end + +function readstat_begin_writing_sas7bcat(writer::WriterPtr, user_ctx::Ptr{Cvoid}) + @ccall libreadstat.readstat_begin_writing_sas7bcat(writer::WriterPtr, + user_ctx::Ptr{Cvoid})::ReadStatError +end + +function readstat_validate_metadata(writer::WriterPtr) + @ccall libreadstat.readstat_validate_metadata(writer::WriterPtr)::ReadStatError +end + +function readstat_validate_variable(writer::WriterPtr, variable::VariablePtr) + @ccall libreadstat.readstat_validate_variable(writer::WriterPtr, + variable::VariablePtr)::ReadStatError +end + +function readstat_begin_row(writer::WriterPtr) + @ccall libreadstat.readstat_begin_row(writer::WriterPtr)::ReadStatError +end + +for (insert_fn, T) in ((:readstat_insert_int8_value, Int8), + (:readstat_insert_int16_value, Int16), + (:readstat_insert_int32_value, Int32), + (:readstat_insert_float_value, Float32), + (:readstat_insert_double_value, Float64)) + @eval function $insert_fn(writer::WriterPtr, variable::VariablePtr, value::$T) + @ccall libreadstat.$insert_fn(writer::WriterPtr, variable::VariablePtr, + value::$T)::ReadStatError + end +end + +function readstat_insert_string_value(writer::WriterPtr, variable::VariablePtr, + value::AbstractString) + @ccall libreadstat.readstat_insert_string_value(writer::WriterPtr, + variable::VariablePtr, value::Cstring)::ReadStatError +end + +function readstat_insert_string_ref(writer::WriterPtr, variable::VariablePtr, + ref::StringRefPtr) + @ccall libreadstat.readstat_insert_string_ref(writer::WriterPtr, + variable::VariablePtr, ref::StringRefPtr)::ReadStatError +end + +function readstat_insert_missing_value(writer::WriterPtr, variable::VariablePtr) + @ccall libreadstat.readstat_insert_missing_value(writer::WriterPtr, + variable::VariablePtr)::ReadStatError +end + +function readstat_insert_tagged_missing_value(writer::WriterPtr, variable::VariablePtr, + tag::Char) + @ccall libreadstat.readstat_insert_tagged_missing_value(writer::WriterPtr, + variable::VariablePtr, tag::Cchar)::ReadStatError +end + +function readstat_end_row(writer::WriterPtr) + @ccall libreadstat.readstat_end_row(writer::WriterPtr)::ReadStatError +end + +function readstat_end_writing(writer::WriterPtr) + @ccall libreadstat.readstat_end_writing(writer::WriterPtr)::ReadStatError +end diff --git a/src/writer/write.jl b/src/writer/write.jl new file mode 100644 index 0000000..aa7e6d3 --- /dev/null +++ b/src/writer/write.jl @@ -0,0 +1,346 @@ +# High-level writing: columns plus names (Queryverse-style, not a table +# protocol) or a ReadStatTable, canonicalized onto the six storage types the +# formats support, with metadata, value labels, tagged missings, and +# date/time re-encoding. + +const _WRITE_PRODUCER = Dict(:dta => :dta, :sav => :spss, :por => :spss, + :sas7bdat => :sas, :xport => :sas) + +# A column ready for the row loop: canonical storage values, NA mask access +# through the DataValueVector API, per-row tags, and variable attributes. +struct PreparedColumn + data::AbstractVector # DataValueVector{S} with S one of the 6 storage types + type::ReadStatType + storage_width::Int + format::String # derived format ("" = keep caller's hint) + tags::Union{Nothing,Vector{Char}} +end + +_plain_to_dv(v::AbstractVector{T}) where {T} = DataValueVector{T}(collect(v), fill(false, length(v))) + +function _prepare_column(col::AbstractVector, producer::Symbol, convert_datetime::Bool, + format::Symbol, tags::Union{Nothing,Vector{Char}}) + col isa LabeledArray && return _prepare_column(rawvalues(col), producer, convert_datetime, + format, tags) + col isa DataValueVector || (col = _plain_to_dv(col)) + T = eltype(eltype(col)) + n = length(col) + + # The por writer only handles doubles (the format is text-based), so + # every numeric column becomes Float64 there. + if T <: Real && format === :por && T !== Float64 + out = DataValueVector{Float64}(Vector{Float64}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = Float64(get(col[i]))) + end + return PreparedColumn(out, READSTAT_TYPE_DOUBLE, 0, "", tags) + elseif T === Bool + out = DataValueVector{Int8}(Vector{Int8}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = Int8(get(col[i]))) + end + return PreparedColumn(out, READSTAT_TYPE_INT8, 0, "", tags) + elseif T === Int8 || T === Int16 || T === Int32 || T === Float32 || T === Float64 + width = format === :xport && T === Float64 ? 8 : 0 + return PreparedColumn(col, rstype(T), width, "", tags) + elseif T <: Integer + # Narrow to Int32 when every value fits, otherwise go through Float64. + fits = all(i -> DataValues.isna(col[i]) || + typemin(Int32) <= get(col[i]) <= typemax(Int32), 1:n) + S = fits ? Int32 : Float64 + out = DataValueVector{S}(Vector{S}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = S(get(col[i]))) + end + return PreparedColumn(out, rstype(S), 0, "", tags) + elseif T <: Real + out = DataValueVector{Float64}(Vector{Float64}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = Float64(get(col[i]))) + end + return PreparedColumn(out, READSTAT_TYPE_DOUBLE, 0, "", tags) + elseif T <: AbstractString + out = DataValueVector{String}(fill("", n), fill(true, n)) + width = 1 + for i in 1:n + if !DataValues.isna(col[i]) + s = String(get(col[i])) + out[i] = s + width = max(width, sizeof(s)) + end + end + return PreparedColumn(out, READSTAT_TYPE_STRING, width, "", tags) + elseif convert_datetime && (T === Date || T === DateTime || T === HMS || T === Time) + f, S, fmt = _dt_encode(producer, T) + out = DataValueVector{S}(Vector{S}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = f(get(col[i]))) + end + width = format === :xport && S === Float64 ? 8 : 0 + return PreparedColumn(out, rstype(S), width, fmt, tags) + else + throw(ArgumentError("cannot write a column with element type $T" * + (T in (Date, DateTime, HMS, Time) ? " with convert_datetime=false" : ""))) + end +end + +function _dt_encode(producer::Symbol, ::Type{Date}) + if producer === :dta + (d -> Int32(Dates.value(d - STATA_EPOCH_DATE)), Int32, "%td") + elseif producer === :spss + (d -> Float64(Dates.value(d - SPSS_EPOCH_DATE)) * 86400.0, Float64, "EDATE10") + else + (d -> Float64(Dates.value(d - SAS_EPOCH_DATE)), Float64, "YYMMDD10") + end +end + +function _dt_encode(producer::Symbol, ::Type{DateTime}) + if producer === :dta + (dt -> Float64(Dates.value(dt - STATA_EPOCH_DATETIME)), Float64, "%tc") + elseif producer === :spss + (dt -> Float64(Dates.value(dt - SPSS_EPOCH_DATETIME)) / 1000.0, Float64, "DATETIME20") + else + (dt -> Float64(Dates.value(dt - SAS_EPOCH_DATETIME)) / 1000.0, Float64, "DATETIME19") + end +end + +_seconds(t::HMS) = unwrap(t) +_seconds(t::Time) = Dates.value(t) / 1.0e9 + +function _dt_encode(producer::Symbol, ::Type{T}) where {T<:Union{HMS,Time}} + if producer === :dta + # Stata has no pure time type; times of day become %tc datetimes on + # the epoch day, matching common Stata practice. + (t -> _seconds(t) * 1000.0, Float64, "%tcHH:MM:SS") + else + (t -> Float64(_seconds(t)), Float64, "TIME8") + end +end + +# Infer the C key type of a label set from its dictionary keys (Char keys +# label tagged missings and do not constrain the type). +function _label_set_type(dict::AbstractDict) + keytypes = [typeof(k) for k in keys(dict) if !(k isa Char)] + any(t -> t <: AbstractString, keytypes) && return READSTAT_TYPE_STRING + any(t -> t <: AbstractFloat, keytypes) && return READSTAT_TYPE_DOUBLE + return READSTAT_TYPE_INT32 +end + +function _write_label_sets!(w::Writer, value_labels) + sets = Dict{Symbol,LabelSet}() + for (name, dict) in value_labels + ls = add_label_set!(w, _label_set_type(dict), name) + for (k, v) in dict + label!(ls, k, v) + end + sets[Symbol(name)] = ls + end + return sets +end + +_getvec(v::Nothing, i) = nothing +_getvec(v::AbstractVector, i) = v[i] + +function write_data_file(dest::Union{AbstractString,IO}, format::Symbol, + columns::AbstractVector, names::AbstractVector{Symbol}; + labels::Union{Nothing,AbstractVector{<:AbstractString}}=nothing, + formats::Union{Nothing,AbstractVector{<:AbstractString}}=nothing, + value_labels=Dict{Symbol,ValueLabelDict}(), + vallabels::Union{Nothing,AbstractVector{Symbol}}=nothing, + measures::Union{Nothing,AbstractVector{ReadStatMeasure}}=nothing, + alignments::Union{Nothing,AbstractVector{ReadStatAlignment}}=nothing, + display_widths::Union{Nothing,AbstractVector{<:Integer}}=nothing, + missing_ranges::Union{Nothing,AbstractVector}=nothing, + tags::Union{Nothing,AbstractVector}=nothing, + file_label::AbstractString="", table_name::AbstractString="", + notes::AbstractVector{<:AbstractString}=String[], + timestamp::Union{Nothing,DateTime}=nothing, + version::Union{Nothing,Integer}=nothing, + compress::Symbol=:none, is_64bit::Bool=true, + fweight::Union{Nothing,Symbol}=nothing, + convert_datetime::Bool=true) + ncols = length(columns) + ncols == length(names) || + throw(ArgumentError("got $(length(columns)) columns for $(length(names)) names")) + for (kw, v) in (("labels", labels), ("formats", formats), ("vallabels", vallabels), + ("measures", measures), ("alignments", alignments), + ("display_widths", display_widths), ("missing_ranges", missing_ranges), + ("tags", tags)) + v === nothing || length(v) == ncols || + throw(ArgumentError("$kw must have one entry per column")) + end + nrows = ncols == 0 ? 0 : length(columns[1]) + all(c -> length(c) == nrows, columns) || + throw(ArgumentError("all columns must have the same length")) + + # SPSS portable is an uppercase-only format; the C writer silently writes + # no data under lowercase names, so normalize them here (a por read + # returns uppercase names regardless). + format === :por && (names = [Symbol(uppercase(String(n))) for n in names]) + + producer = _WRITE_PRODUCER[format] + prepared = PreparedColumn[ + _prepare_column(columns[i], producer, convert_datetime, format, _getvec(tags, i)) + for i in 1:ncols] + + w = Writer(dest) + try + sets = _write_label_sets!(w, value_labels) + vars = Vector{WriterVariable}(undef, ncols) + for i in 1:ncols + p = prepared[i] + # LabeledArray columns bring their own label set when none is named. + vallabel = _getvec(vallabels, i) + if vallabel === nothing && columns[i] isa LabeledArray + vallabel = Symbol(:__auto_labels_, i) + d = getvaluelabels(columns[i]) + ls = add_label_set!(w, _label_set_type(d), vallabel) + for (k, v) in d + label!(ls, k, v) + end + sets[vallabel] = ls + end + fmt = p.format + isempty(fmt) && formats !== nothing && (fmt = formats[i]) + ranges = missing_ranges === nothing ? () : + Tuple((r isa Tuple ? r : (r, r)) for r in missing_ranges[i]) + vars[i] = add_variable!(w, names[i], p.type; + storage_width=p.storage_width, + label=labels === nothing ? "" : labels[i], + format=fmt, + label_set=vallabel === nothing || vallabel === Symbol("") ? nothing : + get(sets, vallabel, nothing), + measure=measures === nothing ? READSTAT_MEASURE_UNKNOWN : measures[i], + alignment=alignments === nothing ? READSTAT_ALIGNMENT_UNKNOWN : alignments[i], + display_width=display_widths === nothing ? 0 : display_widths[i], + missing_ranges=ranges) + end + isempty(file_label) || file_label!(w, file_label) + isempty(table_name) || table_name!(w, table_name) + timestamp === nothing || timestamp!(w, timestamp) + version === nothing || format_version!(w, version) + compress === :none || compression!(w, compress) + format in (:sas7bdat, :xport) && is_64bit!(w, is_64bit) + for note in notes + add_note!(w, note) + end + if fweight !== nothing + j = findfirst(==(fweight), names) + j === nothing && throw(ArgumentError("fweight column $fweight not found")) + fweight!(w, vars[j]) + end + + begin_writing!(w, format, nrows) + for r in 1:nrows + begin_row!(w) + for i in 1:ncols + _insert_cell!(w, vars[i], prepared[i], r) + end + end_row!(w) + end + end_writing!(w) + finally + close(w) + end + return dest +end + +function _insert_cell!(w::Writer, var::WriterVariable, p::PreparedColumn, r::Int) + t = p.tags + if t !== nothing && t[r] != '\0' + insert_tagged_missing!(w, var, t[r]) + return + end + _insert_cell!(w, var, p.data, r) +end + +function _insert_cell!(w::Writer, var::WriterVariable, col::DataValueVector{T}, r::Int) where {T} + v = col[r] + DataValues.isna(v) ? insert_missing!(w, var) : insert_value!(w, var, get(v)) + return +end + +const _WRITE_KWARGS_DOC = """ +`columns` is a vector of columns (`DataValueVector`s, [`LabeledArray`](@ref)s +— which bring their value labels along — or plain vectors) and `names` the +matching column names. Keyword arguments, all optional: + +- `labels`, `formats`, `measures`, `alignments`, `display_widths`: one + variable attribute per column. +- `value_labels::Dict{Symbol,ValueLabelDict}` plus `vallabels` (one label-set + name per column, `Symbol("")` for none): explicit value-label sets. +- `missing_ranges`: per column, a collection of SPSS user-missing values or + `(lo, hi)` range tuples. +- `tags`: per column, `nothing` or a `Vector{Char}` marking tagged missing + values (`'a'`-`'z'`, `'\\0'` elsewhere; Stata/SAS formats). +- `file_label`, `notes`, `timestamp`, `version`, `table_name` (XPORT), + `is_64bit` (SAS), `compress` (`:rows` for sas7bdat/sav, `:binary` for + zsav), `fweight` (name of the frequency-weight column). +- `convert_datetime=true`: encode `Date`/`DateTime`/[`HMS`](@ref)/`Time` + columns into the format's native representation with a matching display + format. + +Instead of a path, `dest` may be any writable `IO`. +""" + +for (fn, fmt) in ((:write_dta, :dta), (:write_sav, :sav), (:write_por, :por), + (:write_sas7bdat, :sas7bdat), (:write_xport, :xport)) + @eval begin + """ + $($(string(fn)))(dest, columns, names::Vector{Symbol}; kwargs...) + $($(string(fn)))(dest, tbl::ReadStatTable; kwargs...) + + Write columns to $($(string(fmt))) format. + + $_WRITE_KWARGS_DOC + """ + $fn(dest::Union{AbstractString,IO}, columns::AbstractVector, + names::AbstractVector{Symbol}; kwargs...) = + write_data_file(dest, $(QuoteNode(fmt)), columns, names; kwargs...) + $fn(dest::Union{AbstractString,IO}, tbl::ReadStatTable; kwargs...) = + _write_table(dest, $(QuoteNode(fmt)), tbl; kwargs...) + end +end + +""" + write_sas7bcat(dest, value_labels::Dict{Symbol,<:AbstractDict}) + +Write a SAS value-label catalog holding the given label sets (as read by +[`read_sas7bcat`](@ref)). + +!!! warning + The catalog writer in readstat 1.1.9 only handles numeric label sets + correctly; string-keyed sets do not survive a round trip. +""" +function write_sas7bcat(dest::Union{AbstractString,IO}, value_labels::AbstractDict) + w = Writer(dest) + try + _write_label_sets!(w, value_labels) + begin_writing!(w, :sas7bcat) + end_writing!(w) + finally + close(w) + end + return dest +end + +# Write a ReadStatTable, carrying its metadata across; explicit keyword +# arguments override what the table provides. +function _write_table(dest, format::Symbol, tbl::ReadStatTable; kwargs...) + meta = filemetadata(tbl) + colmeta = getfield(tbl, :colmeta) + columns = getfield(tbl, :cols) + provided = (; + labels=[m.label for m in colmeta], + value_labels=meta.value_labels, + vallabels=[m.vallabel for m in colmeta], + measures=[m.measure for m in colmeta], + alignments=[m.alignment for m in colmeta], + tags=getfield(tbl, :tags), + file_label=meta.file_label, + notes=meta.notes, + fweight=meta.fweight === Symbol("") ? nothing : meta.fweight, + ) + return write_data_file(dest, format, columns, names(tbl); + merge(provided, NamedTuple(kwargs))...) +end diff --git a/src/writer/writer.jl b/src/writer/writer.jl new file mode 100644 index 0000000..0def980 --- /dev/null +++ b/src/writer/writer.jl @@ -0,0 +1,297 @@ +# Low-level writer: a thin, safe Julia layer over the C writer API. It +# mirrors the C call sequence one-to-one — define label sets, define +# variables, begin writing, insert rows, end — and streams all output bytes +# into any Julia IO through the C data-writer callback. The high-level +# write_dta/write_sav/... functions in writer/write.jl are built on top; +# reach for this layer directly to control every detail the C API exposes. + +""" + Writer(io::IO) + Writer(path::AbstractString) + +A handle for writing a stat-package file through the C writer API. Follow +the C call order: [`add_label_set!`](@ref)/[`label!`](@ref) for value +labels, [`add_variable!`](@ref) for each column, the file-level setters +([`file_label!`](@ref), [`timestamp!`](@ref), [`compression!`](@ref), ...), +then [`begin_writing!`](@ref), one [`begin_row!`](@ref)/`insert_*`/ +[`end_row!`](@ref) cycle per observation, and finally `close`. Output +bytes stream into the given IO (a path is opened for writing and closed +again by `close`). +""" +mutable struct Writer + ptr::WriterPtr + io::IO + owns_io::Bool + err::Union{Nothing,Tuple{Any,Any}} + began::Bool + ended::Bool +end + +struct WriterVariable + ptr::VariablePtr +end + +struct LabelSet + ptr::LabelSetPtr + type::ReadStatType +end + +struct StringRef + ptr::StringRefPtr +end + +function Writer(io::IO) + ptr = readstat_writer_init() + ptr == C_NULL && error("readstat_writer_init failed") + w = Writer(ptr, io, false, nothing, false, false) + _writer_check(w, readstat_set_data_writer(ptr, CF_DATA_WRITER[])) + finalizer(_writer_finalize, w) + return w +end + +Writer(path::AbstractString) = (w = Writer(open(path, "w")); w.owns_io = true; w) + +function _writer_finalize(w::Writer) + if w.ptr != C_NULL + readstat_writer_free(w.ptr) + w.ptr = C_NULL + end + return +end + +function handle_data_write(data::Ptr{Cvoid}, len::Csize_t, ctx::Ptr{Cvoid})::Cssize_t + w = unsafe_pointer_to_objref(ctx)::Writer + try + unsafe_write(w.io, Ptr{UInt8}(data), UInt(len)) + return Cssize_t(len) + catch e + w.err = (e, catch_backtrace()) + return Cssize_t(-1) + end +end + +const CF_DATA_WRITER = Ref(C_NULL) + +function _init_writer_cfunctions() + CF_DATA_WRITER[] = @cfunction(handle_data_write, Cssize_t, (Ptr{Cvoid}, Csize_t, Ptr{Cvoid})) + return nothing +end + +function _writer_check(w::Writer, err::ReadStatError) + if w.err !== nothing + e, _ = w.err + throw(e) + end + err == READSTAT_OK || error("readstat writer error: $(readstat_error_message(err))") + return w +end + +_ptr(w::Writer) = (w.ptr == C_NULL && error("Writer is closed"); w.ptr) + +# The Julia storage type used for each writable ReadStatType. +function rstype(::Type{T}) where {T} + T === Int8 || T === Bool ? READSTAT_TYPE_INT8 : + T === Int16 ? READSTAT_TYPE_INT16 : + T <: Integer ? READSTAT_TYPE_INT32 : + T === Float32 ? READSTAT_TYPE_FLOAT : + T <: Real ? READSTAT_TYPE_DOUBLE : + T <: Union{AbstractString,Char} ? READSTAT_TYPE_STRING : + throw(ArgumentError("no readstat storage type for $T")) +end + +""" + add_label_set!(w::Writer, type, name) -> LabelSet + +Create a named value-label set. `type` is a `ReadStatType` or a Julia type +(`Int32`, `Float64`, `String`); add entries with [`label!`](@ref) and attach +the set to variables via `add_variable!(...; label_set=...)`. +""" +add_label_set!(w::Writer, type::ReadStatType, name::Union{Symbol,AbstractString}) = + LabelSet(readstat_add_label_set(_ptr(w), type, string(name)), type) +add_label_set!(w::Writer, ::Type{T}, name::Union{Symbol,AbstractString}) where {T} = + add_label_set!(w, rstype(T), name) + +""" + label!(ls::LabelSet, value, label::AbstractString) + +Add one entry to a value-label set. Numeric values go to the set's numeric +key type, strings to string keys, and a `Char` labels a tagged missing value +(`.a`-`.z`, Stata/SAS only). +""" +function label!(ls::LabelSet, value::Real, label::AbstractString) + if ls.type == READSTAT_TYPE_DOUBLE || ls.type == READSTAT_TYPE_FLOAT + readstat_label_double_value(ls.ptr, Float64(value), label) + else + readstat_label_int32_value(ls.ptr, Int32(value), label) + end + return ls +end +label!(ls::LabelSet, value::AbstractString, label::AbstractString) = + (readstat_label_string_value(ls.ptr, value, label); ls) +label!(ls::LabelSet, tag::Char, label::AbstractString) = + (readstat_label_tagged_value(ls.ptr, tag, label); ls) + +""" + add_variable!(w::Writer, name, type; storage_width=0, label="", format="", + label_set=nothing, measure=READSTAT_MEASURE_UNKNOWN, + alignment=READSTAT_ALIGNMENT_UNKNOWN, display_width=0, + missing_values=(), missing_ranges=()) -> WriterVariable + +Define the next variable. `type` is a `ReadStatType` or a Julia type; +`storage_width` matters for strings (all formats) and doubles (XPORT). +`missing_values`/`missing_ranges` declare SPSS user-defined missing values +(numbers or strings; ranges as 2-tuples). +""" +function add_variable!(w::Writer, name::Union{Symbol,AbstractString}, type::ReadStatType; + storage_width::Integer=0, label::AbstractString="", + format::AbstractString="", label_set::Union{Nothing,LabelSet}=nothing, + measure::ReadStatMeasure=READSTAT_MEASURE_UNKNOWN, + alignment::ReadStatAlignment=READSTAT_ALIGNMENT_UNKNOWN, + display_width::Integer=0, missing_values=(), missing_ranges=()) + ptr = readstat_add_variable(_ptr(w), string(name), type, storage_width) + ptr == C_NULL && error("readstat_add_variable failed for $name") + isempty(label) || readstat_variable_set_label(ptr, label) + isempty(format) || readstat_variable_set_format(ptr, format) + label_set === nothing || readstat_variable_set_label_set(ptr, label_set.ptr) + measure == READSTAT_MEASURE_UNKNOWN || readstat_variable_set_measure(ptr, measure) + alignment == READSTAT_ALIGNMENT_UNKNOWN || readstat_variable_set_alignment(ptr, alignment) + display_width == 0 || readstat_variable_set_display_width(ptr, display_width) + for v in missing_values + _writer_check(w, v isa AbstractString ? + readstat_variable_add_missing_string_value(ptr, v) : + readstat_variable_add_missing_double_value(ptr, Float64(v))) + end + for (lo, hi) in missing_ranges + _writer_check(w, lo isa AbstractString ? + readstat_variable_add_missing_string_range(ptr, lo, hi) : + readstat_variable_add_missing_double_range(ptr, Float64(lo), Float64(hi))) + end + return WriterVariable(ptr) +end +add_variable!(w::Writer, name::Union{Symbol,AbstractString}, ::Type{T}; kwargs...) where {T} = + add_variable!(w, name, rstype(T); kwargs...) + +""" + add_note!(w::Writer, note) + +Add a file note (SPSS Document Record line — at most 80 characters there — +or a Stata note). +""" +add_note!(w::Writer, note::AbstractString) = (readstat_add_note(_ptr(w), note); w) + +""" + add_string_ref!(w::Writer, s) -> StringRef + +Intern a string for a `READSTAT_TYPE_STRING_REF` (Stata strL) column; +insert it into rows with [`insert_string_ref!`](@ref). Refs can be shared +across columns and rows. +""" +add_string_ref!(w::Writer, s::AbstractString) = StringRef(readstat_add_string_ref(_ptr(w), s)) + +file_label!(w::Writer, s::AbstractString) = + _writer_check(w, readstat_writer_set_file_label(_ptr(w), s)) +timestamp!(w::Writer, t::DateTime) = + _writer_check(w, readstat_writer_set_file_timestamp(_ptr(w), + round(Int64, Dates.datetime2unix(t)))) +fweight!(w::Writer, var::WriterVariable) = + _writer_check(w, readstat_writer_set_fweight_variable(_ptr(w), var.ptr)) +format_version!(w::Writer, v::Integer) = + _writer_check(w, readstat_writer_set_file_format_version(_ptr(w), v)) +table_name!(w::Writer, s::AbstractString) = + _writer_check(w, readstat_writer_set_table_name(_ptr(w), s)) +is_64bit!(w::Writer, b::Bool) = + _writer_check(w, readstat_writer_set_file_format_is_64bit(_ptr(w), b)) + +const COMPRESSION_BY_NAME = Dict(:none => READSTAT_COMPRESS_NONE, + :rows => READSTAT_COMPRESS_ROWS, :binary => READSTAT_COMPRESS_BINARY) + +""" + compression!(w::Writer, c) + +Set the output compression: `:none`, `:rows` (sas7bdat and sav), or +`:binary` (sav only — produces a zsav file). +""" +function compression!(w::Writer, c::Symbol) + haskey(COMPRESSION_BY_NAME, c) || + throw(ArgumentError("compression must be :none, :rows, or :binary")) + _writer_check(w, readstat_writer_set_compression(_ptr(w), COMPRESSION_BY_NAME[c])) +end + +""" + begin_writing!(w::Writer, format::Symbol, row_count) + +Start writing the file body for `format` (`:dta`, `:sav`, `:por`, +`:sas7bdat`, `:xport`, or `:sas7bcat`, which takes no rows). All label +sets, variables, and file-level metadata must be defined beforehand. +""" +function begin_writing!(w::Writer, format::Symbol, row_count::Integer=0) + ptr = _ptr(w) + ctx = pointer_from_objref(w) + err = if format === :dta + readstat_begin_writing_dta(ptr, ctx, row_count) + elseif format === :sav + readstat_begin_writing_sav(ptr, ctx, row_count) + elseif format === :por + readstat_begin_writing_por(ptr, ctx, row_count) + elseif format === :sas7bdat + readstat_begin_writing_sas7bdat(ptr, ctx, row_count) + elseif format === :xport + readstat_begin_writing_xport(ptr, ctx, row_count) + elseif format === :sas7bcat + readstat_begin_writing_sas7bcat(ptr, ctx) + else + throw(ArgumentError("unknown format $format")) + end + _writer_check(w, err) + w.began = true + return w +end + +validate_metadata(w::Writer) = _writer_check(w, readstat_validate_metadata(_ptr(w))) +validate_variable(w::Writer, var::WriterVariable) = + _writer_check(w, readstat_validate_variable(_ptr(w), var.ptr)) + +begin_row!(w::Writer) = _writer_check(w, readstat_begin_row(_ptr(w))) +end_row!(w::Writer) = _writer_check(w, readstat_end_row(_ptr(w))) + +insert_value!(w::Writer, var::WriterVariable, v::Int8) = + _writer_check(w, readstat_insert_int8_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::Int16) = + _writer_check(w, readstat_insert_int16_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::Int32) = + _writer_check(w, readstat_insert_int32_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::Float32) = + _writer_check(w, readstat_insert_float_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::Float64) = + _writer_check(w, readstat_insert_double_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::AbstractString) = + _writer_check(w, readstat_insert_string_value(_ptr(w), var.ptr, v)) +insert_missing!(w::Writer, var::WriterVariable) = + _writer_check(w, readstat_insert_missing_value(_ptr(w), var.ptr)) +insert_tagged_missing!(w::Writer, var::WriterVariable, tag::Char) = + _writer_check(w, readstat_insert_tagged_missing_value(_ptr(w), var.ptr, tag)) +insert_string_ref!(w::Writer, var::WriterVariable, ref::StringRef) = + _writer_check(w, readstat_insert_string_ref(_ptr(w), var.ptr, ref.ptr)) + +""" + end_writing!(w::Writer) + +Finish the file body. Called automatically by `close(w)` when writing has +begun and not yet ended. +""" +function end_writing!(w::Writer) + w.ended && return w + GC.@preserve w _writer_check(w, readstat_end_writing(_ptr(w))) + w.ended = true + return w +end + +function Base.close(w::Writer) + if w.ptr != C_NULL + w.began && !w.ended && end_writing!(w) + readstat_writer_free(w.ptr) + w.ptr = C_NULL + end + w.owns_io && close(w.io) + return nothing +end diff --git a/test/runtests.jl b/test/runtests.jl index a7c84b2..a73e008 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,6 +9,7 @@ include("test_datetime.jl") include("test_io.jl") include("test_threads.jl") include("test_source.jl") +include("test_write.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_write.jl b/test/test_write.jl new file mode 100644 index 0000000..b31fc64 --- /dev/null +++ b/test/test_write.jl @@ -0,0 +1,182 @@ +@testitem "writer round trips" begin + using DataValues + using Dates + using ReadStat: Writer, add_variable!, add_string_ref!, begin_writing!, + begin_row!, insert_string_ref!, end_row! + using ReadStat: CAPI + + tmp = mktempdir() + cols = Any[ + DataValueArray{Int8}([1, 2, NA]), + DataValueArray{Int16}([10, 20, NA]), + DataValueArray{Int32}([100, 200, NA]), + DataValueArray{Float32}([1.5f0, 2.5f0, NA]), + DataValueArray{Float64}([1.25, 2.75, NA]), + DataValueArray{String}(["ab", "cdef", NA]), + DataValueArray{Date}([Date(2020, 1, 2), Date(1959, 12, 31), NA]), + DataValueArray{DateTime}([DateTime(2020, 1, 2, 3, 4, 5), DateTime(1960, 1, 1), NA]), + ] + cnames = [:vbyte, :vint, :vlong, :vfloat, :vdouble, :vstr, :vdate, :vdt] + writers = (dta=write_dta, sav=write_sav, por=write_por, + sas7bdat=write_sas7bdat, xport=write_xport) + + @testset "all types via $fmt" for fmt in (:dta, :sav, :por, :sas7bdat, :xport) + path = joinpath(tmp, "types_out.$(fmt === :xport ? "xpt" : fmt)") + writers[fmt](path, cols, cnames; file_label="roundtrip") + tbl = readstat(path; format=fmt) + + @test size(tbl) == (3, 8) + expected_names = fmt === :por ? + [Symbol(uppercase(String(n))) for n in cnames] : cnames + @test names(tbl) == expected_names + for i in 1:5 + @test tbl[i] == cols[i] # numeric equality across widened storage + end + @test get(tbl[6][1]) == "ab" && get(tbl[6][2]) == "cdef" + # A missing string cell comes back as the format's missing string, + # which is the empty string. + @test isna(tbl[6][3]) || get(tbl[6][3]) == "" + @test tbl[7] == cols[7] + @test tbl[8] == cols[8] + @test filemetadata(tbl).file_label == "roundtrip" + + # Writing into an IO works the same. + io = IOBuffer() + writers[fmt](io, cols, cnames) + @test size(readstat(IOBuffer(take!(io)); format=fmt)) == (3, 8) + end + + @testset "labels, tags, notes, attributes (dta)" begin + path = joinpath(tmp, "labels_out.dta") + write_dta(path, Any[DataValueArray{Int32}([1, 2, NA])], [:grade]; + value_labels=Dict(:gl => ReadStat.ValueLabelDict( + Int32(1) => "low", Int32(2) => "high", 'a' => "refused")), + vallabels=[:gl], + tags=[['\0', '\0', 'a']], + labels=["the grade"], + notes=["a note", "another note"], + file_label="labeled", + timestamp=DateTime(2020, 6, 1, 12, 0, 0)) + tbl = read_dta(path) + @test missingtags(tbl, :grade) == ['\0', '\0', 'a'] + d = valuelabels(tbl, :grade) + @test d[Int32(1)] == "low" && d[Int32(2)] == "high" && d['a'] == "refused" + @test varmetadata(tbl, :grade).label == "the grade" + @test filemetadata(tbl).notes == ["a note", "another note"] + @test year(filemetadata(tbl).modified_time) == 2020 + end + + @testset "LabeledArray columns carry their labels" begin + path = joinpath(tmp, "la_out.dta") + la = LabeledArray(DataValueArray{Int32}([1, 2, 1]), + ReadStat.ValueLabelDict(Int32(1) => "one", Int32(2) => "two")) + write_dta(path, Any[la], [:k]) + tbl = read_dta(path) + d = valuelabels(tbl, :k) + @test d !== nothing && d[Int32(1)] == "one" && d[Int32(2)] == "two" + @test tbl[:k] == DataValueArray{Int32}([1, 2, 1]) + end + + @testset "SPSS user-defined missing values (sav)" begin + path = joinpath(tmp, "um_out.sav") + write_sav(path, Any[DataValueArray{Float64}([1.0, -1.0, 99.0])], [:x]; + missing_ranges=[[-1.0, (90.0, 100.0)]]) + tbl = read_sav(path) + @test isna.(tbl[:x]) == [false, true, true] + kept = read_sav(path; user_missing=:keep) + @test kept[:x] == DataValueArray{Float64}([1.0, -1.0, 99.0]) + ranges = varmetadata(tbl, :x).missing_ranges + @test (90.0, 100.0) in ranges && (-1.0, -1.0) in ranges + end + + @testset "sas7bcat" begin + path = joinpath(tmp, "cat_out.sas7bcat") + write_sas7bcat(path, Dict(:NUMFMT => ReadStat.ValueLabelDict(1.0 => "yes", 2.0 => "no"))) + cat = read_sas7bcat(path) + @test cat[:NUMFMT][1.0] == "yes" && cat[:NUMFMT][2.0] == "no" + end + + @testset "strL through the low-level writer" begin + path = joinpath(tmp, "strl_out.dta") + w = Writer(path) + ref = add_string_ref!(w, "a very long shared string") + v = add_variable!(w, :s, CAPI.READSTAT_TYPE_STRING_REF) + begin_writing!(w, :dta, 2) + for _ in 1:2 + begin_row!(w) + insert_string_ref!(w, v, ref) + end_row!(w) + end + close(w) + tbl = read_dta(path) + @test [get(x) for x in tbl[:s]] == fill("a very long shared string", 2) + end + + @testset "table round trips" begin + src = read_dta(joinpath(@__DIR__, "data", "alltypes.dta")) + path = joinpath(tmp, "tbl_out.dta") + write_dta(path, src) + tbl = read_dta(path) + @test names(tbl) == names(src) + @test missingtags(tbl, :vbyte) == missingtags(src, :vbyte) + @test valuelabels(tbl, :vbyte) == valuelabels(src, :vbyte) + @test tbl[:vdate] == src[:vdate] + @test tbl[:vstrL] == src[:vstrL] + + sav = read_sav(joinpath(@__DIR__, "data", "sample.sav")) + path2 = joinpath(tmp, "tbl_out.sav") + write_sav(path2, sav) + tbl2 = read_sav(path2) + for i in 1:size(sav, 2) + @test isequal(collect(tbl2[i]), collect(sav[i])) + end + @test valuelabels(tbl2, :mylabl) == valuelabels(sav, :mylabl) + end + + @testset "compression" begin + path = joinpath(tmp, "z_out.zsav") + write_sav(path, Any[DataValueArray{Float64}([1.0, 2.0])], [:x]; compress=:binary) + @test size(read_sav(path), 1) == 2 + path2 = joinpath(tmp, "c_out.sas7bdat") + write_sas7bdat(path2, Any[DataValueArray{Float64}([1.0, 2.0])], [:x]; compress=:rows) + @test size(read_sas7bdat(path2), 1) == 2 + end + + @testset "validation errors" begin + one = Any[DataValueArray{Float64}([1.0])] + @test_throws ErrorException write_sav(joinpath(tmp, "badnote.sav"), one, [:x]; + notes=[repeat("x", 200)]) # SPSS notes are capped at 80 characters + @test_throws ArgumentError write_dta(joinpath(tmp, "mismatch.dta"), one, [:x, :y]) + @test_throws ArgumentError write_dta(joinpath(tmp, "raggedy.dta"), + Any[DataValueArray{Float64}([1.0]), DataValueArray{Float64}([1.0, 2.0])], [:x, :y]) + @test_throws ArgumentError write_dta(joinpath(tmp, "nofw.dta"), one, [:x]; fweight=:nope) + end +end + +@testitem "large write/read round trip with ntasks and chunks" begin + using DataValues + + n = 50_000 + isna_mask = [i % 997 == 0 for i in 1:n] + cols = Any[ + DataValueVector{Int32}(collect(Int32, 1:n), copy(isna_mask)), + DataValueVector{Float64}(collect(1:n) ./ 3, fill(false, n)), + DataValueVector{String}(string.(mod.(1:n, 100)), copy(isna_mask)), + ] + path = joinpath(mktempdir(), "big.dta") + write_dta(path, cols, [:id, :val, :bucket]) + + serial = read_dta(path; ntasks=1) + @test size(serial) == (n, 3) + @test isna.(serial[:id]) == isna_mask + @test get(serial[:val][n]) ≈ n / 3 + + threaded = read_dta(path; ntasks=4) + for i in 1:3 + @test isequal(collect(threaded[i]), collect(serial[i])) + end + + reassembled = reduce(vcat, + [collect(p[:id]) for p in chunks(ReadStatSource(path); chunksize=7_000)]) + @test isequal(reassembled, collect(serial[:id])) +end From d3f47748b2bf73d0bdc0ab8e1225f88ac6db94d4 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:49:06 -0700 Subject: [PATCH 11/14] Add the schema/fixed-width text path (read_txt) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit read_txt reads a fixed-width text data file described by a schema file — a Stata dictionary, SAS commands with an INPUT statement, or SPSS DATA LIST syntax — through the C library''s schema parsers and readstat_parse_txt, with the usual usecols/row_limit/row_offset/handler_encoding/ convert_datetime keywords (date/time formats interpret per the schema''s producer). The text parser ignores readstat_set_row_offset, so the offset is emulated by shifting the handler''s row base. The CAPI submodule gains the five schema functions; a Stata dictionary fixture exercises the path end to end. Co-Authored-By: Claude Fable 5 --- src/ReadStat.jl | 4 ++- src/capi/schema.jl | 24 ++++++++++++++++ src/reader/handlers.jl | 3 ++ src/reader/read.jl | 62 ++++++++++++++++++++++++++++++++++++++++++ test/data/fixed.txt | 3 ++ test/data/schema.dct | 5 ++++ test/runtests.jl | 1 + test/test_schema.jl | 27 ++++++++++++++++++ 8 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 src/capi/schema.jl create mode 100644 test/data/fixed.txt create mode 100644 test/data/schema.dct create mode 100644 test/test_schema.jl diff --git a/src/ReadStat.jl b/src/ReadStat.jl index e412d65..2d204ef 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -10,7 +10,8 @@ export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, LabeledValue, LabeledArray, labeled, unwrap, valuelabel, rawvalues, getvaluelabels, HMS, ReadStatSource, ReadStatChunks, schema, colnames, coltypes, nrows, supports, chunks, - write_dta, write_sav, write_por, write_sas7bdat, write_xport, write_sas7bcat + write_dta, write_sav, write_por, write_sas7bdat, write_xport, write_sas7bcat, + read_txt public CAPI @@ -39,6 +40,7 @@ using ReadStat_jll: libreadstat include("capi/enums.jl") include("capi/value.jl") include("capi/parser.jl") +include("capi/schema.jl") include("capi/writer.jl") end # module CAPI diff --git a/src/capi/schema.jl b/src/capi/schema.jl new file mode 100644 index 0000000..f14023e --- /dev/null +++ b/src/capi/schema.jl @@ -0,0 +1,24 @@ +# The schema/plain-text path: parse a SAS command file, SPSS command file, +# or Stata dictionary into a schema, then drive the fixed-width text parser +# with it. + +export readstat_parse_sas_commands, readstat_parse_spss_commands, + readstat_parse_stata_dictionary, readstat_parse_txt, readstat_schema_free + +for schema_fn in (:readstat_parse_sas_commands, :readstat_parse_spss_commands, + :readstat_parse_stata_dictionary) + @eval function $schema_fn(parser::ParserPtr, path::AbstractString, user_ctx, + out_error::Ref{ReadStatError}) + @ccall libreadstat.$schema_fn(parser::ParserPtr, path::Cstring, user_ctx::Any, + out_error::Ref{ReadStatError})::SchemaPtr + end +end + +function readstat_parse_txt(parser::ParserPtr, path::AbstractString, schema::SchemaPtr, user_ctx) + @ccall libreadstat.readstat_parse_txt(parser::ParserPtr, path::Cstring, + schema::SchemaPtr, user_ctx::Any)::ReadStatError +end + +function readstat_schema_free(schema::SchemaPtr) + @ccall libreadstat.readstat_schema_free(schema::SchemaPtr)::Cvoid +end diff --git a/src/reader/handlers.jl b/src/reader/handlers.jl index 0790bdc..a6b62e0 100644 --- a/src/reader/handlers.jl +++ b/src/reader/handlers.jl @@ -97,6 +97,9 @@ function handle_value(obs_index::Cint, variable::VariablePtr, value::ReadStatVal try idx = readstat_variable_get_index_after_skipping(variable) + 1 row = pc.row_base + Int(obs_index) + 1 + # A negative row base emulates a row offset for parsers that ignore + # readstat_set_row_offset (the fixed-width text parser). + row < 1 && return READSTAT_HANDLER_OK cols = pc.cols code, slot = @inbounds cols.slots[idx] diff --git a/src/reader/read.jl b/src/reader/read.jl index 3769500..003e07f 100644 --- a/src/reader/read.jl +++ b/src/reader/read.jl @@ -318,6 +318,68 @@ function read_sas7bcat(path::AbstractString) return pc.meta.value_labels end +const _SCHEMA_PRODUCER = Dict(:sas_commands => :sas7bdat, :spss_commands => :sav, + :stata_dictionary => :dta) + +""" + read_txt(data_path, schema_path; schema_format, kwargs...) -> ReadStatTable + +Read a fixed-width text data file described by a schema file: +`schema_format` selects the schema flavor — `:sas_commands` (a SAS program +with an `INPUT` statement), `:spss_commands` (SPSS `DATA LIST` syntax), or +`:stata_dictionary` (a Stata `.dct` file). Supports the `usecols`, +`row_limit`, `row_offset`, `handler_encoding`, and `convert_datetime` +keyword arguments with the same meaning as the binary readers (date/time +formats are interpreted per the schema's producer). +""" +function read_txt(data_path::AbstractString, schema_path::AbstractString; + schema_format::Symbol, + usecols=nothing, + row_limit::Union{Nothing,Integer}=nothing, + row_offset::Integer=0, + handler_encoding::Union{Nothing,AbstractString}=nothing, + convert_datetime::Bool=true) + haskey(_SCHEMA_PRODUCER, schema_format) || + throw(ArgumentError("schema_format must be :sas_commands, :spss_commands, or :stata_dictionary")) + isfile(data_path) || throw(ArgumentError("file not found: $data_path")) + isfile(schema_path) || throw(ArgumentError("file not found: $schema_path")) + + pc = ParseContext() + pc.usecols = _colselector(usecols) + # The text parser ignores readstat_set_row_offset, so the offset is + # emulated by shifting the row base negative and widening the limit. + offset = Int(row_offset) + pc.row_base = -offset + pc.row_limit = row_limit === nothing ? -1 : Int(row_limit) + offset + pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) + pc.convert_datetime = convert_datetime + pc.file_format = _SCHEMA_PRODUCER[schema_format] + + parser = readstat_parser_init() + schema = SchemaPtr(C_NULL) + local retval + try + _set_handlers!(parser, pc) + out_error = Ref(READSTAT_OK) + schema = if schema_format === :sas_commands + CAPI.readstat_parse_sas_commands(parser, schema_path, pc, out_error) + elseif schema_format === :spss_commands + CAPI.readstat_parse_spss_commands(parser, schema_path, pc, out_error) + else + CAPI.readstat_parse_stata_dictionary(parser, schema_path, pc, out_error) + end + if schema == C_NULL || out_error[] != READSTAT_OK + error("Error parsing schema $schema_path: $(readstat_error_message(out_error[]))") + end + retval = CAPI.readstat_parse_txt(parser, data_path, schema, pc) + finally + schema == C_NULL || CAPI.readstat_schema_free(schema) + readstat_parser_free(parser) + end + _finish_parse(pc, retval, data_path) + return build_table(pc) +end + """ read_meta(path; format=:auto, file_encoding=nothing, handler_encoding=nothing) -> ReadStatTable diff --git a/test/data/fixed.txt b/test/data/fixed.txt new file mode 100644 index 0000000..91fa727 --- /dev/null +++ b/test/data/fixed.txt @@ -0,0 +1,3 @@ + 1Alice 3.50 + 42Bob 12.25 +100Carol .75 diff --git a/test/data/schema.dct b/test/data/schema.dct new file mode 100644 index 0000000..b827199 --- /dev/null +++ b/test/data/schema.dct @@ -0,0 +1,5 @@ +infile dictionary { + _column(1) int id %3f "ID number" + _column(4) str8 name %8s "Name" + _column(12) double wage %6f "Wage" +} diff --git a/test/runtests.jl b/test/runtests.jl index a73e008..c1e9078 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,6 +10,7 @@ include("test_io.jl") include("test_threads.jl") include("test_source.jl") include("test_write.jl") +include("test_schema.jl") include("test_deprecated.jl") @run_package_tests diff --git a/test/test_schema.jl b/test/test_schema.jl new file mode 100644 index 0000000..74ae672 --- /dev/null +++ b/test/test_schema.jl @@ -0,0 +1,27 @@ +@testitem "fixed-width text via schema files" begin + using DataValues + + dir = joinpath(@__DIR__, "data") + data = joinpath(dir, "fixed.txt") + dct = joinpath(dir, "schema.dct") + + tbl = read_txt(data, dct; schema_format=:stata_dictionary) + @test size(tbl) == (3, 3) + @test names(tbl) == [:id, :name, :wage] + @test tbl[:id] == DataValueArray{Int16}([1, 42, 100]) + @test [get(x) for x in tbl[:name]] == ["Alice", "Bob", "Carol"] + @test tbl[:wage] == DataValueArray{Float64}([3.5, 12.25, 0.75]) + @test varmetadata(tbl, :id).label == "ID number" + + # The reader kwargs push down as usual. + part = read_txt(data, dct; schema_format=:stata_dictionary, + usecols=[:name], row_limit=2, row_offset=1) + @test size(part) == (2, 1) + @test [get(x) for x in part[:name]] == ["Bob", "Carol"] + + @test_throws ArgumentError read_txt(data, dct; schema_format=:nonsense) + @test_throws ArgumentError read_txt("no_such.txt", dct; + schema_format=:stata_dictionary) + # A data file is not a valid schema file. + @test_throws Exception read_txt(data, data; schema_format=:stata_dictionary) +end From aeb0e15d997316798b5941e65572d750535730e2 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Mon, 31 Aug 2026 23:51:25 -0700 Subject: [PATCH 12/14] Overhaul README, NEWS, and documentation for 2.0 The README walks through reading (kwargs, labels, date/time, missing semantics), lazy sources with streaming chunks, writing, and 1.x migration; NEWS gets the full 2.0.0 release notes; and the documentation gains an introduction plus an auto-generated API reference over the now fully docstringed public surface. Co-Authored-By: Claude Fable 5 --- NEWS.md | 43 ++++++++++++++++++++++ README.md | 90 +++++++++++++++++++++++++++++++++++++++------ docs/src/index.md | 43 ++++++++++++++++++++++ test/data/README.md | 5 +++ 4 files changed, 169 insertions(+), 12 deletions(-) diff --git a/NEWS.md b/NEWS.md index e1ed876..85ce76d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,46 @@ +# ReadStat.jl v2.0.0 Release Notes + +Breaking release. Highlights: + +* The readers return a `ReadStatTable`: concretely typed `DataValueVector` + columns by name or index, plus complete file-level (`filemetadata`) and + per-variable (`varmetadata`) metadata — creation/modified time, format + version, compression, endianness, encoding, notes, frequency weight, + variable labels, display formats, measures, alignments, and all value + labels. `ReadStatDataFrame` remains as a deprecated alias with `getproperty` + shims for the 1.x fields. +* Reader keyword arguments: `usecols` (column projection inside the C + parse), `row_limit`/`row_offset`, `ntasks` (parallel chunked parsing into + shared buffers), `file_encoding`/`handler_encoding`, `user_missing`, + `convert_datetime`, `apply_value_labels`, `catalog`, and `progress` + (cancellable). New entry points: `readstat` (extension dispatch), + `read_meta` (metadata only), `read_sas7bcat` (SAS value-label catalogs), + and `read_txt` (fixed-width text via SAS/SPSS/Stata schema files). Every + reader also accepts an `IO`. +* Columns with date/time display formats decode to `Date`/`DateTime`/`HMS` + by default (Stata, SAS, and SPSS format tables and epochs); `HMS` carries + times of day and durations beyond 24 hours. +* Value labels: `valuelabels`, and the lazy `LabeledArray`/`LabeledValue` + view (via `labeled` or `apply_value_labels=true`) that displays labels but + computes on the raw codes. Tagged missing values keep their tags + (`missingtags`), including labeled tags; SPSS user-defined missing values + can be kept as data (`user_missing=:keep`) and their rules are always in + `varmetadata(tbl, col).missing_ranges`. +* `ReadStatSource`: a lazy handle with `schema`/`colnames`/`coltypes`/ + `nrows`/`supports`, pushdown-capable `read(src; usecols, rows)`, and + single-pass streaming `chunks`. +* Write support for all formats: `write_dta`, `write_sav`, `write_por`, + `write_sas7bdat`, `write_xport`, and `write_sas7bcat`, from columns or a + `ReadStatTable`, covering value labels, tagged and user-defined missing + values, notes, compression (`.zsav`), and date/time re-encoding; plus the + public low-level `ReadStat.Writer` (one-to-one with the C writer API, + including Stata strL) and the complete raw C bindings in `ReadStat.CAPI`. +* Fixes: the `readstat_value_t` ABI on Windows is now verified behaviorally + against the C library; cells the parser never delivers are NA over defined + storage instead of undefined memory; parsers are freed even when a handler + throws. +* Requires Julia 1.12. + # ReadStat.jl v1.1.0 Release Notes * Add support for SAS XPORT diff --git a/README.md b/README.md index 49c10e5..46c618f 100644 --- a/README.md +++ b/README.md @@ -6,31 +6,97 @@ ## Overview -ReadStat.jl: Read files from Stata, SPSS, and SAS --- +ReadStat.jl reads and writes the data file formats of Stata (`.dta`), SPSS +(`.sav`, `.zsav`, `.por`), and SAS (`.sas7bdat`, `.xpt`, `.sas7bcat` value +catalogs), plus fixed-width text files described by schema files, using the +[ReadStat](https://github.com/WizardMac/ReadStat) C library. Missing data is +represented with [DataValues.jl](https://github.com/queryverse/DataValues.jl). -The ReadStat.jl Julia package uses the [ReadStat](https://github.com/WizardMac/ReadStat) C library to parse binary and transport files from Stata, SPSS and SAS. All functions return a `ReadStatDataFrame` whose fields hold the various informations contained in the passed file (column names, column data, labels, formats...). +For integration with packages like +[DataFrames.jl](https://github.com/JuliaData/DataFrames.jl) you should use +the [StatFiles.jl](https://github.com/queryverse/StatFiles.jl) package. -For integration with packages like [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl) you should use the [StatFiles.jl](https://github.com/queryverse/StatFiles.jl) package. - -## Usage: +## Reading ```julia using ReadStat -read_dta("/path/to/something.dta") +tbl = read_dta("data.dta") # also read_sav, read_por, read_sas7bdat, read_xport +tbl = readstat("data.sav") # infer the format from the extension +tbl = read_dta(io) # any IO holding the file works too + +tbl[:price] # columns by name or index (DataValueVectors) +names(tbl), size(tbl) +filemetadata(tbl) # file label, timestamps, notes, encoding, ... +varmetadata(tbl, :price) # variable label, display format, type, ... +``` + +Reads can push work into the C parser: + +```julia +read_dta("data.dta"; usecols = [:price, :mpg]) # column projection +read_dta("data.dta"; row_offset = 1000, row_limit = 500) +read_dta("data.dta"; ntasks = 8) # parallel chunked parsing +read_meta("data.dta") # metadata only, zero rows +``` + +Columns with date/time display formats decode to `Date`, `DateTime`, or +`HMS` (a time-of-day/duration type with unbounded hours) automatically; +pass `convert_datetime=false` for the raw numbers. -read_por("/path/to/something.por") +Value labels are always parsed (`valuelabels(tbl, :rep77)`), and the +`labeled` view or `apply_value_labels=true` wraps labeled columns in a +`LabeledArray` whose elements display as their labels but compute as their +raw codes. SAS value labels live in catalog files: +`read_sas7bdat("f.sas7bdat"; catalog="formats.sas7bcat")`. -read_sav("/path/to/something.sav") +Tagged missing values (Stata/SAS `.a`-`.z`) read as NA with their tags +available via `missingtags(tbl, col)`; SPSS user-defined missing values +collapse to NA by default or stay data with `user_missing=:keep`. -read_sas7bdat("/path/to/something.sas7bdat") +## Lazy sources and streaming -read_xport("path/to/something.xpt") +`ReadStatSource` is a cheap handle for consumers that plan before they read: + +```julia +src = ReadStatSource("big.dta") +schema(src); colnames(src); coltypes(src); nrows(src) +read(src; usecols = [:price], rows = 1_000:2_000) +for chunk in chunks(src; chunksize = 100_000) # single-pass streaming + # chunk is a complete ReadStatTable +end ``` +## Writing + +```julia +using DataValues + +write_dta("out.dta", + Any[DataValueArray([1, 2, 3]), DataValueArray(["a", "b", "c"])], + [:id, :tag]; + labels = ["identifier", "a string"], file_label = "my data") + +write_sav("out.sav", tbl) # write a ReadStatTable, metadata included +``` + +`write_dta`, `write_sav`, `write_por`, `write_sas7bdat`, `write_xport`, and +`write_sas7bcat` cover value labels, tagged and user-defined missing values, +notes, compression (including `.zsav`), and date/time re-encoding. The +unexported low-level `ReadStat.Writer` follows the C writer API one-to-one +(including Stata strL columns), and `ReadStat.CAPI` exposes the raw C +bindings for anything else. + +## Migrating from 1.x + +The readers now return a `ReadStatTable`. The old `ReadStatDataFrame` name +and its field access (`df.data`, `df.headers`, ...) keep working with +deprecation warnings; switch to indexing and the metadata accessors. Date +and time columns are now decoded by default (`convert_datetime=false` +restores raw numbers), and cells the parser never delivers are NA instead +of undefined memory. + ## Installation -To install the package, run the following: ```julia Pkg.add("ReadStat") diff --git a/docs/src/index.md b/docs/src/index.md index e10b99d..9711342 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1 +1,44 @@ # Introduction + +ReadStat.jl reads and writes the data file formats of Stata (`.dta`), SPSS +(`.sav`, `.zsav`, `.por`), and SAS (`.sas7bdat`, `.xpt`, and `.sas7bcat` +value-label catalogs), plus fixed-width text files described by schema +files, using the [ReadStat](https://github.com/WizardMac/ReadStat) C +library. Missing data is represented with +[DataValues.jl](https://github.com/queryverse/DataValues.jl). + +## Reading + +```julia +using ReadStat + +tbl = read_dta("data.dta") # read_sav, read_por, read_sas7bdat, read_xport +tbl = readstat("data.sav") # dispatch on the file extension +``` + +The result is a [`ReadStatTable`](@ref): columns by name (`tbl[:price]`) or +index, [`filemetadata`](@ref) and [`varmetadata`](@ref) for everything the +file records, [`valuelabels`](@ref) and [`labeled`](@ref) for value labels, +and [`missingtags`](@ref) for tagged missing values. All readers accept the +same keyword arguments for column projection, row selection, parallel +parsing, encodings, missing-value semantics, and date/time conversion — see +[`read_dta`](@ref). + +For metadata without data, use [`read_meta`](@ref). For planning and +streaming — schema first, then a projected read or a single-pass chunked +scan — use [`ReadStatSource`](@ref) with `read` and [`chunks`](@ref). + +## Writing + +[`write_dta`](@ref), `write_sav`, `write_por`, `write_sas7bdat`, +`write_xport`, and [`write_sas7bcat`](@ref) write columns or a whole +`ReadStatTable`, including value labels, missing-value declarations, notes, +compression, and date/time re-encoding. The public but unexported +[`ReadStat.Writer`](@ref) mirrors the C writer API one-to-one, and +`ReadStat.CAPI` holds the raw C bindings. + +# API Reference + +```@autodocs +Modules = [ReadStat] +``` diff --git a/test/data/README.md b/test/data/README.md index 624cb60..823126e 100644 --- a/test/data/README.md +++ b/test/data/README.md @@ -16,6 +16,11 @@ Sourced fixtures (verified against the upstream git blob SHAs when copied): Apache-2.0 license). The two CSV files hold the expected values of `sample_missing.sav` with user-defined missing values collapsed to missing and kept as data, respectively. +- `test_data_linux.sas7bdat`, `test_formats_linux.sas7bcat`, and + `sas_formatted.csv` (the expected labeled values) are copied from + pyreadstat's `test_data/sas_catalog/`. +- `schema.dct` and `fixed.txt` are hand-written fixtures for the + fixed-width-text schema path. The `types.{dta,sav,sas7bdat,xpt,por}` fixtures in the parent directory are the original ReadStat.jl fixtures (`types.por` was copied from From ac72a868ccea1f2d55824ea04efbcce3d67e8bee Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 1 Sep 2026 08:28:23 -0700 Subject: [PATCH 13/14] Support Julia 1.10 and fix the 32-bit unix C ABI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2.0 rewrite turned out to use only one post-1.10 language feature: the `public` keyword. It is now declared via eval on Julia 1.11+ only, and the compat floor returns to 1.10 (matching the rest of the Queryverse); the full test suite passes on 1.10.12. On 32-bit unix, glibc''s off_t and time_t are 32-bit (the jll build uses no large-file/time64 opt-ins), but the wrappers hardcoded Int64: the seek callback''s argument stack shifted, handing it a garbage io context — the linux-x86 CI segfaults in the IO, source/chunks, and writer round-trip tests. readstat_off_t and time_t now size per platform (64-bit on Windows, which uses _off64_t and 64-bit time_t throughout, hence the green windows-x86 leg). Also adds the docstrings whose absence broke the documentation build''s cross-references (ValueLabelDict and the low-level writer''s row and file-metadata functions); the docs now build cleanly. Co-Authored-By: Claude Fable 5 --- NEWS.md | 2 +- Project.toml | 2 +- docs/Project.toml | 1 + src/ReadStat.jl | 19 +++++----- src/capi/parser.jl | 4 +- src/capi/value.jl | 10 ++++- src/capi/writer.jl | 2 +- src/reader/io.jl | 15 +++++--- src/reader/metadata.jl | 11 ++++-- src/writer/writer.jl | 86 ++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 128 insertions(+), 24 deletions(-) diff --git a/NEWS.md b/NEWS.md index 85ce76d..3b84761 100644 --- a/NEWS.md +++ b/NEWS.md @@ -39,7 +39,7 @@ Breaking release. Highlights: against the C library; cells the parser never delivers are NA over defined storage instead of undefined memory; parsers are freed even when a handler throws. -* Requires Julia 1.12. +* Requires Julia 1.10. # ReadStat.jl v1.1.0 Release Notes * Add support for SAS XPORT diff --git a/Project.toml b/Project.toml index 32b4519..3ee1871 100644 --- a/Project.toml +++ b/Project.toml @@ -12,7 +12,7 @@ TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] -julia = "1.12" +julia = "1.10" DataValues = "0.4.13, 0.5, 1" ReadStat_jll = "1.1.9" diff --git a/docs/Project.toml b/docs/Project.toml index 1814eb3..67a8ffd 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,5 +1,6 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +ReadStat = "d71aba96-b539-5138-91ee-935c3ee1374c" [compat] Documenter = "1" diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 2d204ef..498e413 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -13,15 +13,16 @@ export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, write_dta, write_sav, write_por, write_sas7bdat, write_xport, write_sas7bcat, read_txt -public CAPI - -# The low-level writer layer: public, not exported. -public Writer, WriterVariable, LabelSet, StringRef, - add_label_set!, label!, add_variable!, add_note!, add_string_ref!, - file_label!, timestamp!, fweight!, format_version!, table_name!, is_64bit!, - compression!, begin_writing!, validate_metadata, validate_variable, - begin_row!, end_row!, insert_value!, insert_missing!, insert_tagged_missing!, - insert_string_ref!, end_writing! +# The CAPI submodule and the low-level writer layer are public but not +# exported. The `public` keyword only exists on Julia 1.11+, so it is +# declared via eval to keep Julia 1.10 parsing this file. +VERSION >= v"1.11.0-DEV" && eval(Meta.parse( + "public CAPI, Writer, WriterVariable, LabelSet, StringRef, " * + "add_label_set!, label!, add_variable!, add_note!, add_string_ref!, " * + "file_label!, timestamp!, fweight!, format_version!, table_name!, is_64bit!, " * + "compression!, begin_writing!, validate_metadata, validate_variable, " * + "begin_row!, end_row!, insert_value!, insert_missing!, insert_tagged_missing!, " * + "insert_string_ref!, end_writing!")) """ ReadStat.CAPI diff --git a/src/capi/parser.jl b/src/capi/parser.jl index 1aea736..c6aa750 100644 --- a/src/capi/parser.jl +++ b/src/capi/parser.jl @@ -100,11 +100,11 @@ function readstat_get_var_count(metadata::MetadataPtr) end function readstat_get_creation_time(metadata::MetadataPtr) - @ccall libreadstat.readstat_get_creation_time(metadata::MetadataPtr)::Int64 + Int64(@ccall libreadstat.readstat_get_creation_time(metadata::MetadataPtr)::Ctime_t) end function readstat_get_modified_time(metadata::MetadataPtr) - @ccall libreadstat.readstat_get_modified_time(metadata::MetadataPtr)::Int64 + Int64(@ccall libreadstat.readstat_get_modified_time(metadata::MetadataPtr)::Ctime_t) end function readstat_get_file_format_version(metadata::MetadataPtr) diff --git a/src/capi/value.jl b/src/capi/value.jl index 90493be..bc9399e 100644 --- a/src/capi/value.jl +++ b/src/capi/value.jl @@ -4,7 +4,7 @@ # and returns it by value, so an opaque pointer is not an option. Every other # C struct is handled through an opaque pointer plus the C getter functions. -export ReadStatValue, +export ReadStatValue, Coff_t, Ctime_t, ParserPtr, MetadataPtr, VariablePtr, LabelSetPtr, SchemaPtr, WriterPtr, StringRefPtr, readstat_value_type, readstat_value_type_class, readstat_type_class, readstat_value_is_missing, readstat_value_is_system_missing, @@ -12,6 +12,14 @@ export ReadStatValue, readstat_value_tag, readstat_int8_value, readstat_int16_value, readstat_int32_value, readstat_float_value, readstat_double_value, readstat_string_value +# readstat_off_t and time_t as the C library was compiled: 64-bit on Windows +# (_off64_t, 64-bit time_t) and on 64-bit unix, but 32-bit on 32-bit unix, +# where glibc's off_t/time_t stay 32-bit without large-file/time64 opt-ins +# (which the jll build does not use). Getting these wrong shifts the +# argument stack of the seek callback on 32-bit Linux. +const Coff_t = (Sys.iswindows() || Sys.WORD_SIZE == 64) ? Int64 : Int32 +const Ctime_t = (Sys.iswindows() || Sys.WORD_SIZE == 64) ? Int64 : Int32 + # Opaque C struct tags; only ever used as Ptr{...} type parameters. abstract type readstat_parser_s end abstract type readstat_metadata_s end diff --git a/src/capi/writer.jl b/src/capi/writer.jl index ef62b0b..b1bf57f 100644 --- a/src/capi/writer.jl +++ b/src/capi/writer.jl @@ -144,7 +144,7 @@ end function readstat_writer_set_file_timestamp(writer::WriterPtr, timestamp::Integer) @ccall libreadstat.readstat_writer_set_file_timestamp(writer::WriterPtr, - timestamp::Int64)::ReadStatError + timestamp::Ctime_t)::ReadStatError end function readstat_writer_set_fweight_variable(writer::WriterPtr, variable::VariablePtr) diff --git a/src/reader/io.jl b/src/reader/io.jl index 4463f41..92c742c 100644 --- a/src/reader/io.jl +++ b/src/reader/io.jl @@ -42,18 +42,21 @@ end handle_io_open(path::Cstring, io_ctx::Ptr{Cvoid})::Cint = Cint(0) handle_io_close(io_ctx::Ptr{Cvoid})::Cint = Cint(0) -function handle_io_seek(offset::Int64, whence::ReadStatIOFlags, io_ctx::Ptr{Cvoid})::Int64 +# The offset and return type must match readstat_off_t as the C library was +# compiled (32-bit on 32-bit unix); a width mismatch shifts the callback's +# argument stack there. +function handle_io_seek(offset::Coff_t, whence::ReadStatIOFlags, io_ctx::Ptr{Cvoid})::Coff_t src = unsafe_pointer_to_objref(io_ctx)::IOSource try - pos = whence == READSTAT_SEEK_SET ? offset : + pos = whence == READSTAT_SEEK_SET ? Int64(offset) : whence == READSTAT_SEEK_CUR ? Int64(position(src.io)) + offset : src.size + offset - (pos < 0 || pos > src.size) && return Int64(-1) + (pos < 0 || pos > src.size) && return Coff_t(-1) seek(src.io, pos) - return Int64(position(src.io)) + return Coff_t(position(src.io)) catch e src.err = (e, catch_backtrace()) - return Int64(-1) + return Coff_t(-1) end end @@ -79,7 +82,7 @@ const CF_IO_READ = Ref(C_NULL) function _init_io_cfunctions() CF_IO_OPEN[] = @cfunction(handle_io_open, Cint, (Cstring, Ptr{Cvoid})) CF_IO_CLOSE[] = @cfunction(handle_io_close, Cint, (Ptr{Cvoid},)) - CF_IO_SEEK[] = @cfunction(handle_io_seek, Int64, (Int64, ReadStatIOFlags, Ptr{Cvoid})) + CF_IO_SEEK[] = @cfunction(handle_io_seek, Coff_t, (Coff_t, ReadStatIOFlags, Ptr{Cvoid})) CF_IO_READ[] = @cfunction(handle_io_read, Cssize_t, (Ptr{Cvoid}, Csize_t, Ptr{Cvoid})) return nothing end diff --git a/src/reader/metadata.jl b/src/reader/metadata.jl index 6ef5fe3..e0fdd9d 100644 --- a/src/reader/metadata.jl +++ b/src/reader/metadata.jl @@ -1,8 +1,13 @@ # File-level and per-variable metadata captured during a parse. -# One value-label set: maps a raw value to its display label. Keys are -# normalized to Int32 (all integer types), Float64 (all floats), String, or — -# for labeled tagged missing values like Stata's `.a` — the tag Char. +""" + ValueLabelDict + +One value-label set: a dictionary mapping raw values to their display +labels. Keys are normalized to `Int32` (all integer types), `Float64` (all +floats), `String`, or — for labeled tagged missing values like Stata's +`.a` — the tag `Char`. +""" const ValueLabelDict = Dict{Union{Char,Int32,Float64,String},String} """ diff --git a/src/writer/writer.jl b/src/writer/writer.jl index 0def980..d89ab8d 100644 --- a/src/writer/writer.jl +++ b/src/writer/writer.jl @@ -188,17 +188,53 @@ across columns and rows. """ add_string_ref!(w::Writer, s::AbstractString) = StringRef(readstat_add_string_ref(_ptr(w), s)) +""" + file_label!(w::Writer, s) + +Set the file label. +""" file_label!(w::Writer, s::AbstractString) = _writer_check(w, readstat_writer_set_file_label(_ptr(w), s)) + +""" + timestamp!(w::Writer, t::DateTime) + +Set the file timestamp. +""" timestamp!(w::Writer, t::DateTime) = _writer_check(w, readstat_writer_set_file_timestamp(_ptr(w), round(Int64, Dates.datetime2unix(t)))) + +""" + fweight!(w::Writer, var::WriterVariable) + +Mark a variable as the frequency weight. +""" fweight!(w::Writer, var::WriterVariable) = _writer_check(w, readstat_writer_set_fweight_variable(_ptr(w), var.ptr)) + +""" + format_version!(w::Writer, v) + +Set the file format version (e.g. 104-119 for `.dta`, 5 or 8 for XPORT, +2 or 3 for `.sav`). +""" format_version!(w::Writer, v::Integer) = _writer_check(w, readstat_writer_set_file_format_version(_ptr(w), v)) + +""" + table_name!(w::Writer, s) + +Set the table name (used by XPORT files; defaults to `DATASET`). +""" table_name!(w::Writer, s::AbstractString) = _writer_check(w, readstat_writer_set_table_name(_ptr(w), s)) + +""" + is_64bit!(w::Writer, b::Bool) + +Set whether a SAS file is written in 64-bit layout (default true). +""" is_64bit!(w::Writer, b::Bool) = _writer_check(w, readstat_writer_set_file_format_is_64bit(_ptr(w), b)) @@ -247,13 +283,44 @@ function begin_writing!(w::Writer, format::Symbol, row_count::Integer=0) return w end +""" + validate_metadata(w::Writer) + +Run the format's file-level validation (call after [`begin_writing!`](@ref)). +""" validate_metadata(w::Writer) = _writer_check(w, readstat_validate_metadata(_ptr(w))) + +""" + validate_variable(w::Writer, var::WriterVariable) + +Run the format's per-variable validation (call after [`begin_writing!`](@ref)). +""" validate_variable(w::Writer, var::WriterVariable) = _writer_check(w, readstat_validate_variable(_ptr(w), var.ptr)) +""" + begin_row!(w::Writer) + +Start one observation; insert a value for every variable, then call +[`end_row!`](@ref). +""" begin_row!(w::Writer) = _writer_check(w, readstat_begin_row(_ptr(w))) + +""" + end_row!(w::Writer) + +Finish the current observation. +""" end_row!(w::Writer) = _writer_check(w, readstat_end_row(_ptr(w))) +""" + insert_value!(w::Writer, var::WriterVariable, v) + +Insert one cell into the current row: an `Int8`/`Int16`/`Int32`/`Float32`/ +`Float64` or a string, matching the variable's storage type. See also +[`insert_missing!`](@ref), [`insert_tagged_missing!`](@ref), and +[`insert_string_ref!`](@ref). +""" insert_value!(w::Writer, var::WriterVariable, v::Int8) = _writer_check(w, readstat_insert_int8_value(_ptr(w), var.ptr, v)) insert_value!(w::Writer, var::WriterVariable, v::Int16) = @@ -266,10 +333,29 @@ insert_value!(w::Writer, var::WriterVariable, v::Float64) = _writer_check(w, readstat_insert_double_value(_ptr(w), var.ptr, v)) insert_value!(w::Writer, var::WriterVariable, v::AbstractString) = _writer_check(w, readstat_insert_string_value(_ptr(w), var.ptr, v)) + +""" + insert_missing!(w::Writer, var::WriterVariable) + +Insert a (system) missing cell into the current row. +""" insert_missing!(w::Writer, var::WriterVariable) = _writer_check(w, readstat_insert_missing_value(_ptr(w), var.ptr)) + +""" + insert_tagged_missing!(w::Writer, var::WriterVariable, tag::Char) + +Insert a tagged missing cell (`'a'`-`'z'`; Stata and SAS formats only). +""" insert_tagged_missing!(w::Writer, var::WriterVariable, tag::Char) = _writer_check(w, readstat_insert_tagged_missing_value(_ptr(w), var.ptr, tag)) + +""" + insert_string_ref!(w::Writer, var::WriterVariable, ref::StringRef) + +Insert an interned string (from [`add_string_ref!`](@ref)) into a +`READSTAT_TYPE_STRING_REF` (Stata strL) column. +""" insert_string_ref!(w::Writer, var::WriterVariable, ref::StringRef) = _writer_check(w, readstat_insert_string_ref(_ptr(w), var.ptr, ref.ptr)) From 15764cc4ecf357c2ab600eb7f234e55eae2857da Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 1 Sep 2026 09:40:04 -0700 Subject: [PATCH 14/14] Size time_t by word size: 32-bit Windows also uses 32-bit time_t The windows-x86 legs on Julia 1.10/1.11 showed timestamps with garbage in the high half: this MinGW build has 32-bit time_t on i686 (the 1.12-x86 passes were luck in what the second register held). readstat_off_t stays 64-bit on all of Windows (readstat.h typedefs it to _off64_t there, and the IO tests pass on every x86 leg), so only Ctime_t changes to follow the word size on every platform. Co-Authored-By: Claude Fable 5 --- src/capi/value.jl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/capi/value.jl b/src/capi/value.jl index bc9399e..9cbc3e7 100644 --- a/src/capi/value.jl +++ b/src/capi/value.jl @@ -12,13 +12,16 @@ export ReadStatValue, Coff_t, Ctime_t, readstat_value_tag, readstat_int8_value, readstat_int16_value, readstat_int32_value, readstat_float_value, readstat_double_value, readstat_string_value -# readstat_off_t and time_t as the C library was compiled: 64-bit on Windows -# (_off64_t, 64-bit time_t) and on 64-bit unix, but 32-bit on 32-bit unix, -# where glibc's off_t/time_t stay 32-bit without large-file/time64 opt-ins -# (which the jll build does not use). Getting these wrong shifts the -# argument stack of the seek callback on 32-bit Linux. +# readstat_off_t and time_t as the C library was compiled. readstat.h +# typedefs readstat_off_t to _off64_t on Windows, so it is 64-bit on both +# Windows architectures; elsewhere it is off_t, which is 32-bit on 32-bit +# unix (no large-file opt-in in the jll build). time_t follows the word +# size everywhere: 32-bit on i686 for both MinGW (no _USE_32BIT_TIME_T +# opt-out) and glibc (no time64 opt-in). Getting these wrong shifts the +# seek callback's argument stack (segfaults) or fills the high half of +# timestamps with register garbage. const Coff_t = (Sys.iswindows() || Sys.WORD_SIZE == 64) ? Int64 : Int32 -const Ctime_t = (Sys.iswindows() || Sys.WORD_SIZE == 64) ? Int64 : Int32 +const Ctime_t = Sys.WORD_SIZE == 64 ? Int64 : Int32 # Opaque C struct tags; only ever used as Ptr{...} type parameters. abstract type readstat_parser_s end