diff --git a/NEWS.md b/NEWS.md index e1ed876..3b84761 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,46 @@ +# ReadStat.jl v2.0.0 Release Notes + +Breaking release. Highlights: + +* The readers return a `ReadStatTable`: concretely typed `DataValueVector` + columns by name or index, plus complete file-level (`filemetadata`) and + per-variable (`varmetadata`) metadata — creation/modified time, format + version, compression, endianness, encoding, notes, frequency weight, + variable labels, display formats, measures, alignments, and all value + labels. `ReadStatDataFrame` remains as a deprecated alias with `getproperty` + shims for the 1.x fields. +* Reader keyword arguments: `usecols` (column projection inside the C + parse), `row_limit`/`row_offset`, `ntasks` (parallel chunked parsing into + shared buffers), `file_encoding`/`handler_encoding`, `user_missing`, + `convert_datetime`, `apply_value_labels`, `catalog`, and `progress` + (cancellable). New entry points: `readstat` (extension dispatch), + `read_meta` (metadata only), `read_sas7bcat` (SAS value-label catalogs), + and `read_txt` (fixed-width text via SAS/SPSS/Stata schema files). Every + reader also accepts an `IO`. +* Columns with date/time display formats decode to `Date`/`DateTime`/`HMS` + by default (Stata, SAS, and SPSS format tables and epochs); `HMS` carries + times of day and durations beyond 24 hours. +* Value labels: `valuelabels`, and the lazy `LabeledArray`/`LabeledValue` + view (via `labeled` or `apply_value_labels=true`) that displays labels but + computes on the raw codes. Tagged missing values keep their tags + (`missingtags`), including labeled tags; SPSS user-defined missing values + can be kept as data (`user_missing=:keep`) and their rules are always in + `varmetadata(tbl, col).missing_ranges`. +* `ReadStatSource`: a lazy handle with `schema`/`colnames`/`coltypes`/ + `nrows`/`supports`, pushdown-capable `read(src; usecols, rows)`, and + single-pass streaming `chunks`. +* Write support for all formats: `write_dta`, `write_sav`, `write_por`, + `write_sas7bdat`, `write_xport`, and `write_sas7bcat`, from columns or a + `ReadStatTable`, covering value labels, tagged and user-defined missing + values, notes, compression (`.zsav`), and date/time re-encoding; plus the + public low-level `ReadStat.Writer` (one-to-one with the C writer API, + including Stata strL) and the complete raw C bindings in `ReadStat.CAPI`. +* Fixes: the `readstat_value_t` ABI on Windows is now verified behaviorally + against the C library; cells the parser never delivers are NA over defined + storage instead of undefined memory; parsers are freed even when a handler + throws. +* Requires Julia 1.10. + # ReadStat.jl v1.1.0 Release Notes * Add support for SAS XPORT diff --git a/Project.toml b/Project.toml index 1891f22..3ee1871 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "ReadStat" uuid = "d71aba96-b539-5138-91ee-935c3ee1374c" -version = "1.2.1-DEV" +version = "2.0.0-DEV" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/README.md b/README.md index 49c10e5..46c618f 100644 --- a/README.md +++ b/README.md @@ -6,31 +6,97 @@ ## Overview -ReadStat.jl: Read files from Stata, SPSS, and SAS --- +ReadStat.jl reads and writes the data file formats of Stata (`.dta`), SPSS +(`.sav`, `.zsav`, `.por`), and SAS (`.sas7bdat`, `.xpt`, `.sas7bcat` value +catalogs), plus fixed-width text files described by schema files, using the +[ReadStat](https://github.com/WizardMac/ReadStat) C library. Missing data is +represented with [DataValues.jl](https://github.com/queryverse/DataValues.jl). -The ReadStat.jl Julia package uses the [ReadStat](https://github.com/WizardMac/ReadStat) C library to parse binary and transport files from Stata, SPSS and SAS. All functions return a `ReadStatDataFrame` whose fields hold the various informations contained in the passed file (column names, column data, labels, formats...). +For integration with packages like +[DataFrames.jl](https://github.com/JuliaData/DataFrames.jl) you should use +the [StatFiles.jl](https://github.com/queryverse/StatFiles.jl) package. -For integration with packages like [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl) you should use the [StatFiles.jl](https://github.com/queryverse/StatFiles.jl) package. - -## Usage: +## Reading ```julia using ReadStat -read_dta("/path/to/something.dta") +tbl = read_dta("data.dta") # also read_sav, read_por, read_sas7bdat, read_xport +tbl = readstat("data.sav") # infer the format from the extension +tbl = read_dta(io) # any IO holding the file works too + +tbl[:price] # columns by name or index (DataValueVectors) +names(tbl), size(tbl) +filemetadata(tbl) # file label, timestamps, notes, encoding, ... +varmetadata(tbl, :price) # variable label, display format, type, ... +``` + +Reads can push work into the C parser: + +```julia +read_dta("data.dta"; usecols = [:price, :mpg]) # column projection +read_dta("data.dta"; row_offset = 1000, row_limit = 500) +read_dta("data.dta"; ntasks = 8) # parallel chunked parsing +read_meta("data.dta") # metadata only, zero rows +``` + +Columns with date/time display formats decode to `Date`, `DateTime`, or +`HMS` (a time-of-day/duration type with unbounded hours) automatically; +pass `convert_datetime=false` for the raw numbers. -read_por("/path/to/something.por") +Value labels are always parsed (`valuelabels(tbl, :rep77)`), and the +`labeled` view or `apply_value_labels=true` wraps labeled columns in a +`LabeledArray` whose elements display as their labels but compute as their +raw codes. SAS value labels live in catalog files: +`read_sas7bdat("f.sas7bdat"; catalog="formats.sas7bcat")`. -read_sav("/path/to/something.sav") +Tagged missing values (Stata/SAS `.a`-`.z`) read as NA with their tags +available via `missingtags(tbl, col)`; SPSS user-defined missing values +collapse to NA by default or stay data with `user_missing=:keep`. -read_sas7bdat("/path/to/something.sas7bdat") +## Lazy sources and streaming -read_xport("path/to/something.xpt") +`ReadStatSource` is a cheap handle for consumers that plan before they read: + +```julia +src = ReadStatSource("big.dta") +schema(src); colnames(src); coltypes(src); nrows(src) +read(src; usecols = [:price], rows = 1_000:2_000) +for chunk in chunks(src; chunksize = 100_000) # single-pass streaming + # chunk is a complete ReadStatTable +end ``` +## Writing + +```julia +using DataValues + +write_dta("out.dta", + Any[DataValueArray([1, 2, 3]), DataValueArray(["a", "b", "c"])], + [:id, :tag]; + labels = ["identifier", "a string"], file_label = "my data") + +write_sav("out.sav", tbl) # write a ReadStatTable, metadata included +``` + +`write_dta`, `write_sav`, `write_por`, `write_sas7bdat`, `write_xport`, and +`write_sas7bcat` cover value labels, tagged and user-defined missing values, +notes, compression (including `.zsav`), and date/time re-encoding. The +unexported low-level `ReadStat.Writer` follows the C writer API one-to-one +(including Stata strL columns), and `ReadStat.CAPI` exposes the raw C +bindings for anything else. + +## Migrating from 1.x + +The readers now return a `ReadStatTable`. The old `ReadStatDataFrame` name +and its field access (`df.data`, `df.headers`, ...) keep working with +deprecation warnings; switch to indexing and the metadata accessors. Date +and time columns are now decoded by default (`convert_datetime=false` +restores raw numbers), and cells the parser never delivers are NA instead +of undefined memory. + ## Installation -To install the package, run the following: ```julia Pkg.add("ReadStat") diff --git a/docs/Project.toml b/docs/Project.toml index 1814eb3..67a8ffd 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,5 +1,6 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +ReadStat = "d71aba96-b539-5138-91ee-935c3ee1374c" [compat] Documenter = "1" diff --git a/docs/src/index.md b/docs/src/index.md index e10b99d..9711342 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1 +1,44 @@ # Introduction + +ReadStat.jl reads and writes the data file formats of Stata (`.dta`), SPSS +(`.sav`, `.zsav`, `.por`), and SAS (`.sas7bdat`, `.xpt`, and `.sas7bcat` +value-label catalogs), plus fixed-width text files described by schema +files, using the [ReadStat](https://github.com/WizardMac/ReadStat) C +library. Missing data is represented with +[DataValues.jl](https://github.com/queryverse/DataValues.jl). + +## Reading + +```julia +using ReadStat + +tbl = read_dta("data.dta") # read_sav, read_por, read_sas7bdat, read_xport +tbl = readstat("data.sav") # dispatch on the file extension +``` + +The result is a [`ReadStatTable`](@ref): columns by name (`tbl[:price]`) or +index, [`filemetadata`](@ref) and [`varmetadata`](@ref) for everything the +file records, [`valuelabels`](@ref) and [`labeled`](@ref) for value labels, +and [`missingtags`](@ref) for tagged missing values. All readers accept the +same keyword arguments for column projection, row selection, parallel +parsing, encodings, missing-value semantics, and date/time conversion — see +[`read_dta`](@ref). + +For metadata without data, use [`read_meta`](@ref). For planning and +streaming — schema first, then a projected read or a single-pass chunked +scan — use [`ReadStatSource`](@ref) with `read` and [`chunks`](@ref). + +## Writing + +[`write_dta`](@ref), `write_sav`, `write_por`, `write_sas7bdat`, +`write_xport`, and [`write_sas7bcat`](@ref) write columns or a whole +`ReadStatTable`, including value labels, missing-value declarations, notes, +compression, and date/time re-encoding. The public but unexported +[`ReadStat.Writer`](@ref) mirrors the C writer API one-to-one, and +`ReadStat.CAPI` holds the raw C bindings. + +# API Reference + +```@autodocs +Modules = [ReadStat] +``` diff --git a/src/C_interface.jl b/src/C_interface.jl deleted file mode 100644 index eeede3a..0000000 --- a/src/C_interface.jl +++ /dev/null @@ -1,80 +0,0 @@ -function readstat_get_file_label(metadata::Ptr{Nothing}) - ptr = ccall((:readstat_get_file_label, libreadstat), Cstring, (Ptr{Nothing},), metadata) - return ptr == C_NULL ? "" : unsafe_string(ptr) -end - -function readstat_get_modified_time(metadata::Ptr{Nothing}) - return ccall((:readstat_get_modified_time, libreadstat), Int64, (Ptr{Nothing},), metadata) -end - -function readstat_get_file_format_version(metadata::Ptr{Nothing}) - return ccall((:readstat_get_file_format_version, libreadstat), Cint, (Ptr{Nothing},), metadata) -end - -function readstat_get_row_count(metadata::Ptr{Nothing}) - return ccall((:readstat_get_row_count, libreadstat), Cint, (Ptr{Nothing},), metadata) -end - -function readstat_get_var_count(metadata::Ptr{Nothing}) - return ccall((:readstat_get_var_count, libreadstat), Cint, (Ptr{Nothing},), metadata) -end - -function readstat_value_is_missing(value::ReadStatValue, variable::Ptr{Nothing}) - return Bool(ccall((:readstat_value_is_missing, libreadstat), Cint, (ReadStatValue,Ptr{Nothing}), value, variable)) -end - -function readstat_variable_get_index(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_index, libreadstat), Cint, (Ptr{Nothing},), variable) -end - -function readstat_variable_get_name(variable::Ptr{Nothing}) - return unsafe_string(ccall((:readstat_variable_get_name, libreadstat), Cstring, (Ptr{Nothing},), variable)) -end - -function readstat_variable_get_type(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_type, libreadstat), Cint, (Ptr{Nothing},), variable) -end - -function readstat_variable_get_storage_width(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_storage_width, libreadstat), Csize_t, (Ptr{Nothing},), variable) -end - -function readstat_variable_get_measure(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_measure, libreadstat), Cint, (Ptr{Nothing},), variable) -end - -function readstat_variable_get_alignment(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_alignment, libreadstat), Cint, (Ptr{Nothing},), variable) -end - -function readstat_parser_free(parser::Ptr{Nothing}) - return ccall((:readstat_parser_free, libreadstat), Nothing, (Ptr{Nothing},), parser) -end - -function readstat_value_type(val::Value) - return ccall((:readstat_value_type, libreadstat), Cint, (Value,), val) -end - -function readstat_parse(filename::String, type::Val{:dta}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_dta, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_parse(filename::String, type::Val{:sav}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_sav, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_parse(filename::String, type::Val{:por}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_por, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_parse(filename::String, type::Val{:sas7bdat}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_sas7bdat, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_parse(filename::String, type::Val{:xport}, parser::Ptr{Nothing}, ds::ReadStatDataFrame) - return ccall((:readstat_parse_xport, libreadstat), Cint, (Ptr{Nothing}, Cstring, Any), parser, string(filename), ds) -end - -function readstat_variable_get_missing_ranges_count(variable::Ptr{Nothing}) - return ccall((:readstat_variable_get_missing_ranges_count, libreadstat), Cint, (Ptr{Nothing},), variable) -end diff --git a/src/ReadStat.jl b/src/ReadStat.jl index 2d26bbf..498e413 100644 --- a/src/ReadStat.jl +++ b/src/ReadStat.jl @@ -1,275 +1,81 @@ module ReadStat -using ReadStat_jll - -############################################################################## -## -## Import -## -############################################################################## - -using DataValues: DataValueVector -import DataValues +using DataValues: DataValues, DataValueVector using Dates -export ReadStatDataFrame, read_dta, read_sav, read_por, read_sas7bdat, read_xport - -############################################################################## -## -## Julia types that mirror C types -## -############################################################################## - -const READSTAT_TYPE_STRING = Cint(0) -const READSTAT_TYPE_INT8 = Cint(1) -const READSTAT_TYPE_INT16 = Cint(2) -const READSTAT_TYPE_INT32 = Cint(3) -const READSTAT_TYPE_FLOAT = Cint(4) -const READSTAT_TYPE_DOUBLE = Cint(5) -const READSTAT_TYPE_STRING_REF = Cint(6) - -# Julia type for each readstat_type_t, indexed by the enum value plus one. -# STRING_REF is a reference into a string table, so it surfaces as a String -# just like STRING does. +export ReadStatTable, ReadStatMeta, ReadStatVarMeta, ReadStatDataFrame, + read_dta, read_sav, read_por, read_sas7bdat, read_xport, read_sas7bcat, + readstat, read_meta, + filemetadata, varmetadata, valuelabels, missingtags, + LabeledValue, LabeledArray, labeled, unwrap, valuelabel, rawvalues, getvaluelabels, + HMS, + ReadStatSource, ReadStatChunks, schema, colnames, coltypes, nrows, supports, chunks, + write_dta, write_sav, write_por, write_sas7bdat, write_xport, write_sas7bcat, + read_txt + +# The CAPI submodule and the low-level writer layer are public but not +# exported. The `public` keyword only exists on Julia 1.11+, so it is +# declared via eval to keep Julia 1.10 parsing this file. +VERSION >= v"1.11.0-DEV" && eval(Meta.parse( + "public CAPI, Writer, WriterVariable, LabelSet, StringRef, " * + "add_label_set!, label!, add_variable!, add_note!, add_string_ref!, " * + "file_label!, timestamp!, fweight!, format_version!, table_name!, is_64bit!, " * + "compression!, begin_writing!, validate_metadata, validate_variable, " * + "begin_row!, end_row!, insert_value!, insert_missing!, insert_tagged_missing!, " * + "insert_string_ref!, end_writing!")) + +""" + ReadStat.CAPI + +Thin wrappers around the public C API of the readstat library (readstat.h, +v1.1.9). Function names, argument order, and semantics follow the C header +one-to-one; C structs are handled as opaque pointers except `ReadStatValue`, +which the C API passes by value. Higher-level functionality lives in the +parent `ReadStat` module — reach for `CAPI` only when the high-level API does +not expose what you need. +""" +module CAPI + +using ReadStat_jll: libreadstat + +include("capi/enums.jl") +include("capi/value.jl") +include("capi/parser.jl") +include("capi/schema.jl") +include("capi/writer.jl") + +end # module CAPI + +using .CAPI + +# Julia type corresponding to each ReadStatType, indexed by the enum value +# plus one. STRING_REF is a reference into a string table, so it surfaces as +# a String just like STRING does. const READSTAT_TYPES = (String, Int8, Int16, Int32, Float32, Float64, String) -const READSTAT_ERROR_OPEN = Cint(1) -const READSTAT_ERROR_READ = Cint(2) -const READSTAT_ERROR_MALLOC = Cint(3) -const READSTAT_ERROR_USER_ABORT = Cint(4) -const READSTAT_ERROR_PARSE = Cint(5) - -############################################################################## -## -## Pure Julia types -## -############################################################################## - -struct ReadStatValue - union::Int64 - readstat_types_t::Cint - tag::Cchar - @static if Sys.iswindows() - bits::Cuint - else - bits::UInt8 - end -end - -const Value = ReadStatValue - -mutable struct ReadStatDataFrame - data::Vector{Any} - headers::Vector{Symbol} - types::Vector{DataType} - labels::Vector{String} - formats::Vector{String} - storagewidths::Vector{Csize_t} - measures::Vector{Cint} - alignments::Vector{Cint} - val_label_keys::Vector{String} - val_label_dict::Dict{String, Dict{Any,String}} - rows::Int - columns::Int - filelabel::String - timestamp::DateTime - format::Clong - types_as_int::Vector{Cint} - hasmissings::Vector{Bool} - - ReadStatDataFrame() = - new(Any[], Symbol[], DataType[], String[], String[], Csize_t[], Cint[], Cint[], - String[], Dict{String, Dict{Any,String}}(), 0, 0, "", Dates.unix2datetime(0), 0, Cint[], Bool[]) -end - -include("C_interface.jl") - -############################################################################## -## -## Julia functions -## -############################################################################## - -function handle_metadata!(metadata::Ptr{Nothing}, ds_ptr::Ptr{ReadStatDataFrame}) - ds = unsafe_pointer_to_objref(ds_ptr) - ds.filelabel = readstat_get_file_label(metadata) - ds.timestamp = Dates.unix2datetime(readstat_get_modified_time(metadata)) - ds.format = readstat_get_file_format_version(metadata) - ds.rows = readstat_get_row_count(metadata) - ds.columns = readstat_get_var_count(metadata) - return Cint(0) -end - -get_name(variable::Ptr{Nothing}) = Symbol(readstat_variable_get_name(variable)) - -function get_label(var::Ptr{Nothing}) - ptr = ccall((:readstat_variable_get_label, libreadstat), Cstring, (Ptr{Nothing},), var) - ptr == C_NULL ? "" : unsafe_string(ptr) -end - -function get_format(var::Ptr{Nothing}) - ptr = ccall((:readstat_variable_get_format, libreadstat), Cstring, (Ptr{Nothing},), var) - ptr == C_NULL ? "" : unsafe_string(ptr) -end - -get_type(data_type::Cint) = READSTAT_TYPES[data_type + 1] -get_type(variable::Ptr{Nothing}) = get_type(readstat_variable_get_type(variable)) - -get_storagewidth(variable::Ptr{Nothing}) = readstat_variable_get_storage_width(variable) - -get_measure(variable::Ptr{Nothing}) = readstat_variable_get_measure(variable) - -get_alignment(variable::Ptr{Nothing}) = readstat_variable_get_alignment(variable) - -function handle_variable!(var_index::Cint, variable::Ptr{Nothing}, - val_label::Cstring, ds_ptr::Ptr{ReadStatDataFrame}) - col = var_index + 1 - ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame - missing_count = readstat_variable_get_missing_ranges_count(variable) - - push!(ds.val_label_keys, (val_label == C_NULL ? "" : unsafe_string(val_label))) - push!(ds.headers, get_name(variable)) - push!(ds.labels, get_label(variable)) - push!(ds.formats, get_format(variable)) - jtype = get_type(variable) - push!(ds.types, jtype) - push!(ds.types_as_int, readstat_variable_get_type(variable)) - push!(ds.hasmissings, missing_count > 0) - # SAS XPORT sets ds.rows == -1 - if ds.rows >= 0 - push!(ds.data, DataValueVector{jtype}(Vector{jtype}(undef, ds.rows), fill(false, ds.rows))) - else - push!(ds.data, DataValueVector{jtype}(Vector{jtype}(undef, 0), fill(false, 0))) - end - push!(ds.storagewidths, get_storagewidth(variable)) - push!(ds.measures, get_measure(variable)) - push!(ds.alignments, get_alignment(variable)) - - return Cint(0) -end - -get_type(val::Value) = get_type(readstat_value_type(val)) - -Base.convert(::Type{Int8}, val::Value) = ccall((:readstat_int8_value, libreadstat), Int8, (Value,), val) -Base.convert(::Type{Int16}, val::Value) = ccall((:readstat_int16_value, libreadstat), Int16, (Value,), val) -Base.convert(::Type{Int32}, val::Value) = ccall((:readstat_int32_value, libreadstat), Int32, (Value,), val) -Base.convert(::Type{Float32}, val::Value) = ccall((:readstat_float_value, libreadstat), Float32, (Value,), val) -Base.convert(::Type{Float64}, val::Value) = ccall((:readstat_double_value, libreadstat), Float64, (Value,), val) -function Base.convert(::Type{String}, val::Value) - ptr = ccall((:readstat_string_value, libreadstat), Cstring, (Value,), val) - ptr ≠ C_NULL ? unsafe_string(ptr) : "" -end -as_native(val::Value) = convert(get_type(val), val) - -function handle_value!(obs_index::Cint, variable::Ptr{Nothing}, - value::ReadStatValue, ds_ptr::Ptr{ReadStatDataFrame}) - ds = unsafe_pointer_to_objref(ds_ptr)::ReadStatDataFrame - var_index = readstat_variable_get_index(variable) + 1 - data = ds.data - @inbounds type_as_int = ds.types_as_int[var_index] - - ismissing = if @inbounds(ds.hasmissings[var_index]) - readstat_value_is_missing(value, variable) - else - readstat_value_is_missing(value, C_NULL) - end - - col = data[var_index] - @assert eltype(eltype(col)) == get_type(type_as_int) - - if ismissing - if obs_index < length(col) - DataValues.unsafe_setindex_isna!(col, true, obs_index + 1) - else - push!(col, DataValues.NA) - end - else - readfield!(col, obs_index + 1, value) - end - - return Cint(0) +jltype(t::ReadStatType) = READSTAT_TYPES[Int(t) + 1] + +include("reader/metadata.jl") +include("values/labeled.jl") +include("values/datetime.jl") +include("reader/columns.jl") +include("reader/context.jl") +include("reader/handlers.jl") +include("reader/table.jl") +include("reader/io.jl") +include("reader/threaded.jl") +include("reader/read.jl") +include("reader/source.jl") +include("reader/chunks.jl") +include("writer/writer.jl") +include("writer/write.jl") +include("deprecated.jl") + +function __init__() + _init_cfunctions() + _init_io_cfunctions() + _init_writer_cfunctions() + return nothing end -function readfield!(dest::DataValueVector{String}, row, val::ReadStatValue) - ptr = ccall((:readstat_string_value, libreadstat), Cstring, (ReadStatValue,), val) - - if row <= length(dest) - if ptr ≠ C_NULL - @inbounds DataValues.unsafe_setindex_value!(dest, unsafe_string(ptr), row) - end - elseif row == length(dest) + 1 - _val = ptr ≠ C_NULL ? unsafe_string(ptr) : "" - DataValues.push!(dest, _val) - else - throw(ArgumentError("illegal row index: $row")) - end -end - -for (j_type, rs_name) in ( - (Int8, :readstat_int8_value), - (Int16, :readstat_int16_value), - (Int32, :readstat_int32_value), - (Float32, :readstat_float_value), - (Float64, :readstat_double_value)) - @eval function readfield!(dest::DataValueVector{$j_type}, row, val::ReadStatValue) - _val = ccall(($(QuoteNode(rs_name)), libreadstat), $j_type, (ReadStatValue,), val) - if row <= length(dest) - @inbounds DataValues.unsafe_setindex_value!(dest, _val, row) - elseif row == length(dest) + 1 - DataValues.push!(dest, _val) - else - throw(ArgumentError("illegal row index: $row")) - end - end -end - -function handle_value_label!(val_labels::Cstring, value::Value, label::Cstring, ds_ptr::Ptr{ReadStatDataFrame}) - val_labels ≠ C_NULL || return Cint(0) - ds = unsafe_pointer_to_objref(ds_ptr) - dict = get!(ds.val_label_dict, unsafe_string(val_labels), Dict{Any,String}()) - dict[as_native(value)] = unsafe_string(label) - - return Cint(0) -end - -function read_data_file(filename::AbstractString, filetype::Val) - # initialize ds - ds = ReadStatDataFrame() - # initialize parser - parser = Parser() - # parse - parse_data_file!(ds, parser, filename, filetype) - # return dataframe instead of ReadStatDataFrame - return ds -end - -function Parser() - parser = ccall((:readstat_parser_init, libreadstat), Ptr{Nothing}, ()) - meta_fxn = @cfunction(handle_metadata!, Cint, (Ptr{Nothing}, Ptr{ReadStatDataFrame})) - var_fxn = @cfunction(handle_variable!, Cint, (Cint, Ptr{Nothing}, Cstring, Ptr{ReadStatDataFrame})) - val_fxn = @cfunction(handle_value!, Cint, (Cint, Ptr{Nothing}, ReadStatValue, Ptr{ReadStatDataFrame})) - label_fxn = @cfunction(handle_value_label!, Cint, (Cstring, Value, Cstring, Ptr{ReadStatDataFrame})) - ccall((:readstat_set_metadata_handler, libreadstat), Cint, (Ptr{Nothing}, Ptr{Nothing}), parser, meta_fxn) - ccall((:readstat_set_variable_handler, libreadstat), Cint, (Ptr{Nothing}, Ptr{Nothing}), parser, var_fxn) - ccall((:readstat_set_value_handler, libreadstat), Cint, (Ptr{Nothing}, Ptr{Nothing}), parser, val_fxn) - ccall((:readstat_set_value_label_handler, libreadstat), Cint, (Ptr{Nothing}, Ptr{Nothing}), parser, label_fxn) - return parser -end - -function error_message(retval::Integer) - unsafe_string(ccall((:readstat_error_message, libreadstat), Ptr{Cchar}, (Cint,), retval)) -end - -function parse_data_file!(ds::ReadStatDataFrame, parser::Ptr{Nothing}, filename::AbstractString, filetype::Val) - retval = readstat_parse(filename, filetype, parser, ds) - readstat_parser_free(parser) - retval == 0 || error("Error parsing $filename: $(error_message(retval))") -end - -read_dta(filename::AbstractString) = read_data_file(filename, Val(:dta)) -read_sav(filename::AbstractString) = read_data_file(filename, Val(:sav)) -read_por(filename::AbstractString) = read_data_file(filename, Val(:por)) -read_sas7bdat(filename::AbstractString) = read_data_file(filename, Val(:sas7bdat)) -read_xport(filename::AbstractString) = read_data_file(filename, Val(:xport)) - -end #module ReadStat +end # module ReadStat diff --git a/src/capi/enums.jl b/src/capi/enums.jl new file mode 100644 index 0000000..c95bdcd --- /dev/null +++ b/src/capi/enums.jl @@ -0,0 +1,112 @@ +# Enums and handler status codes mirroring readstat.h (v1.1.9). + +# Return codes for parse handlers. SKIP_VARIABLE is only valid from the +# variable handler; subsequent variables keep their original index values. +const READSTAT_HANDLER_OK = Cint(0) +const READSTAT_HANDLER_ABORT = Cint(1) +const READSTAT_HANDLER_SKIP_VARIABLE = Cint(2) + +@enum ReadStatType::Cint begin + READSTAT_TYPE_STRING = 0 + READSTAT_TYPE_INT8 + READSTAT_TYPE_INT16 + READSTAT_TYPE_INT32 + READSTAT_TYPE_FLOAT + READSTAT_TYPE_DOUBLE + READSTAT_TYPE_STRING_REF +end + +@enum ReadStatTypeClass::Cint begin + READSTAT_TYPE_CLASS_STRING = 0 + READSTAT_TYPE_CLASS_NUMERIC +end + +@enum ReadStatMeasure::Cint begin + READSTAT_MEASURE_UNKNOWN = 0 + READSTAT_MEASURE_NOMINAL + READSTAT_MEASURE_ORDINAL + READSTAT_MEASURE_SCALE +end + +@enum ReadStatAlignment::Cint begin + READSTAT_ALIGNMENT_UNKNOWN = 0 + READSTAT_ALIGNMENT_LEFT + READSTAT_ALIGNMENT_CENTER + READSTAT_ALIGNMENT_RIGHT +end + +@enum ReadStatCompress::Cint begin + READSTAT_COMPRESS_NONE = 0 + READSTAT_COMPRESS_ROWS + READSTAT_COMPRESS_BINARY +end + +@enum ReadStatEndian::Cint begin + READSTAT_ENDIAN_NONE = 0 + READSTAT_ENDIAN_LITTLE + READSTAT_ENDIAN_BIG +end + +@enum ReadStatIOFlags::Cint begin + READSTAT_SEEK_SET = 0 + READSTAT_SEEK_CUR + READSTAT_SEEK_END +end + +@enum ReadStatError::Cint begin + READSTAT_OK = 0 + READSTAT_ERROR_OPEN + READSTAT_ERROR_READ + READSTAT_ERROR_MALLOC + READSTAT_ERROR_USER_ABORT + READSTAT_ERROR_PARSE + READSTAT_ERROR_UNSUPPORTED_COMPRESSION + READSTAT_ERROR_UNSUPPORTED_CHARSET + READSTAT_ERROR_COLUMN_COUNT_MISMATCH + READSTAT_ERROR_ROW_COUNT_MISMATCH + READSTAT_ERROR_ROW_WIDTH_MISMATCH + READSTAT_ERROR_BAD_FORMAT_STRING + READSTAT_ERROR_VALUE_TYPE_MISMATCH + READSTAT_ERROR_WRITE + READSTAT_ERROR_WRITER_NOT_INITIALIZED + READSTAT_ERROR_SEEK + READSTAT_ERROR_CONVERT + READSTAT_ERROR_CONVERT_BAD_STRING + READSTAT_ERROR_CONVERT_SHORT_STRING + READSTAT_ERROR_CONVERT_LONG_STRING + READSTAT_ERROR_NUMERIC_VALUE_IS_OUT_OF_RANGE + READSTAT_ERROR_TAGGED_VALUE_IS_OUT_OF_RANGE + READSTAT_ERROR_STRING_VALUE_IS_TOO_LONG + READSTAT_ERROR_TAGGED_VALUES_NOT_SUPPORTED + READSTAT_ERROR_UNSUPPORTED_FILE_FORMAT_VERSION + READSTAT_ERROR_NAME_BEGINS_WITH_ILLEGAL_CHARACTER + READSTAT_ERROR_NAME_CONTAINS_ILLEGAL_CHARACTER + READSTAT_ERROR_NAME_IS_RESERVED_WORD + READSTAT_ERROR_NAME_IS_TOO_LONG + READSTAT_ERROR_BAD_TIMESTAMP_STRING + READSTAT_ERROR_BAD_FREQUENCY_WEIGHT + READSTAT_ERROR_TOO_MANY_MISSING_VALUE_DEFINITIONS + READSTAT_ERROR_NOTE_IS_TOO_LONG + READSTAT_ERROR_STRING_REFS_NOT_SUPPORTED + READSTAT_ERROR_STRING_REF_IS_REQUIRED + READSTAT_ERROR_ROW_IS_TOO_WIDE_FOR_PAGE + READSTAT_ERROR_TOO_FEW_COLUMNS + READSTAT_ERROR_TOO_MANY_COLUMNS + READSTAT_ERROR_NAME_IS_ZERO_LENGTH + READSTAT_ERROR_BAD_TIMESTAMP_VALUE +end + +function readstat_error_message(error_code::ReadStatError) + unsafe_string(@ccall libreadstat.readstat_error_message(error_code::ReadStatError)::Ptr{Cchar}) +end + +export READSTAT_HANDLER_OK, READSTAT_HANDLER_ABORT, READSTAT_HANDLER_SKIP_VARIABLE, + readstat_error_message + +for enum_type in (ReadStatType, ReadStatTypeClass, ReadStatMeasure, ReadStatAlignment, + ReadStatCompress, ReadStatEndian, ReadStatIOFlags, ReadStatError) + @eval export $(nameof(enum_type)) + for instance in instances(enum_type) + @eval export $(Symbol(instance)) + end +end diff --git a/src/capi/parser.jl b/src/capi/parser.jl new file mode 100644 index 0000000..c6aa750 --- /dev/null +++ b/src/capi/parser.jl @@ -0,0 +1,195 @@ +# Parser lifecycle, handler registration, parser configuration, metadata and +# variable accessors, and the parse entry points from readstat.h (v1.1.9). + +export readstat_parser_init, readstat_parser_free, + readstat_set_metadata_handler, readstat_set_note_handler, readstat_set_variable_handler, + readstat_set_fweight_handler, readstat_set_value_handler, readstat_set_value_label_handler, + readstat_set_error_handler, readstat_set_progress_handler, + readstat_set_open_handler, readstat_set_close_handler, readstat_set_seek_handler, + readstat_set_read_handler, readstat_set_update_handler, readstat_set_io_ctx, + readstat_set_file_character_encoding, readstat_set_handler_character_encoding, + readstat_set_row_limit, readstat_set_row_offset, + readstat_parse_dta, readstat_parse_sav, readstat_parse_por, + readstat_parse_sas7bdat, readstat_parse_sas7bcat, readstat_parse_xport, + readstat_get_row_count, readstat_get_var_count, readstat_get_creation_time, + readstat_get_modified_time, readstat_get_file_format_version, + readstat_get_file_format_is_64bit, readstat_get_compression, readstat_get_endianness, + readstat_get_table_name, readstat_get_file_label, readstat_get_file_encoding, + readstat_variable_get_index, readstat_variable_get_index_after_skipping, + readstat_variable_get_name, readstat_variable_get_label, readstat_variable_get_format, + readstat_variable_get_type, readstat_variable_get_type_class, + readstat_variable_get_storage_width, readstat_variable_get_display_width, + readstat_variable_get_measure, readstat_variable_get_alignment, + readstat_variable_get_missing_ranges_count, + readstat_variable_get_missing_range_lo, readstat_variable_get_missing_range_hi + +_string_or_empty(ptr::Cstring) = ptr == C_NULL ? "" : unsafe_string(ptr) +_string_or_empty(ptr::Ptr{Cchar}) = ptr == C_NULL ? "" : unsafe_string(ptr) + +# --------------------------------------------------------------------------- +# Parser lifecycle + +readstat_parser_init() = @ccall libreadstat.readstat_parser_init()::ParserPtr + +function readstat_parser_free(parser::ParserPtr) + @ccall libreadstat.readstat_parser_free(parser::ParserPtr)::Cvoid +end + +# --------------------------------------------------------------------------- +# Handler registration. Each takes a C function pointer (from @cfunction). + +for setter in (:readstat_set_metadata_handler, :readstat_set_note_handler, + :readstat_set_variable_handler, :readstat_set_fweight_handler, + :readstat_set_value_handler, :readstat_set_value_label_handler, + :readstat_set_error_handler, :readstat_set_progress_handler, + :readstat_set_open_handler, :readstat_set_close_handler, + :readstat_set_seek_handler, :readstat_set_read_handler, + :readstat_set_update_handler) + @eval function $setter(parser::ParserPtr, handler::Ptr{Cvoid}) + @ccall libreadstat.$setter(parser::ParserPtr, handler::Ptr{Cvoid})::ReadStatError + end +end + +function readstat_set_io_ctx(parser::ParserPtr, io_ctx::Ptr{Cvoid}) + @ccall libreadstat.readstat_set_io_ctx(parser::ParserPtr, io_ctx::Ptr{Cvoid})::ReadStatError +end + +# --------------------------------------------------------------------------- +# Parser configuration + +# Override the encoding declared in the file (iconv-compatible name); useful +# e.g. for pre-14 Stata files written in a system codepage. +function readstat_set_file_character_encoding(parser::ParserPtr, encoding::AbstractString) + @ccall libreadstat.readstat_set_file_character_encoding(parser::ParserPtr, encoding::Cstring)::ReadStatError +end + +# Encoding delivered to the handlers; defaults to UTF-8. C_NULL disables +# transliteration entirely. +function readstat_set_handler_character_encoding(parser::ParserPtr, encoding::Union{AbstractString,Ptr{Nothing}}) + @ccall libreadstat.readstat_set_handler_character_encoding(parser::ParserPtr, encoding::Cstring)::ReadStatError +end + +function readstat_set_row_limit(parser::ParserPtr, row_limit::Integer) + @ccall libreadstat.readstat_set_row_limit(parser::ParserPtr, row_limit::Clong)::ReadStatError +end + +function readstat_set_row_offset(parser::ParserPtr, row_offset::Integer) + @ccall libreadstat.readstat_set_row_offset(parser::ParserPtr, row_offset::Clong)::ReadStatError +end + +# --------------------------------------------------------------------------- +# Parse entry points. `user_ctx` is passed as `Any` so the ccall roots the +# context object for the duration of the parse. + +for parse_fn in (:readstat_parse_dta, :readstat_parse_sav, :readstat_parse_por, + :readstat_parse_sas7bdat, :readstat_parse_sas7bcat, :readstat_parse_xport) + @eval function $parse_fn(parser::ParserPtr, path::AbstractString, user_ctx) + @ccall libreadstat.$parse_fn(parser::ParserPtr, path::Cstring, user_ctx::Any)::ReadStatError + end +end + +# --------------------------------------------------------------------------- +# Metadata accessors (valid inside the metadata handler) + +function readstat_get_row_count(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_row_count(metadata::MetadataPtr)::Cint +end + +function readstat_get_var_count(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_var_count(metadata::MetadataPtr)::Cint +end + +function readstat_get_creation_time(metadata::MetadataPtr) + Int64(@ccall libreadstat.readstat_get_creation_time(metadata::MetadataPtr)::Ctime_t) +end + +function readstat_get_modified_time(metadata::MetadataPtr) + Int64(@ccall libreadstat.readstat_get_modified_time(metadata::MetadataPtr)::Ctime_t) +end + +function readstat_get_file_format_version(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_file_format_version(metadata::MetadataPtr)::Cint +end + +function readstat_get_file_format_is_64bit(metadata::MetadataPtr) + (@ccall libreadstat.readstat_get_file_format_is_64bit(metadata::MetadataPtr)::Cint) != 0 +end + +function readstat_get_compression(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_compression(metadata::MetadataPtr)::ReadStatCompress +end + +function readstat_get_endianness(metadata::MetadataPtr) + @ccall libreadstat.readstat_get_endianness(metadata::MetadataPtr)::ReadStatEndian +end + +function readstat_get_table_name(metadata::MetadataPtr) + _string_or_empty(@ccall libreadstat.readstat_get_table_name(metadata::MetadataPtr)::Cstring) +end + +function readstat_get_file_label(metadata::MetadataPtr) + _string_or_empty(@ccall libreadstat.readstat_get_file_label(metadata::MetadataPtr)::Cstring) +end + +function readstat_get_file_encoding(metadata::MetadataPtr) + _string_or_empty(@ccall libreadstat.readstat_get_file_encoding(metadata::MetadataPtr)::Cstring) +end + +# --------------------------------------------------------------------------- +# Variable accessors (valid inside the variable/value handlers) + +function readstat_variable_get_index(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_index(variable::VariablePtr)::Cint +end + +function readstat_variable_get_index_after_skipping(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_index_after_skipping(variable::VariablePtr)::Cint +end + +function readstat_variable_get_name(variable::VariablePtr) + _string_or_empty(@ccall libreadstat.readstat_variable_get_name(variable::VariablePtr)::Cstring) +end + +function readstat_variable_get_label(variable::VariablePtr) + _string_or_empty(@ccall libreadstat.readstat_variable_get_label(variable::VariablePtr)::Cstring) +end + +function readstat_variable_get_format(variable::VariablePtr) + _string_or_empty(@ccall libreadstat.readstat_variable_get_format(variable::VariablePtr)::Cstring) +end + +function readstat_variable_get_type(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_type(variable::VariablePtr)::ReadStatType +end + +function readstat_variable_get_type_class(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_type_class(variable::VariablePtr)::ReadStatTypeClass +end + +function readstat_variable_get_storage_width(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_storage_width(variable::VariablePtr)::Csize_t +end + +function readstat_variable_get_display_width(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_display_width(variable::VariablePtr)::Cint +end + +function readstat_variable_get_measure(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_measure(variable::VariablePtr)::ReadStatMeasure +end + +function readstat_variable_get_alignment(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_alignment(variable::VariablePtr)::ReadStatAlignment +end + +function readstat_variable_get_missing_ranges_count(variable::VariablePtr) + @ccall libreadstat.readstat_variable_get_missing_ranges_count(variable::VariablePtr)::Cint +end + +function readstat_variable_get_missing_range_lo(variable::VariablePtr, i::Integer) + @ccall libreadstat.readstat_variable_get_missing_range_lo(variable::VariablePtr, i::Cint)::ReadStatValue +end + +function readstat_variable_get_missing_range_hi(variable::VariablePtr, i::Integer) + @ccall libreadstat.readstat_variable_get_missing_range_hi(variable::VariablePtr, i::Cint)::ReadStatValue +end diff --git a/src/capi/schema.jl b/src/capi/schema.jl new file mode 100644 index 0000000..f14023e --- /dev/null +++ b/src/capi/schema.jl @@ -0,0 +1,24 @@ +# The schema/plain-text path: parse a SAS command file, SPSS command file, +# or Stata dictionary into a schema, then drive the fixed-width text parser +# with it. + +export readstat_parse_sas_commands, readstat_parse_spss_commands, + readstat_parse_stata_dictionary, readstat_parse_txt, readstat_schema_free + +for schema_fn in (:readstat_parse_sas_commands, :readstat_parse_spss_commands, + :readstat_parse_stata_dictionary) + @eval function $schema_fn(parser::ParserPtr, path::AbstractString, user_ctx, + out_error::Ref{ReadStatError}) + @ccall libreadstat.$schema_fn(parser::ParserPtr, path::Cstring, user_ctx::Any, + out_error::Ref{ReadStatError})::SchemaPtr + end +end + +function readstat_parse_txt(parser::ParserPtr, path::AbstractString, schema::SchemaPtr, user_ctx) + @ccall libreadstat.readstat_parse_txt(parser::ParserPtr, path::Cstring, + schema::SchemaPtr, user_ctx::Any)::ReadStatError +end + +function readstat_schema_free(schema::SchemaPtr) + @ccall libreadstat.readstat_schema_free(schema::SchemaPtr)::Cvoid +end diff --git a/src/capi/value.jl b/src/capi/value.jl new file mode 100644 index 0000000..9cbc3e7 --- /dev/null +++ b/src/capi/value.jl @@ -0,0 +1,120 @@ +# The readstat_value_t mirror and its accessors. +# +# readstat_value_t is the only C struct this package mirrors: the C API passes +# and returns it by value, so an opaque pointer is not an option. Every other +# C struct is handled through an opaque pointer plus the C getter functions. + +export ReadStatValue, Coff_t, Ctime_t, + ParserPtr, MetadataPtr, VariablePtr, LabelSetPtr, SchemaPtr, WriterPtr, StringRefPtr, + readstat_value_type, readstat_value_type_class, readstat_type_class, + readstat_value_is_missing, readstat_value_is_system_missing, + readstat_value_is_tagged_missing, readstat_value_is_defined_missing, + readstat_value_tag, readstat_int8_value, readstat_int16_value, readstat_int32_value, + readstat_float_value, readstat_double_value, readstat_string_value + +# readstat_off_t and time_t as the C library was compiled. readstat.h +# typedefs readstat_off_t to _off64_t on Windows, so it is 64-bit on both +# Windows architectures; elsewhere it is off_t, which is 32-bit on 32-bit +# unix (no large-file opt-in in the jll build). time_t follows the word +# size everywhere: 32-bit on i686 for both MinGW (no _USE_32BIT_TIME_T +# opt-out) and glibc (no time64 opt-in). Getting these wrong shifts the +# seek callback's argument stack (segfaults) or fills the high half of +# timestamps with register garbage. +const Coff_t = (Sys.iswindows() || Sys.WORD_SIZE == 64) ? Int64 : Int32 +const Ctime_t = Sys.WORD_SIZE == 64 ? Int64 : Int32 + +# Opaque C struct tags; only ever used as Ptr{...} type parameters. +abstract type readstat_parser_s end +abstract type readstat_metadata_s end +abstract type readstat_variable_s end +abstract type readstat_label_set_s end +abstract type readstat_schema_s end +abstract type readstat_writer_s end +abstract type readstat_string_ref_s end + +const ParserPtr = Ptr{readstat_parser_s} +const MetadataPtr = Ptr{readstat_metadata_s} +const VariablePtr = Ptr{readstat_variable_s} +const LabelSetPtr = Ptr{readstat_label_set_s} +const SchemaPtr = Ptr{readstat_schema_s} +const WriterPtr = Ptr{readstat_writer_s} +const StringRefPtr = Ptr{readstat_string_ref_s} + +# Layout must match readstat.h's readstat_value_t under the jll's compiler: +# 8-byte value union, Cint type at offset 8, char tag at 12, then two +# `unsigned int:1` bitfields. Where the bitfields land depends on the +# platform's bitfield ABI: Itanium/SysV GCC packs them into byte 13 (struct +# size 16), while MinGW GCC defaults to MS-compatible bitfield layout, which +# starts a fresh 4-byte-aligned unsigned int unit at offset 16 (struct size +# 24). The `bits` field is never decoded in Julia — the is_missing/tag +# accessors below are the only sanctioned way in. test/test_abi.jl checks +# this layout behaviorally against the C library. +struct ReadStatValue + v::Int64 + type::ReadStatType + tag::Cchar + @static if Sys.iswindows() + bits::Cuint + else + bits::UInt8 + end +end + +function readstat_value_type(value::ReadStatValue) + @ccall libreadstat.readstat_value_type(value::ReadStatValue)::ReadStatType +end + +function readstat_value_type_class(value::ReadStatValue) + @ccall libreadstat.readstat_value_type_class(value::ReadStatValue)::ReadStatTypeClass +end + +function readstat_type_class(type::ReadStatType) + @ccall libreadstat.readstat_type_class(type::ReadStatType)::ReadStatTypeClass +end + +# `variable` may be a VariablePtr or C_NULL; with C_NULL only system/tagged +# missingness is considered. +function readstat_value_is_missing(value::ReadStatValue, variable::Ptr) + (@ccall libreadstat.readstat_value_is_missing(value::ReadStatValue, variable::VariablePtr)::Cint) != 0 +end + +function readstat_value_is_system_missing(value::ReadStatValue) + (@ccall libreadstat.readstat_value_is_system_missing(value::ReadStatValue)::Cint) != 0 +end + +function readstat_value_is_tagged_missing(value::ReadStatValue) + (@ccall libreadstat.readstat_value_is_tagged_missing(value::ReadStatValue)::Cint) != 0 +end + +function readstat_value_is_defined_missing(value::ReadStatValue, variable::Ptr) + (@ccall libreadstat.readstat_value_is_defined_missing(value::ReadStatValue, variable::VariablePtr)::Cint) != 0 +end + +function readstat_value_tag(value::ReadStatValue) + Char(@ccall libreadstat.readstat_value_tag(value::ReadStatValue)::Cchar) +end + +function readstat_int8_value(value::ReadStatValue) + @ccall libreadstat.readstat_int8_value(value::ReadStatValue)::Int8 +end + +function readstat_int16_value(value::ReadStatValue) + @ccall libreadstat.readstat_int16_value(value::ReadStatValue)::Int16 +end + +function readstat_int32_value(value::ReadStatValue) + @ccall libreadstat.readstat_int32_value(value::ReadStatValue)::Int32 +end + +function readstat_float_value(value::ReadStatValue) + @ccall libreadstat.readstat_float_value(value::ReadStatValue)::Float32 +end + +function readstat_double_value(value::ReadStatValue) + @ccall libreadstat.readstat_double_value(value::ReadStatValue)::Float64 +end + +# Returns C_NULL-able pointer; callers decide how to treat NULL vs "". +function readstat_string_value(value::ReadStatValue) + @ccall libreadstat.readstat_string_value(value::ReadStatValue)::Cstring +end diff --git a/src/capi/writer.jl b/src/capi/writer.jl new file mode 100644 index 0000000..b1bf57f --- /dev/null +++ b/src/capi/writer.jl @@ -0,0 +1,250 @@ +# The full writer API from readstat.h (v1.1.9). + +export readstat_writer_init, readstat_writer_free, readstat_set_data_writer, + readstat_add_label_set, readstat_label_double_value, readstat_label_int32_value, + readstat_label_string_value, readstat_label_tagged_value, + readstat_add_variable, readstat_variable_set_label, readstat_variable_set_format, + readstat_variable_set_label_set, readstat_variable_set_measure, + readstat_variable_set_alignment, readstat_variable_set_display_width, + readstat_variable_add_missing_double_value, readstat_variable_add_missing_double_range, + readstat_variable_add_missing_string_value, readstat_variable_add_missing_string_range, + readstat_get_variable, readstat_add_note, readstat_add_string_ref, + readstat_get_string_ref, + readstat_writer_set_file_label, readstat_writer_set_file_timestamp, + readstat_writer_set_fweight_variable, readstat_writer_set_file_format_version, + readstat_writer_set_table_name, readstat_writer_set_file_format_is_64bit, + readstat_writer_set_compression, readstat_writer_set_error_handler, + readstat_begin_writing_dta, readstat_begin_writing_por, readstat_begin_writing_sas7bcat, + readstat_begin_writing_sas7bdat, readstat_begin_writing_sav, readstat_begin_writing_xport, + readstat_validate_metadata, readstat_validate_variable, + readstat_begin_row, readstat_insert_int8_value, readstat_insert_int16_value, + readstat_insert_int32_value, readstat_insert_float_value, readstat_insert_double_value, + readstat_insert_string_value, readstat_insert_string_ref, readstat_insert_missing_value, + readstat_insert_tagged_missing_value, readstat_end_row, readstat_end_writing + +readstat_writer_init() = @ccall libreadstat.readstat_writer_init()::WriterPtr + +function readstat_writer_free(writer::WriterPtr) + @ccall libreadstat.readstat_writer_free(writer::WriterPtr)::Cvoid +end + +function readstat_set_data_writer(writer::WriterPtr, data_writer::Ptr{Cvoid}) + @ccall libreadstat.readstat_set_data_writer(writer::WriterPtr, data_writer::Ptr{Cvoid})::ReadStatError +end + +# --------------------------------------------------------------------------- +# Label sets + +function readstat_add_label_set(writer::WriterPtr, type::ReadStatType, name::AbstractString) + @ccall libreadstat.readstat_add_label_set(writer::WriterPtr, type::ReadStatType, + name::Cstring)::LabelSetPtr +end + +function readstat_label_double_value(label_set::LabelSetPtr, value::Real, label::AbstractString) + @ccall libreadstat.readstat_label_double_value(label_set::LabelSetPtr, value::Cdouble, + label::Cstring)::Cvoid +end + +function readstat_label_int32_value(label_set::LabelSetPtr, value::Integer, label::AbstractString) + @ccall libreadstat.readstat_label_int32_value(label_set::LabelSetPtr, value::Int32, + label::Cstring)::Cvoid +end + +function readstat_label_string_value(label_set::LabelSetPtr, value::AbstractString, label::AbstractString) + @ccall libreadstat.readstat_label_string_value(label_set::LabelSetPtr, value::Cstring, + label::Cstring)::Cvoid +end + +function readstat_label_tagged_value(label_set::LabelSetPtr, tag::Char, label::AbstractString) + @ccall libreadstat.readstat_label_tagged_value(label_set::LabelSetPtr, tag::Cchar, + label::Cstring)::Cvoid +end + +# --------------------------------------------------------------------------- +# Variables + +function readstat_add_variable(writer::WriterPtr, name::AbstractString, type::ReadStatType, + storage_width::Integer) + @ccall libreadstat.readstat_add_variable(writer::WriterPtr, name::Cstring, + type::ReadStatType, storage_width::Csize_t)::VariablePtr +end + +function readstat_variable_set_label(variable::VariablePtr, label::AbstractString) + @ccall libreadstat.readstat_variable_set_label(variable::VariablePtr, label::Cstring)::Cvoid +end + +function readstat_variable_set_format(variable::VariablePtr, format::AbstractString) + @ccall libreadstat.readstat_variable_set_format(variable::VariablePtr, format::Cstring)::Cvoid +end + +function readstat_variable_set_label_set(variable::VariablePtr, label_set::LabelSetPtr) + @ccall libreadstat.readstat_variable_set_label_set(variable::VariablePtr, + label_set::LabelSetPtr)::Cvoid +end + +function readstat_variable_set_measure(variable::VariablePtr, measure::ReadStatMeasure) + @ccall libreadstat.readstat_variable_set_measure(variable::VariablePtr, + measure::ReadStatMeasure)::Cvoid +end + +function readstat_variable_set_alignment(variable::VariablePtr, alignment::ReadStatAlignment) + @ccall libreadstat.readstat_variable_set_alignment(variable::VariablePtr, + alignment::ReadStatAlignment)::Cvoid +end + +function readstat_variable_set_display_width(variable::VariablePtr, display_width::Integer) + @ccall libreadstat.readstat_variable_set_display_width(variable::VariablePtr, + display_width::Cint)::Cvoid +end + +function readstat_variable_add_missing_double_value(variable::VariablePtr, value::Real) + @ccall libreadstat.readstat_variable_add_missing_double_value(variable::VariablePtr, + value::Cdouble)::ReadStatError +end + +function readstat_variable_add_missing_double_range(variable::VariablePtr, lo::Real, hi::Real) + @ccall libreadstat.readstat_variable_add_missing_double_range(variable::VariablePtr, + lo::Cdouble, hi::Cdouble)::ReadStatError +end + +function readstat_variable_add_missing_string_value(variable::VariablePtr, value::AbstractString) + @ccall libreadstat.readstat_variable_add_missing_string_value(variable::VariablePtr, + value::Cstring)::ReadStatError +end + +function readstat_variable_add_missing_string_range(variable::VariablePtr, + lo::AbstractString, hi::AbstractString) + @ccall libreadstat.readstat_variable_add_missing_string_range(variable::VariablePtr, + lo::Cstring, hi::Cstring)::ReadStatError +end + +function readstat_get_variable(writer::WriterPtr, index::Integer) + @ccall libreadstat.readstat_get_variable(writer::WriterPtr, index::Cint)::VariablePtr +end + +# --------------------------------------------------------------------------- +# Notes, string refs, file-level metadata + +function readstat_add_note(writer::WriterPtr, note::AbstractString) + @ccall libreadstat.readstat_add_note(writer::WriterPtr, note::Cstring)::Cvoid +end + +function readstat_add_string_ref(writer::WriterPtr, string::AbstractString) + @ccall libreadstat.readstat_add_string_ref(writer::WriterPtr, string::Cstring)::StringRefPtr +end + +function readstat_get_string_ref(writer::WriterPtr, index::Integer) + @ccall libreadstat.readstat_get_string_ref(writer::WriterPtr, index::Cint)::StringRefPtr +end + +function readstat_writer_set_file_label(writer::WriterPtr, file_label::AbstractString) + @ccall libreadstat.readstat_writer_set_file_label(writer::WriterPtr, + file_label::Cstring)::ReadStatError +end + +function readstat_writer_set_file_timestamp(writer::WriterPtr, timestamp::Integer) + @ccall libreadstat.readstat_writer_set_file_timestamp(writer::WriterPtr, + timestamp::Ctime_t)::ReadStatError +end + +function readstat_writer_set_fweight_variable(writer::WriterPtr, variable::VariablePtr) + @ccall libreadstat.readstat_writer_set_fweight_variable(writer::WriterPtr, + variable::VariablePtr)::ReadStatError +end + +function readstat_writer_set_file_format_version(writer::WriterPtr, version::Integer) + @ccall libreadstat.readstat_writer_set_file_format_version(writer::WriterPtr, + version::UInt8)::ReadStatError +end + +function readstat_writer_set_table_name(writer::WriterPtr, table_name::AbstractString) + @ccall libreadstat.readstat_writer_set_table_name(writer::WriterPtr, + table_name::Cstring)::ReadStatError +end + +function readstat_writer_set_file_format_is_64bit(writer::WriterPtr, is_64bit::Bool) + @ccall libreadstat.readstat_writer_set_file_format_is_64bit(writer::WriterPtr, + is_64bit::Cint)::ReadStatError +end + +function readstat_writer_set_compression(writer::WriterPtr, compression::ReadStatCompress) + @ccall libreadstat.readstat_writer_set_compression(writer::WriterPtr, + compression::ReadStatCompress)::ReadStatError +end + +function readstat_writer_set_error_handler(writer::WriterPtr, error_handler::Ptr{Cvoid}) + @ccall libreadstat.readstat_writer_set_error_handler(writer::WriterPtr, + error_handler::Ptr{Cvoid})::ReadStatError +end + +# --------------------------------------------------------------------------- +# Writing + +for begin_fn in (:readstat_begin_writing_dta, :readstat_begin_writing_por, + :readstat_begin_writing_sas7bdat, :readstat_begin_writing_sav, + :readstat_begin_writing_xport) + @eval function $begin_fn(writer::WriterPtr, user_ctx::Ptr{Cvoid}, row_count::Integer) + @ccall libreadstat.$begin_fn(writer::WriterPtr, user_ctx::Ptr{Cvoid}, + row_count::Clong)::ReadStatError + end +end + +function readstat_begin_writing_sas7bcat(writer::WriterPtr, user_ctx::Ptr{Cvoid}) + @ccall libreadstat.readstat_begin_writing_sas7bcat(writer::WriterPtr, + user_ctx::Ptr{Cvoid})::ReadStatError +end + +function readstat_validate_metadata(writer::WriterPtr) + @ccall libreadstat.readstat_validate_metadata(writer::WriterPtr)::ReadStatError +end + +function readstat_validate_variable(writer::WriterPtr, variable::VariablePtr) + @ccall libreadstat.readstat_validate_variable(writer::WriterPtr, + variable::VariablePtr)::ReadStatError +end + +function readstat_begin_row(writer::WriterPtr) + @ccall libreadstat.readstat_begin_row(writer::WriterPtr)::ReadStatError +end + +for (insert_fn, T) in ((:readstat_insert_int8_value, Int8), + (:readstat_insert_int16_value, Int16), + (:readstat_insert_int32_value, Int32), + (:readstat_insert_float_value, Float32), + (:readstat_insert_double_value, Float64)) + @eval function $insert_fn(writer::WriterPtr, variable::VariablePtr, value::$T) + @ccall libreadstat.$insert_fn(writer::WriterPtr, variable::VariablePtr, + value::$T)::ReadStatError + end +end + +function readstat_insert_string_value(writer::WriterPtr, variable::VariablePtr, + value::AbstractString) + @ccall libreadstat.readstat_insert_string_value(writer::WriterPtr, + variable::VariablePtr, value::Cstring)::ReadStatError +end + +function readstat_insert_string_ref(writer::WriterPtr, variable::VariablePtr, + ref::StringRefPtr) + @ccall libreadstat.readstat_insert_string_ref(writer::WriterPtr, + variable::VariablePtr, ref::StringRefPtr)::ReadStatError +end + +function readstat_insert_missing_value(writer::WriterPtr, variable::VariablePtr) + @ccall libreadstat.readstat_insert_missing_value(writer::WriterPtr, + variable::VariablePtr)::ReadStatError +end + +function readstat_insert_tagged_missing_value(writer::WriterPtr, variable::VariablePtr, + tag::Char) + @ccall libreadstat.readstat_insert_tagged_missing_value(writer::WriterPtr, + variable::VariablePtr, tag::Cchar)::ReadStatError +end + +function readstat_end_row(writer::WriterPtr) + @ccall libreadstat.readstat_end_row(writer::WriterPtr)::ReadStatError +end + +function readstat_end_writing(writer::WriterPtr) + @ccall libreadstat.readstat_end_writing(writer::WriterPtr)::ReadStatError +end diff --git a/src/deprecated.jl b/src/deprecated.jl new file mode 100644 index 0000000..44e4f0d --- /dev/null +++ b/src/deprecated.jl @@ -0,0 +1,90 @@ +# 1.x compatibility shims, to be removed in 3.0. +# +# ReadStat 1.x returned a `ReadStatDataFrame` whose data and metadata were +# reached through 15 public fields. The old name aliases the new table type, +# and property access reconstructs the old field values with a deprecation +# warning, so existing consumers (StatFiles.jl in particular uses `.data` and +# `.headers`) keep working against 2.0. + +""" + ReadStatDataFrame + +Deprecated alias for [`ReadStatTable`](@ref). The 1.x field access +(`df.data`, `df.headers`, ...) still works but emits deprecation warnings; +use the 2.0 accessors instead. +""" +const ReadStatDataFrame = ReadStatTable + +function _dep(old::Symbol, instead::AbstractString) + Base.depwarn("`ReadStatDataFrame.$old` is deprecated; use $instead instead.", old) +end + +function Base.getproperty(tbl::ReadStatTable, s::Symbol) + if s === :cols || s === :names || s === :lookup || s === :meta || s === :colmeta || s === :tags + return getfield(tbl, s) + elseif s === :data + _dep(s, "`tbl[i]` / `tbl[:name]`") + return Any[c for c in getfield(tbl, :cols)] + elseif s === :headers + _dep(s, "`names(tbl)`") + return copy(getfield(tbl, :names)) + elseif s === :types + _dep(s, "`eltype(eltype(tbl[i]))`") + return DataType[jltype(m.type) for m in getfield(tbl, :colmeta)] + elseif s === :labels + _dep(s, "`varmetadata(tbl, i).label`") + return String[m.label for m in getfield(tbl, :colmeta)] + elseif s === :formats + _dep(s, "`varmetadata(tbl, i).format`") + return String[m.format for m in getfield(tbl, :colmeta)] + elseif s === :storagewidths + _dep(s, "`varmetadata(tbl, i).storage_width`") + return Csize_t[m.storage_width for m in getfield(tbl, :colmeta)] + elseif s === :measures + _dep(s, "`varmetadata(tbl, i).measure`") + return Cint[Cint(Int(m.measure)) for m in getfield(tbl, :colmeta)] + elseif s === :alignments + _dep(s, "`varmetadata(tbl, i).alignment`") + return Cint[Cint(Int(m.alignment)) for m in getfield(tbl, :colmeta)] + elseif s === :val_label_keys + _dep(s, "`varmetadata(tbl, i).vallabel`") + return String[string(m.vallabel) for m in getfield(tbl, :colmeta)] + elseif s === :val_label_dict + _dep(s, "`valuelabels(tbl, col)` / `filemetadata(tbl).value_labels`") + return Dict{String,Dict{Any,String}}( + string(k) => Dict{Any,String}(kk => vv for (kk, vv) in v) + for (k, v) in getfield(tbl, :meta).value_labels) + elseif s === :rows + _dep(s, "`size(tbl, 1)` or `filemetadata(tbl).row_count`") + return getfield(tbl, :meta).row_count + elseif s === :columns + _dep(s, "`size(tbl, 2)`") + return getfield(tbl, :meta).var_count + elseif s === :filelabel + _dep(s, "`filemetadata(tbl).file_label`") + return getfield(tbl, :meta).file_label + elseif s === :timestamp + _dep(s, "`filemetadata(tbl).modified_time`") + return getfield(tbl, :meta).modified_time + elseif s === :format + _dep(s, "`filemetadata(tbl).file_format_version`") + return Clong(getfield(tbl, :meta).file_format_version) + elseif s === :types_as_int + _dep(s, "`varmetadata(tbl, i).type`") + return Cint[Cint(Int(m.type)) for m in getfield(tbl, :colmeta)] + elseif s === :hasmissings + _dep(s, "`varmetadata(tbl, i).missing_ranges`") + return Bool[!isempty(m.missing_ranges) for m in getfield(tbl, :colmeta)] + else + # Fall through for reflection etc. + return getfield(tbl, s) + end +end + +function Base.propertynames(::ReadStatTable, private::Bool=false) + private ? (fieldnames(ReadStatTable)..., DEPRECATED_PROPERTIES...) : DEPRECATED_PROPERTIES +end + +const DEPRECATED_PROPERTIES = (:data, :headers, :types, :labels, :formats, :storagewidths, + :measures, :alignments, :val_label_keys, :val_label_dict, :rows, :columns, :filelabel, + :timestamp, :format, :types_as_int, :hasmissings) diff --git a/src/reader/chunks.jl b/src/reader/chunks.jl new file mode 100644 index 0000000..7c4e91e --- /dev/null +++ b/src/reader/chunks.jl @@ -0,0 +1,121 @@ +# Streaming chunked reads: a single pass over the file that hands finished +# ReadStatTables of `chunksize` rows through a bounded Channel. This is O(n) +# for a full scan — deliberately not one re-parse per chunk, which would be +# quadratic on compressed formats — and it works for formats that do not +# record a row count. + +# State threaded through the value handler while a chunked parse runs. The +# schema and metadata come from a pre-pass so every chunk (and the labels on +# it) is complete from the first chunk on, regardless of where the file +# stores its label sets. +mutable struct ChunkSink + channel::Channel{ReadStatTable} + chunksize::Int + names::Vector{Symbol} + varmeta::Vector{ReadStatVarMeta} + meta::ReadStatMeta + convert_datetime::Bool + apply_value_labels::Bool + file_format::Symbol +end + +_maybe_flush!(pc::ParseContext, sink::ChunkSink, row::Int) = + (row == sink.chunksize && _flush_chunk!(pc, sink, row); nothing) + +function _flush_chunk!(pc::ParseContext, sink::ChunkSink, nrows::Int) + trimcolumns!(pc.cols, nrows) + bt = ParseContext() + bt.names = sink.names + bt.varmeta = sink.varmeta + bt.meta = sink.meta + bt.cols = pc.cols + bt.convert_datetime = sink.convert_datetime + bt.apply_value_labels = sink.apply_value_labels + bt.file_format = sink.file_format + put!(sink.channel, build_table(bt)) + # Fresh buffers for the next chunk; rows keep arriving with absolute + # observation indices, so the base shifts back by what was emitted. + cols = TypedColumns() + for vm in sink.varmeta + addcolumn!(cols, jltype(vm.type), sink.chunksize) + end + pc.cols = cols + pc.row_base -= nrows + pc.rows_complete = 0 + return +end + +""" + ReadStatChunks + +Iterator over the chunks of a [`chunks`](@ref) read. Each element is a +complete [`ReadStatTable`](@ref) of up to `chunksize` rows sharing the +file's metadata. Iteration is single-pass and backed by a bounded channel: +the parse runs on its own task and blocks once two chunks are waiting. +Call `close(it)` if iteration stops early, so the parse task is released. +""" +struct ReadStatChunks + channel::Channel{ReadStatTable} +end + +Base.IteratorSize(::Type{ReadStatChunks}) = Base.SizeUnknown() +Base.eltype(::Type{ReadStatChunks}) = ReadStatTable +Base.close(it::ReadStatChunks) = close(it.channel) + +function Base.iterate(it::ReadStatChunks, state=nothing) + try + return (take!(it.channel), nothing) + catch e + e isa InvalidStateException && return nothing + rethrow() + end +end + +""" + chunks(src::ReadStatSource; chunksize=65_536, usecols=nothing, rows=nothing) + -> ReadStatChunks + +Stream the source as [`ReadStatTable`](@ref)s of up to `chunksize` rows, in +one pass over the file. `usecols` and `rows` push a projection and a 1-based +row range into the parse, exactly as in `read(src; ...)`. The chunks carry +the file's complete metadata (from a metadata pre-pass), so value labels and +date/time conversion apply from the first chunk on. + +Iterate the result with `for chunk in chunks(src) ... end`; when stopping +early, `close` the iterator to release the parse task. +""" +function chunks(src::ReadStatSource; chunksize::Integer=65_536, usecols=nothing, + rows::Union{Nothing,AbstractUnitRange{<:Integer}}=nothing) + chunksize >= 1 || throw(ArgumentError("chunksize must be at least 1")) + rows === nothing || first(rows) >= 1 || + throw(ArgumentError("rows must be a 1-based range")) + + # Pre-pass with the same projection, so schema and chunks line up. + pre = _source_meta(src, usecols) + + chunksize = Int(chunksize) + channel = Channel{ReadStatTable}(2; spawn=true) do ch + pc = ParseContext() + pc.usecols = _colselector(usecols) + pc.file_encoding = src.file_encoding + pc.handler_encoding = src.handler_encoding + pc.keep_user_missing = src.user_missing === :keep + if rows !== nothing + pc.row_offset = first(rows) - 1 + pc.row_limit = length(rows) + end + pc.preassigned_cols = true + cols = TypedColumns() + for vm in pre.varmeta + addcolumn!(cols, jltype(vm.type), chunksize) + end + pc.cols = cols + sink = ChunkSink(ch, chunksize, pre.names, pre.varmeta, pre.meta, + src.convert_datetime, src.apply_value_labels, src.format) + pc.chunk_sink = sink + parse_file!(pc, src.source, src.format) + pc.rows_complete > 0 && _flush_chunk!(pc, sink, pc.rows_complete) + return + end + return ReadStatChunks(channel) +end diff --git a/src/reader/columns.jl b/src/reader/columns.jl new file mode 100644 index 0000000..395c641 --- /dev/null +++ b/src/reader/columns.jl @@ -0,0 +1,174 @@ +# Typed column buffers filled by the value handler. +# +# Columns are parsed into raw `Vector{T}` + `Vector{Bool}` NA-mask pairs and +# only wrapped into `DataValueVector`s once parsing is done. This keeps the +# hot path free of any DataValues internals (whose representation may change) +# and lets a future multi-parser run fill disjoint row regions of shared +# buffers from several threads — plain `Vector` writes to disjoint indices +# are thread-safe, which would not hold for a shared bit-packed mask. +# +# The mask starts all-`true` (every cell missing until the C library delivers +# it), so cells the parser never emits surface as NA rather than as undefined +# memory. String buffers are backed by `""` so every slot is always a defined +# object. + +mutable struct ColumnBuf{T} + values::Vector{T} + isna::Vector{Bool} + # Tags of tagged missing values ('a'-'z', '\0' where untagged), allocated + # lazily the first time a tagged missing arrives in this column. + tags::Union{Nothing,Vector{Char}} +end + +newbuf(::Type{T}, n::Int) where {T} = ColumnBuf{T}(Vector{T}(undef, n), fill(true, n), nothing) +newbuf(::Type{String}, n::Int) = ColumnBuf{String}(fill("", n), fill(true, n), nothing) + +# Column storage grouped by element type, so every access from the value +# handler goes through a small typecode branch into fully type-stable code +# instead of a `Vector{Any}` of columns. `slots[i]` maps the i-th kept column +# (C `index_after_skipping` plus one) to `(typecode, index)` within the +# matching typed vector. +struct TypedColumns + strings::Vector{ColumnBuf{String}} + int8s::Vector{ColumnBuf{Int8}} + int16s::Vector{ColumnBuf{Int16}} + int32s::Vector{ColumnBuf{Int32}} + floats::Vector{ColumnBuf{Float32}} + doubles::Vector{ColumnBuf{Float64}} + slots::Vector{Tuple{UInt8,Int}} +end + +TypedColumns() = TypedColumns(ColumnBuf{String}[], ColumnBuf{Int8}[], ColumnBuf{Int16}[], + ColumnBuf{Int32}[], ColumnBuf{Float32}[], ColumnBuf{Float64}[], Tuple{UInt8,Int}[]) + +const CODE_STRING = 0x01 +const CODE_INT8 = 0x02 +const CODE_INT16 = 0x03 +const CODE_INT32 = 0x04 +const CODE_FLOAT = 0x05 +const CODE_DOUBLE = 0x06 + +# `n` is the preallocated length; pass 0 when the row count is unknown and +# the buffers grow row by row instead. +function addcolumn!(cols::TypedColumns, T::DataType, n::Int) + if T === String + push!(cols.strings, newbuf(String, n)) + push!(cols.slots, (CODE_STRING, length(cols.strings))) + elseif T === Int8 + push!(cols.int8s, newbuf(Int8, n)) + push!(cols.slots, (CODE_INT8, length(cols.int8s))) + elseif T === Int16 + push!(cols.int16s, newbuf(Int16, n)) + push!(cols.slots, (CODE_INT16, length(cols.int16s))) + elseif T === Int32 + push!(cols.int32s, newbuf(Int32, n)) + push!(cols.slots, (CODE_INT32, length(cols.int32s))) + elseif T === Float32 + push!(cols.floats, newbuf(Float32, n)) + push!(cols.slots, (CODE_FLOAT, length(cols.floats))) + elseif T === Float64 + push!(cols.doubles, newbuf(Float64, n)) + push!(cols.slots, (CODE_DOUBLE, length(cols.doubles))) + else + throw(ArgumentError("unsupported column type $T")) + end + return cols +end + +@inline function setvalue!(buf::ColumnBuf{T}, row::Int, v::T) where {T} + n = length(buf.values) + if row <= n + @inbounds buf.values[row] = v + @inbounds buf.isna[row] = false + elseif row == n + 1 + push!(buf.values, v) + push!(buf.isna, false) + buf.tags === nothing || push!(buf.tags, '\0') + else + throw(ArgumentError("out-of-order row index $row for column of length $n")) + end + return buf +end + +@inline function setmissing!(buf::ColumnBuf{T}, row::Int, tag::Char='\0') where {T} + n = length(buf.values) + if row <= n + @inbounds buf.isna[row] = true + elseif row == n + 1 + push!(buf.values, _navalue(T)) + push!(buf.isna, true) + buf.tags === nothing || push!(buf.tags, '\0') + else + throw(ArgumentError("out-of-order row index $row for column of length $n")) + end + if tag != '\0' + t = buf.tags + if t === nothing + t = fill('\0', length(buf.values)) + buf.tags = t + end + @inbounds t[row] = tag + end + return buf +end + +_navalue(::Type{T}) where {T<:Number} = zero(T) +_navalue(::Type{String}) = "" + +function trim!(buf::ColumnBuf, n::Int) + if length(buf.values) > n + resize!(buf.values, n) + resize!(buf.isna, n) + buf.tags === nothing || resize!(buf.tags, n) + end + return buf +end + +# Trim every buffer to `n` rows (used when a parse was stopped early and the +# preallocated or partially filled buffers extend past the last complete row). +function trimcolumns!(cols::TypedColumns, n::Int) + for group in (cols.strings, cols.int8s, cols.int16s, cols.int32s, cols.floats, cols.doubles) + foreach(buf -> trim!(buf, n), group) + end + return cols +end + +# Wrap a finished buffer without copying. The constructor takes ownership of +# both vectors; nothing may touch the ColumnBuf afterwards. +finalize_column(buf::ColumnBuf{T}) where {T} = DataValueVector{T}(buf.values, buf.isna) + +function getbuf(cols::TypedColumns, i::Int) + code, slot = cols.slots[i] + if code == CODE_STRING + cols.strings[slot] + elseif code == CODE_INT8 + cols.int8s[slot] + elseif code == CODE_INT16 + cols.int16s[slot] + elseif code == CODE_INT32 + cols.int32s[slot] + elseif code == CODE_FLOAT + cols.floats[slot] + else + cols.doubles[slot] + end +end + +finalize_column(cols::TypedColumns, i::Int) = finalize_column(getbuf(cols, i)) + +column_tags(cols::TypedColumns, i::Int) = getbuf(cols, i).tags + +# A per-task view for multi-task reads: shares the value and NA-mask vectors +# (each task writes only its disjoint row region) but gives every task its +# own lazily allocated tag vector, since concurrent lazy allocation on a +# shared ColumnBuf would race. Tags are merged after the tasks join. +function share_buffers(cols::TypedColumns) + TypedColumns( + [ColumnBuf{String}(b.values, b.isna, nothing) for b in cols.strings], + [ColumnBuf{Int8}(b.values, b.isna, nothing) for b in cols.int8s], + [ColumnBuf{Int16}(b.values, b.isna, nothing) for b in cols.int16s], + [ColumnBuf{Int32}(b.values, b.isna, nothing) for b in cols.int32s], + [ColumnBuf{Float32}(b.values, b.isna, nothing) for b in cols.floats], + [ColumnBuf{Float64}(b.values, b.isna, nothing) for b in cols.doubles], + cols.slots) +end diff --git a/src/reader/context.jl b/src/reader/context.jl new file mode 100644 index 0000000..b43dc43 --- /dev/null +++ b/src/reader/context.jl @@ -0,0 +1,65 @@ +# Per-parse state. One ParseContext per parser; nothing here is shared, so a +# future multi-parser run needs no synchronization beyond disjoint buffers. + +mutable struct ParseContext + meta::ReadStatMeta + names::Vector{Symbol} + varmeta::Vector{ReadStatVarMeta} + cols::TypedColumns + # Exception thrown by a handler, captured so it never unwinds through C; + # rethrown by the parse driver after readstat_parse returns. + err::Union{Nothing,Tuple{Any,Any}} + # Messages the C library reports through its error handler; surfaced as + # Julia warnings after the parse. + warnings::Vector{String} + + # Configuration for this parse: + usecols::Union{Nothing,Function} # (name::Symbol, index::Int) -> Bool + progress::Union{Nothing,Function} # (fraction::Float64) -> Bool (false aborts) + row_offset::Int # 0 = from the start + row_limit::Int # -1 = no limit + file_encoding::Union{Nothing,String} + handler_encoding::Union{Nothing,String} + collect_values::Bool + # true: SPSS user-defined missing values are kept as data (only system + # and tagged missings become NA); false: they collapse to NA. + keep_user_missing::Bool + # true: wrap value-labeled columns in a LabeledArray when building the table. + apply_value_labels::Bool + # true: decode columns whose display format is a date/time format. + convert_datetime::Bool + # Row index (0-based) of this parse's first row within the final table; + # nonzero only for the chunk contexts of a multi-task read, whose + # obs_index values are relative to their own row_offset. + row_base::Int + # true for chunk contexts whose `cols` were prepared up front (shared + # buffers): the variable handler must not allocate columns, only apply + # the usecols decision. + preassigned_cols::Bool + # A ChunkSink (see reader/chunks.jl) when this parse streams finished + # chunks into a Channel instead of building one table; nothing otherwise. + chunk_sink::Any + # the format being parsed (:dta, :sav, ...); selects the producer's + # date/time format tables. + file_format::Symbol # false for metadata-only parses + + # Progress-abort bookkeeping: the last fully delivered row, so a + # partially parsed table can be trimmed to complete rows. + aborted::Bool + rows_complete::Int +end + +ParseContext() = ParseContext(ReadStatMeta(), Symbol[], ReadStatVarMeta[], TypedColumns(), + nothing, String[], nothing, nothing, 0, -1, nothing, nothing, true, false, false, true, + 0, false, nothing, :none, false, 0) + +# Rows to preallocate per column: what the file reports, minus the offset, +# capped by the limit; 0 when unknown (buffers then grow row by row). +function alloc_rows(pc::ParseContext) + pc.collect_values || return 0 + rc = pc.meta.row_count + rc < 0 && return 0 + n = max(rc - pc.row_offset, 0) + pc.row_limit >= 0 && (n = min(n, pc.row_limit)) + return n +end diff --git a/src/reader/handlers.jl b/src/reader/handlers.jl new file mode 100644 index 0000000..a6b62e0 --- /dev/null +++ b/src/reader/handlers.jl @@ -0,0 +1,261 @@ +# Parse handler bodies. All of them are top-level functions whose @cfunction +# pointers are created once in __init__ and shared by every parse — per-parse +# state lives exclusively in the ParseContext passed through user_ctx. +# +# Every handler body is wrapped in try/catch: a Julia exception must never +# unwind through the C parser, so it is captured on the context and the +# handler returns READSTAT_HANDLER_ABORT; the parse driver rethrows it once +# readstat_parse has returned (with READSTAT_ERROR_USER_ABORT). + +function handle_metadata(metadata::MetadataPtr, ctx::Any)::Cint + pc = ctx::ParseContext + try + m = pc.meta + m.row_count = readstat_get_row_count(metadata) + m.var_count = readstat_get_var_count(metadata) + m.creation_time = Dates.unix2datetime(readstat_get_creation_time(metadata)) + m.modified_time = Dates.unix2datetime(readstat_get_modified_time(metadata)) + m.file_format_version = readstat_get_file_format_version(metadata) + m.is_64bit = readstat_get_file_format_is_64bit(metadata) + m.compression = readstat_get_compression(metadata) + m.endianness = readstat_get_endianness(metadata) + m.table_name = readstat_get_table_name(metadata) + m.file_label = readstat_get_file_label(metadata) + m.file_encoding = readstat_get_file_encoding(metadata) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function _missing_ranges(variable::VariablePtr) + n = readstat_variable_get_missing_ranges_count(variable) + ranges = Vector{Tuple{Any,Any}}(undef, n) + for i in 0:(n - 1) + lo = readstat_variable_get_missing_range_lo(variable, i) + hi = readstat_variable_get_missing_range_hi(variable, i) + ranges[i + 1] = (_native_value(lo), _native_value(hi)) + end + return ranges +end + +function _native_value(value::ReadStatValue) + t = readstat_value_type(value) + if t == READSTAT_TYPE_INT8 + readstat_int8_value(value) + elseif t == READSTAT_TYPE_INT16 + readstat_int16_value(value) + elseif t == READSTAT_TYPE_INT32 + readstat_int32_value(value) + elseif t == READSTAT_TYPE_FLOAT + readstat_float_value(value) + elseif t == READSTAT_TYPE_DOUBLE + readstat_double_value(value) + else + ptr = readstat_string_value(value) + ptr == C_NULL ? "" : unsafe_string(ptr) + end +end + +function handle_variable(index::Cint, variable::VariablePtr, val_labels::Cstring, ctx::Any)::Cint + pc = ctx::ParseContext + try + name = Symbol(readstat_variable_get_name(variable)) + sel = pc.usecols + if sel !== nothing && !(sel(name, Int(index) + 1)::Bool) + return READSTAT_HANDLER_SKIP_VARIABLE + end + # Chunk contexts of a multi-task read come with shared, fully + # prepared columns; only the skip decision above matters here. + pc.preassigned_cols && return READSTAT_HANDLER_OK + t = readstat_variable_get_type(variable) + vm = ReadStatVarMeta( + name, + readstat_variable_get_label(variable), + readstat_variable_get_format(variable), + t, + readstat_variable_get_type_class(variable), + val_labels == C_NULL ? Symbol("") : Symbol(unsafe_string(val_labels)), + Int(readstat_variable_get_storage_width(variable)), + Int(readstat_variable_get_display_width(variable)), + readstat_variable_get_measure(variable), + readstat_variable_get_alignment(variable), + _missing_ranges(variable)) + push!(pc.varmeta, vm) + push!(pc.names, vm.name) + addcolumn!(pc.cols, jltype(t), alloc_rows(pc)) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function handle_value(obs_index::Cint, variable::VariablePtr, value::ReadStatValue, ctx::Any)::Cint + pc = ctx::ParseContext + try + idx = readstat_variable_get_index_after_skipping(variable) + 1 + row = pc.row_base + Int(obs_index) + 1 + # A negative row base emulates a row offset for parsers that ignore + # readstat_set_row_offset (the fixed-width text parser). + row < 1 && return READSTAT_HANDLER_OK + cols = pc.cols + code, slot = @inbounds cols.slots[idx] + + # Three kinds of missing values (readstat.h): system missing and + # tagged missing (Stata/SAS .a-.z) always become NA, the latter with + # its tag recorded; SPSS user-defined missing values become NA by + # default but stay data under `user_missing=:keep`. + local miss::Bool, tag::Char + if readstat_value_is_tagged_missing(value) + miss = true + tag = readstat_value_tag(value) + else + tag = '\0' + miss = pc.keep_user_missing ? readstat_value_is_system_missing(value) : + readstat_value_is_missing(value, variable) + end + + if code == CODE_STRING + buf = @inbounds cols.strings[slot] + if miss + setmissing!(buf, row, tag) + else + ptr = readstat_string_value(value) + setvalue!(buf, row, ptr == C_NULL ? "" : unsafe_string(ptr)) + end + elseif code == CODE_INT8 + buf = @inbounds cols.int8s[slot] + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_int8_value(value)) + elseif code == CODE_INT16 + buf = @inbounds cols.int16s[slot] + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_int16_value(value)) + elseif code == CODE_INT32 + buf = @inbounds cols.int32s[slot] + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_int32_value(value)) + elseif code == CODE_FLOAT + buf = @inbounds cols.floats[slot] + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_float_value(value)) + else + buf = @inbounds cols.doubles[slot] + miss ? setmissing!(buf, row, tag) : setvalue!(buf, row, readstat_double_value(value)) + end + # Track the last fully delivered row so a parse stopped early (by the + # progress callback) can be trimmed to complete rows; in chunked + # streaming mode a completed chunk is flushed to its channel here. + if idx == length(cols.slots) + pc.rows_complete = row + sink = pc.chunk_sink + sink === nothing || _maybe_flush!(pc, sink, row) + end + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function handle_value_label(val_labels::Cstring, value::ReadStatValue, label::Cstring, ctx::Any)::Cint + pc = ctx::ParseContext + try + val_labels == C_NULL && return READSTAT_HANDLER_OK + name = Symbol(unsafe_string(val_labels)) + dict = get!(ValueLabelDict, pc.meta.value_labels, name) + key = if readstat_value_is_tagged_missing(value) + readstat_value_tag(value) + else + t = readstat_value_type(value) + if t == READSTAT_TYPE_INT8 + Int32(readstat_int8_value(value)) + elseif t == READSTAT_TYPE_INT16 + Int32(readstat_int16_value(value)) + elseif t == READSTAT_TYPE_INT32 + readstat_int32_value(value) + elseif t == READSTAT_TYPE_FLOAT + Float64(readstat_float_value(value)) + elseif t == READSTAT_TYPE_DOUBLE + readstat_double_value(value) + else + ptr = readstat_string_value(value) + ptr == C_NULL ? "" : unsafe_string(ptr) + end + end + dict[key] = label == C_NULL ? "" : unsafe_string(label) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function handle_note(note_index::Cint, note::Cstring, ctx::Any)::Cint + pc = ctx::ParseContext + try + note == C_NULL || push!(pc.meta.notes, unsafe_string(note)) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +function handle_fweight(variable::VariablePtr, ctx::Any)::Cint + pc = ctx::ParseContext + try + pc.meta.fweight = Symbol(readstat_variable_get_name(variable)) + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +# The C error handler reports warnings the parse survives; it returns void, +# so exceptions can neither abort nor propagate — swallow them. +function handle_error(message::Cstring, ctx::Any)::Cvoid + pc = ctx::ParseContext + try + message == C_NULL || push!(pc.warnings, unsafe_string(message)) + catch + end + return nothing +end + +function handle_progress(progress::Cdouble, ctx::Any)::Cint + pc = ctx::ParseContext + try + f = pc.progress + if f !== nothing && f(Float64(progress)) === false + pc.aborted = true + return READSTAT_HANDLER_ABORT + end + return READSTAT_HANDLER_OK + catch e + pc.err = (e, catch_backtrace()) + return READSTAT_HANDLER_ABORT + end +end + +# @cfunction pointers are runtime values, so they are created in __init__ and +# cached here rather than serialized into the precompile image. +const CF_METADATA = Ref(C_NULL) +const CF_VARIABLE = Ref(C_NULL) +const CF_VALUE = Ref(C_NULL) +const CF_VALUE_LABEL = Ref(C_NULL) +const CF_NOTE = Ref(C_NULL) +const CF_FWEIGHT = Ref(C_NULL) +const CF_ERROR = Ref(C_NULL) +const CF_PROGRESS = Ref(C_NULL) + +function _init_cfunctions() + CF_METADATA[] = @cfunction(handle_metadata, Cint, (MetadataPtr, Any)) + CF_VARIABLE[] = @cfunction(handle_variable, Cint, (Cint, VariablePtr, Cstring, Any)) + CF_VALUE[] = @cfunction(handle_value, Cint, (Cint, VariablePtr, ReadStatValue, Any)) + CF_VALUE_LABEL[] = @cfunction(handle_value_label, Cint, (Cstring, ReadStatValue, Cstring, Any)) + CF_NOTE[] = @cfunction(handle_note, Cint, (Cint, Cstring, Any)) + CF_FWEIGHT[] = @cfunction(handle_fweight, Cint, (VariablePtr, Any)) + CF_ERROR[] = @cfunction(handle_error, Cvoid, (Cstring, Any)) + CF_PROGRESS[] = @cfunction(handle_progress, Cint, (Cdouble, Any)) + return nothing +end diff --git a/src/reader/io.jl b/src/reader/io.jl new file mode 100644 index 0000000..92c742c --- /dev/null +++ b/src/reader/io.jl @@ -0,0 +1,88 @@ +# Parsing from Julia IO streams via the C library's custom IO handlers. +# +# The C parsers drive all reading through an open/close/seek/read handler +# quartet; installing Julia implementations lets any seekable IO (an open +# file, an IOBuffer over bytes from anywhere) be parsed without touching the +# file system. Non-seekable streams are slurped into an IOBuffer first — the +# parsers seek backwards routinely, so streaming through them directly is +# not possible. +# +# The stream must contain the complete file starting at position 0: seek +# offsets from the C side are absolute. + +mutable struct IOSource + io::IO + size::Int64 + err::Union{Nothing,Tuple{Any,Any}} +end + +# Buffer a non-seekable stream in memory; the parsers seek backwards. +function _ensure_seekable(io::IO) + seekable = try + seek(io, position(io)) + true + catch + false + end + return seekable ? io : IOBuffer(read(io)) +end + +function IOSource(io::IO) + io = _ensure_seekable(io) + seekend(io) + size = Int64(position(io)) + # Rewind so the same stream can be parsed repeatedly (ReadStatSource + # reads its schema first and the data later). + seekstart(io) + return IOSource(io, size, nothing) +end + +# The C side treats an open-handler return of -1 as failure; the stream is +# already open, so both open and close are no-ops. +handle_io_open(path::Cstring, io_ctx::Ptr{Cvoid})::Cint = Cint(0) +handle_io_close(io_ctx::Ptr{Cvoid})::Cint = Cint(0) + +# The offset and return type must match readstat_off_t as the C library was +# compiled (32-bit on 32-bit unix); a width mismatch shifts the callback's +# argument stack there. +function handle_io_seek(offset::Coff_t, whence::ReadStatIOFlags, io_ctx::Ptr{Cvoid})::Coff_t + src = unsafe_pointer_to_objref(io_ctx)::IOSource + try + pos = whence == READSTAT_SEEK_SET ? Int64(offset) : + whence == READSTAT_SEEK_CUR ? Int64(position(src.io)) + offset : + src.size + offset + (pos < 0 || pos > src.size) && return Coff_t(-1) + seek(src.io, pos) + return Coff_t(position(src.io)) + catch e + src.err = (e, catch_backtrace()) + return Coff_t(-1) + end +end + +function handle_io_read(buf::Ptr{Cvoid}, nbyte::Csize_t, io_ctx::Ptr{Cvoid})::Cssize_t + src = unsafe_pointer_to_objref(io_ctx)::IOSource + try + io = src.io + n = min(Int(nbyte), max(src.size - Int64(position(io)), 0)) + n <= 0 && return Cssize_t(0) + unsafe_read(io, Ptr{UInt8}(buf), UInt(n)) + return Cssize_t(n) + catch e + src.err = (e, catch_backtrace()) + return Cssize_t(-1) + end +end + +const CF_IO_OPEN = Ref(C_NULL) +const CF_IO_CLOSE = Ref(C_NULL) +const CF_IO_SEEK = Ref(C_NULL) +const CF_IO_READ = Ref(C_NULL) + +function _init_io_cfunctions() + CF_IO_OPEN[] = @cfunction(handle_io_open, Cint, (Cstring, Ptr{Cvoid})) + CF_IO_CLOSE[] = @cfunction(handle_io_close, Cint, (Ptr{Cvoid},)) + CF_IO_SEEK[] = @cfunction(handle_io_seek, Coff_t, (Coff_t, ReadStatIOFlags, Ptr{Cvoid})) + CF_IO_READ[] = @cfunction(handle_io_read, Cssize_t, (Ptr{Cvoid}, Csize_t, Ptr{Cvoid})) + return nothing +end diff --git a/src/reader/metadata.jl b/src/reader/metadata.jl new file mode 100644 index 0000000..e0fdd9d --- /dev/null +++ b/src/reader/metadata.jl @@ -0,0 +1,68 @@ +# File-level and per-variable metadata captured during a parse. + +""" + ValueLabelDict + +One value-label set: a dictionary mapping raw values to their display +labels. Keys are normalized to `Int32` (all integer types), `Float64` (all +floats), `String`, or — for labeled tagged missing values like Stata's +`.a` — the tag `Char`. +""" +const ValueLabelDict = Dict{Union{Char,Int32,Float64,String},String} + +""" + ReadStatVarMeta + +Per-variable metadata from a stat-package file: `name`, `label`, `format` +(the producer's display format string), `type`/`type_class` (the raw C +storage type), `vallabel` (name of the value-label set, `Symbol("")` when +none), `storage_width`, `display_width`, `measure`, `alignment`, and +`missing_ranges` (SPSS user-defined missing values as `(lo, hi)` pairs, where +a single missing value has `lo == hi`). +""" +struct ReadStatVarMeta + name::Symbol + label::String + format::String + type::ReadStatType + type_class::ReadStatTypeClass + vallabel::Symbol + storage_width::Int + display_width::Int + measure::ReadStatMeasure + alignment::ReadStatAlignment + missing_ranges::Vector{Tuple{Any,Any}} +end + +""" + ReadStatMeta + +File-level metadata: `row_count` (`-1` when the format does not record it — +XPORT, POR, and some non-conforming SAV files; when a read used a +`row_limit`, the C library caps the reported count at that limit — use +[`read_meta`](@ref) for the true count), `var_count`, `creation_time` +and `modified_time`, `file_format_version`, `is_64bit` (SAS), `compression`, +`endianness`, `table_name` (XPORT), `file_label`, `file_encoding`, `notes`, +`fweight` (frequency-weight variable, `Symbol("")` when none), and +`value_labels` (label-set name => [`ValueLabelDict`](@ref)). +""" +mutable struct ReadStatMeta + row_count::Int + var_count::Int + creation_time::DateTime + modified_time::DateTime + file_format_version::Int + is_64bit::Bool + compression::ReadStatCompress + endianness::ReadStatEndian + table_name::String + file_label::String + file_encoding::String + notes::Vector{String} + fweight::Symbol + value_labels::Dict{Symbol,ValueLabelDict} +end + +ReadStatMeta() = ReadStatMeta(-1, 0, Dates.unix2datetime(0), Dates.unix2datetime(0), 0, + false, READSTAT_COMPRESS_NONE, READSTAT_ENDIAN_NONE, "", "", "", String[], + Symbol(""), Dict{Symbol,ValueLabelDict}()) diff --git a/src/reader/read.jl b/src/reader/read.jl new file mode 100644 index 0000000..003e07f --- /dev/null +++ b/src/reader/read.jl @@ -0,0 +1,406 @@ +# Parse driver and the public read entry points. + +const FORMAT_BY_EXT = Dict( + ".dta" => :dta, + ".sav" => :sav, + ".zsav" => :sav, + ".por" => :por, + ".sas7bdat" => :sas7bdat, + ".xpt" => :xport, + ".xport" => :xport, +) + +function _sniff_format(path::AbstractString, format::Symbol) + format === :auto || return format + ext = lowercase(splitext(path)[2]) + fmt = get(FORMAT_BY_EXT, ext, nothing) + fmt === nothing && + throw(ArgumentError("cannot infer the file format from extension \"$ext\"; pass `format=...`")) + return fmt +end + +function _sniff_format(io::IO, format::Symbol) + format === :auto && + throw(ArgumentError("the file format cannot be inferred from an IO stream; pass `format=...`")) + return format +end + +function _parse_format(parser::ParserPtr, path::AbstractString, format::Symbol, ctx) + if format === :dta + CAPI.readstat_parse_dta(parser, path, ctx) + elseif format === :sav + CAPI.readstat_parse_sav(parser, path, ctx) + elseif format === :por + CAPI.readstat_parse_por(parser, path, ctx) + elseif format === :sas7bdat + CAPI.readstat_parse_sas7bdat(parser, path, ctx) + elseif format === :xport + CAPI.readstat_parse_xport(parser, path, ctx) + elseif format === :sas7bcat + CAPI.readstat_parse_sas7bcat(parser, path, ctx) + else + throw(ArgumentError("unknown format $format")) + end +end + +# Normalize the `usecols` kwarg into a `(name, index) -> Bool` predicate. +_colselector(::Nothing) = nothing +_colselector(s::Symbol) = (name, i) -> name === s +_colselector(idx::Integer) = let idx = Int(idx); (name, i) -> i == idx; end +_colselector(v::AbstractVector{Symbol}) = let s = Set(v); (name, i) -> name in s; end +_colselector(v::AbstractVector{<:Integer}) = let s = Set{Int}(v); (name, i) -> i in s; end +_colselector(r::Regex) = (name, i) -> occursin(r, String(name)) +_colselector(f::Function) = (name, i) -> f(name)::Bool + +function _set_handlers!(parser::ParserPtr, pc::ParseContext) + readstat_set_metadata_handler(parser, CF_METADATA[]) + readstat_set_variable_handler(parser, CF_VARIABLE[]) + readstat_set_value_label_handler(parser, CF_VALUE_LABEL[]) + readstat_set_note_handler(parser, CF_NOTE[]) + readstat_set_fweight_handler(parser, CF_FWEIGHT[]) + readstat_set_error_handler(parser, CF_ERROR[]) + pc.collect_values && readstat_set_value_handler(parser, CF_VALUE[]) + pc.progress === nothing || readstat_set_progress_handler(parser, CF_PROGRESS[]) + pc.row_offset > 0 && readstat_set_row_offset(parser, pc.row_offset) + pc.row_limit >= 0 && readstat_set_row_limit(parser, pc.row_limit) + pc.file_encoding === nothing || + readstat_set_file_character_encoding(parser, pc.file_encoding) + pc.handler_encoding === nothing || + readstat_set_handler_character_encoding(parser, pc.handler_encoding) + return parser +end + +function _finish_parse(pc::ParseContext, retval::ReadStatError, what::AbstractString) + if pc.err !== nothing + e, _ = pc.err + throw(e) + end + for w in pc.warnings + @warn "readstat: $w" _module = ReadStat _file = what + end + if retval == READSTAT_ERROR_USER_ABORT && pc.aborted + # The progress callback stopped the parse; the caller gets the rows + # delivered so far. + elseif retval != READSTAT_OK + error("Error parsing $what: $(readstat_error_message(retval))") + end + return pc +end + +function parse_file!(pc::ParseContext, path::AbstractString, format::Symbol) + isfile(path) || throw(ArgumentError("file not found: $path")) + parser = readstat_parser_init() + local retval + try + _set_handlers!(parser, pc) + retval = _parse_format(parser, path, format, pc) + finally + readstat_parser_free(parser) + end + return _finish_parse(pc, retval, path) +end + +function parse_file!(pc::ParseContext, io::IO, format::Symbol) + src = IOSource(io) + parser = readstat_parser_init() + local retval + try + _set_handlers!(parser, pc) + readstat_set_open_handler(parser, CF_IO_OPEN[]) + readstat_set_close_handler(parser, CF_IO_CLOSE[]) + readstat_set_seek_handler(parser, CF_IO_SEEK[]) + readstat_set_read_handler(parser, CF_IO_READ[]) + GC.@preserve src begin + readstat_set_io_ctx(parser, pointer_from_objref(src)) + retval = _parse_format(parser, "", format, pc) + end + finally + readstat_parser_free(parser) + end + if src.err !== nothing + e, _ = src.err + throw(e) + end + return _finish_parse(pc, retval, "IO stream") +end + +function build_table(pc::ParseContext) + pc.aborted && trimcolumns!(pc.cols, pc.rows_complete) + n = length(pc.names) + columns = Vector{AbstractVector}(undef, n) + tags = Vector{Union{Nothing,Vector{Char}}}(undef, n) + for i in 1:n + col = finalize_column(pc.cols, i) + vm = pc.varmeta[i] + labels = vm.vallabel === Symbol("") ? nothing : + get(pc.meta.value_labels, vm.vallabel, nothing) + # Value labels take precedence over a date/time display format: a + # labeled column holds codes, not calendar values. + if labels === nothing && pc.convert_datetime && eltype(eltype(col)) <: Number + rule = datetime_rule(vm.format, pc.file_format) + if rule !== nothing + f, target = rule + col = convert_datetime_column(col, f, target) + end + end + if pc.apply_value_labels && labels !== nothing + col = LabeledArray(col, labels) + end + columns[i] = col + tags[i] = column_tags(pc.cols, i) + end + return ReadStatTable(columns, pc.names, pc.meta, pc.varmeta, tags) +end + +function read_data_file(source::Union{AbstractString,IO}, format::Symbol; + usecols=nothing, + row_limit::Union{Nothing,Integer}=nothing, + row_offset::Integer=0, + file_encoding::Union{Nothing,AbstractString}=nothing, + handler_encoding::Union{Nothing,AbstractString}=nothing, + user_missing::Symbol=:na, + convert_datetime::Bool=true, + apply_value_labels::Bool=false, + catalog::Union{Nothing,AbstractString}=nothing, + ntasks::Union{Nothing,Integer}=nothing, + progress=nothing) + user_missing in (:na, :keep) || + throw(ArgumentError("user_missing must be :na or :keep")) + row_offset >= 0 || throw(ArgumentError("row_offset must be non-negative")) + row_limit === nothing || row_limit >= 0 || + throw(ArgumentError("row_limit must be non-negative")) + pc = ParseContext() + pc.usecols = _colselector(usecols) + pc.progress = progress + pc.row_offset = Int(row_offset) + if row_limit == 0 + # The C library treats a row limit of 0 as "no limit", so a zero-row + # read is done by not collecting values at all. No C row limit is set + # either: the sav parser does not reliably deliver value labels when + # one is in effect. + pc.collect_values = false + else + pc.row_limit = row_limit === nothing ? -1 : Int(row_limit) + end + pc.file_encoding = file_encoding === nothing ? nothing : String(file_encoding) + pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) + pc.keep_user_missing = user_missing === :keep + pc.apply_value_labels = apply_value_labels + pc.convert_datetime = convert_datetime + pc.file_format = format + catalog === nothing || format === :sas7bdat || + throw(ArgumentError("`catalog` is only supported when reading sas7bdat files")) + + parsed = false + if source isa AbstractString && pc.collect_values && progress === nothing && + format in _THREADED_FORMATS + T = ntasks === nothing ? _auto_ntasks(source) : max(Int(ntasks), 1) + T > 1 && (parsed = threaded_parse!(pc, source, format, T)) + end + parsed || parse_file!(pc, source, format) + catalog === nothing || merge!(pc.meta.value_labels, read_sas7bcat(catalog)) + return build_table(pc) +end + +const _READ_KWARGS_DOC = """ +Instead of a path, every reader also accepts an `IO` containing the complete +file (a non-seekable stream is buffered in memory first). The `progress` +callback only fires for path input. + +All readers accept the same keyword arguments: + +- `usecols`: read only these columns — a `Symbol`, column index, vector of + either, `Regex`, or a predicate called with each column name. Skipped + columns are never parsed (the C library skips them). +- `row_limit`: read at most this many rows. +- `row_offset`: skip this many rows from the start. +- `file_encoding`: override the character encoding declared in the file + (an iconv-compatible name such as `"WINDOWS-1252"`). +- `handler_encoding`: the encoding delivered to Julia; defaults to UTF-8. +- `user_missing`: `:na` (default) collapses SPSS user-defined missing values + to NA; `:keep` keeps them as data (the rules stay available in + `varmetadata(tbl, col).missing_ranges`). System missing and tagged missing + values are always NA; see [`missingtags`](@ref) for the tags. +- `convert_datetime`: decode columns whose display format is a date/time + format into `Date`/`DateTime`/[`HMS`](@ref) columns (default `true`); + `false` keeps the raw numbers. Value-labeled columns are never converted. +- `apply_value_labels`: when `true`, every value-labeled column is wrapped + in a [`LabeledArray`](@ref) (labels for display, codes for computation). + The raw label dictionaries are always available via [`valuelabels`](@ref) + regardless. +- `catalog` (sas7bdat only): path to the `.sas7bcat` catalog holding the + file's value labels; they are merged into the table's value labels. +- `ntasks`: number of parallel parsers for path input to formats that record + a row count (`.dta`, `.sav`, `.sas7bdat`): the row range is split into + contiguous chunks that parse concurrently into shared buffers. By default + large files use up to 8 threads and small files parse serially; pass an + integer to force a task count, or `1` to disable. +- `progress`: a function called with the parse fraction (0.0-1.0); return + `false` to stop the parse and get the rows read so far (disables + `ntasks`). +""" + +""" + read_dta(path; kwargs...) -> ReadStatTable + +Read a Stata `.dta` file. See [`ReadStatTable`](@ref) for how to access the +data and metadata. + +$_READ_KWARGS_DOC +""" +read_dta(source::Union{AbstractString,IO}; kwargs...) = read_data_file(source, :dta; kwargs...) + +""" + read_sav(path; kwargs...) -> ReadStatTable + +Read an SPSS `.sav` (or `.zsav`) file. + +$_READ_KWARGS_DOC +""" +read_sav(source::Union{AbstractString,IO}; kwargs...) = read_data_file(source, :sav; kwargs...) + +""" + read_por(path; kwargs...) -> ReadStatTable + +Read an SPSS portable `.por` file. The format does not record a row count, +so `filemetadata(tbl).row_count` is `-1`. + +$_READ_KWARGS_DOC +""" +read_por(source::Union{AbstractString,IO}; kwargs...) = read_data_file(source, :por; kwargs...) + +""" + read_sas7bdat(path; kwargs...) -> ReadStatTable + +Read a SAS `.sas7bdat` data file. + +$_READ_KWARGS_DOC +""" +read_sas7bdat(source::Union{AbstractString,IO}; kwargs...) = + read_data_file(source, :sas7bdat; kwargs...) + +""" + read_xport(path; kwargs...) -> ReadStatTable + +Read a SAS transport (XPORT) `.xpt` file. The format does not record a row +count, so `filemetadata(tbl).row_count` is `-1`. + +$_READ_KWARGS_DOC +""" +read_xport(source::Union{AbstractString,IO}; kwargs...) = + read_data_file(source, :xport; kwargs...) + +""" + readstat(path; format=:auto, kwargs...) -> ReadStatTable + +Read a stat-package data file, inferring the format from the file extension +(`.dta`, `.sav`/`.zsav`, `.por`, `.sas7bdat`, `.xpt`/`.xport`) unless +`format` is given explicitly (`:dta`, `:sav`, `:por`, `:sas7bdat`, +`:xport`). For `IO` input the format cannot be inferred, so `format` is +required. + +$_READ_KWARGS_DOC +""" +readstat(source::Union{AbstractString,IO}; format::Symbol=:auto, kwargs...) = + read_data_file(source, _sniff_format(source, format); kwargs...) + +""" + read_sas7bcat(path) -> Dict{Symbol, ValueLabelDict} + +Read the value-label sets from a SAS `.sas7bcat` catalog file, keyed by +format name. Usually not called directly — pass the catalog path to +`read_sas7bdat(...; catalog=...)` to attach the labels to a data file. +""" +function read_sas7bcat(path::AbstractString) + pc = ParseContext() + pc.collect_values = false + parse_file!(pc, path, :sas7bcat) + return pc.meta.value_labels +end + +const _SCHEMA_PRODUCER = Dict(:sas_commands => :sas7bdat, :spss_commands => :sav, + :stata_dictionary => :dta) + +""" + read_txt(data_path, schema_path; schema_format, kwargs...) -> ReadStatTable + +Read a fixed-width text data file described by a schema file: +`schema_format` selects the schema flavor — `:sas_commands` (a SAS program +with an `INPUT` statement), `:spss_commands` (SPSS `DATA LIST` syntax), or +`:stata_dictionary` (a Stata `.dct` file). Supports the `usecols`, +`row_limit`, `row_offset`, `handler_encoding`, and `convert_datetime` +keyword arguments with the same meaning as the binary readers (date/time +formats are interpreted per the schema's producer). +""" +function read_txt(data_path::AbstractString, schema_path::AbstractString; + schema_format::Symbol, + usecols=nothing, + row_limit::Union{Nothing,Integer}=nothing, + row_offset::Integer=0, + handler_encoding::Union{Nothing,AbstractString}=nothing, + convert_datetime::Bool=true) + haskey(_SCHEMA_PRODUCER, schema_format) || + throw(ArgumentError("schema_format must be :sas_commands, :spss_commands, or :stata_dictionary")) + isfile(data_path) || throw(ArgumentError("file not found: $data_path")) + isfile(schema_path) || throw(ArgumentError("file not found: $schema_path")) + + pc = ParseContext() + pc.usecols = _colselector(usecols) + # The text parser ignores readstat_set_row_offset, so the offset is + # emulated by shifting the row base negative and widening the limit. + offset = Int(row_offset) + pc.row_base = -offset + pc.row_limit = row_limit === nothing ? -1 : Int(row_limit) + offset + pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) + pc.convert_datetime = convert_datetime + pc.file_format = _SCHEMA_PRODUCER[schema_format] + + parser = readstat_parser_init() + schema = SchemaPtr(C_NULL) + local retval + try + _set_handlers!(parser, pc) + out_error = Ref(READSTAT_OK) + schema = if schema_format === :sas_commands + CAPI.readstat_parse_sas_commands(parser, schema_path, pc, out_error) + elseif schema_format === :spss_commands + CAPI.readstat_parse_spss_commands(parser, schema_path, pc, out_error) + else + CAPI.readstat_parse_stata_dictionary(parser, schema_path, pc, out_error) + end + if schema == C_NULL || out_error[] != READSTAT_OK + error("Error parsing schema $schema_path: $(readstat_error_message(out_error[]))") + end + retval = CAPI.readstat_parse_txt(parser, data_path, schema, pc) + finally + schema == C_NULL || CAPI.readstat_schema_free(schema) + readstat_parser_free(parser) + end + _finish_parse(pc, retval, data_path) + return build_table(pc) +end + +""" + read_meta(path; format=:auto, file_encoding=nothing, handler_encoding=nothing) + -> ReadStatTable + +Read only the metadata of a stat-package data file: the returned table has +zero rows, but its [`filemetadata`](@ref), [`varmetadata`](@ref), and +[`valuelabels`](@ref) are fully populated, including the row count recorded +in the file (`filemetadata(tbl).row_count`). This is much cheaper than +reading the data. +""" +function read_meta(source::Union{AbstractString,IO}; format::Symbol=:auto, + file_encoding::Union{Nothing,AbstractString}=nothing, + handler_encoding::Union{Nothing,AbstractString}=nothing) + pc = ParseContext() + pc.collect_values = false + # No row limit here: the C library caps the row count it reports at any + # row limit in effect, and read_meta must report the true count. + pc.file_encoding = file_encoding === nothing ? nothing : String(file_encoding) + pc.handler_encoding = handler_encoding === nothing ? nothing : String(handler_encoding) + fmt = _sniff_format(source, format) + pc.file_format = fmt + parse_file!(pc, source, fmt) + return build_table(pc) +end diff --git a/src/reader/source.jl b/src/reader/source.jl new file mode 100644 index 0000000..87f03b9 --- /dev/null +++ b/src/reader/source.jl @@ -0,0 +1,157 @@ +# A lazy handle over a stat-package file, designed for consumers that push +# work down into the reader — a display layer that wants ten rows, or a +# query engine that plans against the schema and then reads a projection of +# a row range, possibly in chunks. + +""" + ReadStatSource(path_or_io; format=:auto, kwargs...) + +A cheap, lazy handle over a stat-package data file: constructing one parses +nothing. [`schema`](@ref) returns (and caches) the metadata-only table; +[`colnames`](@ref), [`coltypes`](@ref), and [`nrows`](@ref) answer planning +questions from it; `read(src; usecols=..., rows=...)` reads a projection of +a row range; and [`chunks`](@ref) streams the data in table-sized pieces. +[`supports`](@ref) reports which pushdowns the underlying format can honor. + +The keyword arguments (`convert_datetime`, `apply_value_labels`, +`user_missing`, `file_encoding`, `handler_encoding`, `catalog`) fix how this +source presents its data; every read through the source applies them. For +`IO` input the stream must contain the complete file and `format` is +required; a non-seekable stream is buffered in memory once. +""" +mutable struct ReadStatSource + source::Union{String,IO} + format::Symbol + convert_datetime::Bool + apply_value_labels::Bool + user_missing::Symbol + file_encoding::Union{Nothing,String} + handler_encoding::Union{Nothing,String} + catalog::Union{Nothing,String} + schema::Union{Nothing,ReadStatTable} +end + +function ReadStatSource(source::Union{AbstractString,IO}; format::Symbol=:auto, + convert_datetime::Bool=true, apply_value_labels::Bool=false, + user_missing::Symbol=:na, + file_encoding::Union{Nothing,AbstractString}=nothing, + handler_encoding::Union{Nothing,AbstractString}=nothing, + catalog::Union{Nothing,AbstractString}=nothing) + fmt = _sniff_format(source, format) + user_missing in (:na, :keep) || + throw(ArgumentError("user_missing must be :na or :keep")) + src = source isa AbstractString ? String(source) : _ensure_seekable(source) + return ReadStatSource(src, fmt, convert_datetime, apply_value_labels, user_missing, + file_encoding === nothing ? nothing : String(file_encoding), + handler_encoding === nothing ? nothing : String(handler_encoding), + catalog === nothing ? nothing : String(catalog), + nothing) +end + +# Metadata-only parse honoring the source's presentation options (and an +# optional projection, used by `chunks` so its schema matches its chunks). +function _source_meta(src::ReadStatSource, usecols) + pc = ParseContext() + pc.collect_values = false + pc.usecols = _colselector(usecols) + pc.file_encoding = src.file_encoding + pc.handler_encoding = src.handler_encoding + pc.convert_datetime = src.convert_datetime + pc.apply_value_labels = src.apply_value_labels + pc.file_format = src.format + parse_file!(pc, src.source, src.format) + src.catalog === nothing || merge!(pc.meta.value_labels, read_sas7bcat(src.catalog)) + return pc +end + +""" + schema(src::ReadStatSource) -> ReadStatTable + +The metadata of the source as a zero-row table (cached after the first +call): column names and types, file- and variable-level metadata, value +labels, and notes — everything a consumer needs to plan a read without +touching the data. +""" +function schema(src::ReadStatSource) + if src.schema === nothing + src.schema = build_table(_source_meta(src, nothing)) + end + return src.schema::ReadStatTable +end + +""" + colnames(src::ReadStatSource) -> Vector{Symbol} + +The column names of the source (from the cached [`schema`](@ref)). +""" +colnames(src::ReadStatSource) = names(schema(src)) + +""" + coltypes(src::ReadStatSource) -> Vector{Type} + +The element types the source's columns will have when read — after date/time +conversion and value-label application per the source's options (e.g. +`DataValue{Float64}`, `DataValue{Date}`). +""" +coltypes(src::ReadStatSource) = Type[eltype(schema(src)[i]) for i in 1:size(schema(src), 2)] + +""" + nrows(src::ReadStatSource) -> Union{Int, Missing} + +The number of rows recorded in the file, or `missing` when the format does +not record one (XPORT, POR, some non-conforming SAV files). +""" +function nrows(src::ReadStatSource) + rc = filemetadata(schema(src)).row_count + return rc < 0 ? missing : rc +end + +""" + supports(src::ReadStatSource, feature::Symbol) -> Bool + +Whether the source can honor a pushdown feature: + +- `:projection`: column selection during the parse (always true). +- `:row_range`: `rows=` ranges via the C row offset/limit (always true). +- `:row_count`: a row count known without reading the data. +- `:parallel`: multi-task reading (path input to a format that records its + row count). +""" +function supports(src::ReadStatSource, feature::Symbol) + feature === :projection && return true + feature === :row_range && return true + feature === :row_count && return !ismissing(nrows(src)) + feature === :parallel && + return src.source isa String && src.format in _THREADED_FORMATS + return false +end + +""" + read(src::ReadStatSource; usecols=nothing, rows=nothing, ntasks=nothing, + progress=nothing) -> ReadStatTable + +Read the source, optionally pushing down a column projection (`usecols`, as +in the readers) and a 1-based row range (`rows`, a `UnitRange`); both are +applied inside the C parse. All other behavior follows the options fixed at +[`ReadStatSource`](@ref) construction. +""" +function Base.read(src::ReadStatSource; usecols=nothing, + rows::Union{Nothing,AbstractUnitRange{<:Integer}}=nothing, + ntasks::Union{Nothing,Integer}=nothing, progress=nothing) + row_offset = 0 + row_limit = nothing + if rows !== nothing + first(rows) >= 1 || throw(ArgumentError("rows must be a 1-based range")) + row_offset = first(rows) - 1 + row_limit = length(rows) + end + return read_data_file(src.source, src.format; usecols, row_limit, row_offset, + file_encoding=src.file_encoding, handler_encoding=src.handler_encoding, + user_missing=src.user_missing, convert_datetime=src.convert_datetime, + apply_value_labels=src.apply_value_labels, catalog=src.catalog, ntasks, progress) +end + +function Base.show(io::IO, src::ReadStatSource) + what = src.source isa String ? repr(src.source) : "" + print(io, "ReadStatSource(", what, ", ", src.format, ")") +end diff --git a/src/reader/table.jl b/src/reader/table.jl new file mode 100644 index 0000000..a1066e3 --- /dev/null +++ b/src/reader/table.jl @@ -0,0 +1,159 @@ +""" + ReadStatTable + +The result of reading a stat-package file: a set of named `DataValueVector` +columns plus the file's complete metadata. + +Columns are accessed by position (`tbl[1]`) or name (`tbl[:price]`); +`names(tbl)` lists the column names and `size(tbl)` returns +`(rows, columns)`. File-level metadata is available via +[`filemetadata`](@ref), per-variable metadata via [`varmetadata`](@ref), and +value-label sets via [`valuelabels`](@ref). +""" +struct ReadStatTable + cols::Vector{AbstractVector} + names::Vector{Symbol} + lookup::Dict{Symbol,Int} + meta::ReadStatMeta + colmeta::Vector{ReadStatVarMeta} + # Per-column tags of tagged missing values ('a'-'z'; '\0' where untagged), + # allocated lazily only for columns that contain any. + tags::Vector{Union{Nothing,Vector{Char}}} +end + +function ReadStatTable(cols::Vector{AbstractVector}, names::Vector{Symbol}, + meta::ReadStatMeta, colmeta::Vector{ReadStatVarMeta}, + tags::Vector{Union{Nothing,Vector{Char}}}= + Union{Nothing,Vector{Char}}[nothing for _ in names]) + lookup = Dict{Symbol,Int}(name => i for (i, name) in enumerate(names)) + return ReadStatTable(cols, names, lookup, meta, colmeta, tags) +end + +ncols(tbl::ReadStatTable) = length(getfield(tbl, :cols)) +nrows(tbl::ReadStatTable) = ncols(tbl) == 0 ? 0 : length(getfield(tbl, :cols)[1]) + +Base.names(tbl::ReadStatTable) = getfield(tbl, :names) +Base.size(tbl::ReadStatTable) = (nrows(tbl), ncols(tbl)) +function Base.size(tbl::ReadStatTable, dim::Integer) + dim == 1 ? nrows(tbl) : dim == 2 ? ncols(tbl) : + throw(ArgumentError("dimension must be 1 or 2")) +end + +function columnindex(tbl::ReadStatTable, name::Symbol) + i = get(getfield(tbl, :lookup), name, 0) + i == 0 && throw(ArgumentError("no column named $name")) + return i +end +columnindex(tbl::ReadStatTable, i::Integer) = Int(i) + +Base.getindex(tbl::ReadStatTable, i::Integer) = getfield(tbl, :cols)[i] +Base.getindex(tbl::ReadStatTable, name::Symbol) = getfield(tbl, :cols)[columnindex(tbl, name)] + +""" + filemetadata(tbl::ReadStatTable) -> ReadStatMeta + +The file-level metadata of the table: row/variable counts as recorded in the +file, timestamps, format version, compression, endianness, file label, +encoding, notes, and all value-label sets. +""" +filemetadata(tbl::ReadStatTable) = getfield(tbl, :meta) + +""" + varmetadata(tbl::ReadStatTable, col) -> ReadStatVarMeta + +Per-variable metadata for the column given by index or name: variable label, +display format, raw storage type, value-label set name, widths, measure, +alignment, and SPSS missing-value rules. +""" +varmetadata(tbl::ReadStatTable, col::Union{Integer,Symbol}) = + getfield(tbl, :colmeta)[columnindex(tbl, col)] + +""" + valuelabels(tbl::ReadStatTable, col) -> Union{Nothing, ValueLabelDict} + +The value-label dictionary attached to the column given by index or name, or +`nothing` when the column has no value labels. The returned dictionary maps +raw values (and `Char` tags of labeled tagged missing values) to their +labels. +""" +function valuelabels(tbl::ReadStatTable, col::Union{Integer,Symbol}) + vm = varmetadata(tbl, col) + vm.vallabel === Symbol("") && return nothing + return get(filemetadata(tbl).value_labels, vm.vallabel, nothing) +end + +""" + labeled(tbl::ReadStatTable, col) -> LabeledArray + +A lazy labeled view of the value-labeled column given by index or name: the +elements display as their labels but compute as their raw codes (see +[`LabeledValue`](@ref)). Throws when the column has no value labels; reading +with `apply_value_labels=true` wraps every labeled column this way up front. +""" +function labeled(tbl::ReadStatTable, col::Union{Integer,Symbol}) + i = columnindex(tbl, col) + c = getfield(tbl, :cols)[i] + c isa LabeledArray && return c + d = valuelabels(tbl, i) + d === nothing && + throw(ArgumentError("column $(getfield(tbl, :names)[i]) has no value labels")) + return LabeledArray(c, d) +end + +""" + missingtags(tbl::ReadStatTable, col) -> Union{Nothing, Vector{Char}} + +Tags of the tagged missing values (Stata/SAS `.a`-`.z`) in the column given +by index or name: a vector with one `Char` per row, `'a'`-`'z'` where the +cell is a tagged missing value and `'\\0'` everywhere else. Returns `nothing` +when the column contains no tagged missing values. +""" +missingtags(tbl::ReadStatTable, col::Union{Integer,Symbol}) = + getfield(tbl, :tags)[columnindex(tbl, col)] + +############################################################################## +## +## Display +## +############################################################################## + +function Base.show(io::IO, tbl::ReadStatTable) + r, c = size(tbl) + print(io, "$(r)x$(c) ReadStatTable") +end + +function Base.show(io::IO, ::MIME"text/plain", tbl::ReadStatTable) + r, c = size(tbl) + meta = filemetadata(tbl) + print(io, "$(r)x$(c) ReadStatTable") + isempty(meta.file_label) || print(io, ": ", meta.file_label) + c == 0 && return + + maxrows = min(r, get(io, :limit, true) ? 10 : r) + maxcols = min(c, 20) + cells = Matrix{String}(undef, maxrows + 1, maxcols) + for j in 1:maxcols + cells[1, j] = string(names(tbl)[j]) + col = tbl[j] + for i in 1:maxrows + cells[i + 1, j] = _showcell(col[i]) + end + end + widths = [maximum(textwidth, view(cells, :, j)) for j in 1:maxcols] + for i in 1:(maxrows + 1) + print(io, "\n ") + for j in 1:maxcols + print(io, lpad(cells[i, j], widths[j])) + j < maxcols && print(io, " ") + end + c > maxcols && print(io, " …") + if i == 1 + print(io, "\n ", join([repeat("─", w) for w in widths], " ")) + end + end + maxrows < r && print(io, "\n â‹® ($(r - maxrows) more rows)") + return +end + +_showcell(v) = sprint(print, v) +_showcell(v::DataValues.DataValue) = DataValues.isna(v) ? "NA" : sprint(print, get(v)) diff --git a/src/reader/threaded.jl b/src/reader/threaded.jl new file mode 100644 index 0000000..bd85df6 --- /dev/null +++ b/src/reader/threaded.jl @@ -0,0 +1,124 @@ +# Multi-task reading: split the row range into contiguous chunks, one C +# parser per task, all writing directly into shared preallocated column +# buffers (disjoint row regions of plain Vectors, which is thread-safe). +# There are no per-task buffers to merge and no chained columns afterwards; +# the only post-join work is combining the rare tagged-missing tag vectors. +# +# A metadata pre-pass (no value handler, no row limit) supplies the true row +# count, the schema for preallocation, and the value labels and notes — the +# C parsers do not reliably deliver value labels under a row limit, so the +# chunk parses cannot be trusted for metadata. + +const _THREADED_FORMATS = (:dta, :sav, :sas7bdat) + +function _auto_ntasks(path::AbstractString) + Threads.nthreads() == 1 && return 1 + sz = try + filesize(path) + catch + 0 + end + # Small files are not worth the metadata pre-pass and task setup. + return sz < 4_000_000 ? 1 : min(Threads.nthreads(), 8) +end + +# Parse `path` into `pc` using `ntasks` chunk parsers. Returns false when a +# multi-task read is not possible (unknown row count, too few rows), in +# which case the caller falls back to a serial parse. +function threaded_parse!(pc::ParseContext, path::AbstractString, format::Symbol, ntasks::Int) + pre = ParseContext() + pre.usecols = pc.usecols + pre.file_encoding = pc.file_encoding + pre.handler_encoding = pc.handler_encoding + pre.collect_values = false + parse_file!(pre, path, format) + + total = pre.meta.row_count + total < 0 && return false + nrows = max(total - pc.row_offset, 0) + pc.row_limit >= 0 && (nrows = min(nrows, pc.row_limit)) + if nrows == 0 + # Nothing to read; keep the pre-pass metadata over empty columns. + empty = TypedColumns() + for vm in pre.varmeta + addcolumn!(empty, jltype(vm.type), 0) + end + _adopt!(pc, pre, empty) + return true + end + ntasks = min(ntasks, nrows) + if ntasks <= 1 + # Not worth splitting, but the pre-pass already paid for the + # metadata; run the single data parse and keep the pre-pass results. + chunk = _chunk_context(pc, pre, 0, nrows) + parse_file!(chunk, path, format) + _adopt!(pc, pre, chunk.cols) + return true + end + + shared = TypedColumns() + for vm in pre.varmeta + addcolumn!(shared, jltype(vm.type), nrows) + end + + bounds = [round(Int, i * nrows / ntasks) for i in 0:ntasks] + ctxs = Vector{ParseContext}(undef, ntasks) + tasks = Vector{Task}(undef, ntasks) + for i in 1:ntasks + chunk = _chunk_context(pc, pre, bounds[i], bounds[i + 1] - bounds[i]) + chunk.cols = share_buffers(shared) + chunk.preassigned_cols = true + ctxs[i] = chunk + tasks[i] = Threads.@spawn parse_file!($chunk, $path, $format) + end + firsterr = nothing + for t in tasks + try + wait(t) + catch e + firsterr === nothing && (firsterr = e) + end + end + if firsterr !== nothing + throw(firsterr isa TaskFailedException ? firsterr.task.exception : firsterr) + end + + # Merge the per-task tag vectors (each full-length, '\0' where untagged). + for i in 1:length(shared.slots) + buf = getbuf(shared, i) + for chunk in ctxs + t = column_tags(chunk.cols, i) + t === nothing && continue + dest = buf.tags + dest === nothing && (buf.tags = dest = fill('\0', nrows)) + @inbounds for j in eachindex(t) + t[j] != '\0' && (dest[j] = t[j]) + end + end + end + + _adopt!(pc, pre, shared) + return true +end + +function _chunk_context(pc::ParseContext, pre::ParseContext, base::Int, len::Int) + chunk = ParseContext() + chunk.usecols = pc.usecols + chunk.file_encoding = pc.file_encoding + chunk.handler_encoding = pc.handler_encoding + chunk.keep_user_missing = pc.keep_user_missing + chunk.row_offset = pc.row_offset + base + chunk.row_limit = len + chunk.row_base = base + return chunk +end + +# Take over the pre-pass metadata and the filled buffers as this context's +# parse result, so the ordinary build_table path applies. +function _adopt!(pc::ParseContext, pre::ParseContext, cols::TypedColumns) + pc.meta = pre.meta + pc.names = pre.names + pc.varmeta = pre.varmeta + pc.cols = cols + return pc +end diff --git a/src/values/datetime.jl b/src/values/datetime.jl new file mode 100644 index 0000000..f4eeb37 --- /dev/null +++ b/src/values/datetime.jl @@ -0,0 +1,152 @@ +# Date/time decoding driven by the producers' display-format strings. +# +# Stat packages store dates and times as plain numbers plus a display format; +# the format string is the only record of which numbers are calendar values. +# The tables below map format names to (kind, epoch, unit) rules per +# producer: Stata counts from 1960-01-01 (milliseconds for %tc, days for %td, +# plus week/month/quarter/half-year/year counts), SAS counts seconds or days +# from 1960-01-01, and SPSS counts seconds from 1582-10-14. Times of day are +# plain second counts that may exceed 24 hours (durations), which +# `Dates.Time` cannot represent — they decode into `HMS`. + +""" + HMS + +A time of day or duration as a number of seconds, with unbounded hours: SAS +and SPSS time values can exceed 24 hours or be negative, which `Dates.Time` +cannot represent. Displays as `H:MM:SS[.fff]`; access the raw seconds with +[`unwrap`](@ref), the parts with `Dates.hour`/`minute`/`second`, and convert +to `Dates.Time` via `Time(hms)` when the value is within a calendar day. +""" +struct HMS + seconds::Float64 +end + +""" + unwrap(t::HMS) -> Float64 + +The raw number of seconds in an `HMS` value. +""" +unwrap(t::HMS) = t.seconds + +Base.:(==)(a::HMS, b::HMS) = a.seconds == b.seconds +Base.isequal(a::HMS, b::HMS) = isequal(a.seconds, b.seconds) +Base.isless(a::HMS, b::HMS) = isless(a.seconds, b.seconds) +Base.hash(t::HMS, h::UInt) = hash(t.seconds, hash(:HMS, h)) + +Dates.hour(t::HMS) = + t.seconds < 0 ? -Int(fld(-t.seconds, 3600)) : Int(fld(t.seconds, 3600)) +Dates.minute(t::HMS) = Int(fld(mod(abs(t.seconds), 3600), 60)) +Dates.second(t::HMS) = floor(Int, mod(abs(t.seconds), 60)) +Dates.millisecond(t::HMS) = round(Int, mod(abs(t.seconds), 1) * 1000) + +function Dates.Time(t::HMS) + 0 <= t.seconds < 86400 || + throw(ArgumentError("HMS value $(t) is outside a calendar day")) + return Time(0) + Millisecond(round(Int64, t.seconds * 1000)) +end + +function Base.show(io::IO, t::HMS) + s = abs(t.seconds) + t.seconds < 0 && print(io, '-') + print(io, Int(fld(s, 3600)), ':', lpad(Int(fld(mod(s, 3600), 60)), 2, '0'), ':') + sec = mod(s, 60) + isec = floor(Int, sec) + print(io, lpad(isec, 2, '0')) + frac = sec - isec + frac > 0 && print(io, '.', lpad(round(Int, frac * 1000), 3, '0')) + return +end + +############################################################################## +## +## Format classification +## +############################################################################## + +const STATA_EPOCH_DATETIME = DateTime(1960, 1, 1) +const STATA_EPOCH_DATE = Date(1960, 1, 1) +const SAS_EPOCH_DATETIME = DateTime(1960, 1, 1) +const SAS_EPOCH_DATE = Date(1960, 1, 1) +const SPSS_EPOCH_DATETIME = DateTime(1582, 10, 14) +const SPSS_EPOCH_DATE = Date(1582, 10, 14) + +# SAS format base names (with trailing width digits and decimals stripped). +const SAS_DATETIME_FORMATS = Set(["DATETIME", "E8601DT", "E8601DX", "E8601DZ", + "E8601LX", "E8601DN", "E8601LZ", "E8601TX", "E8601TZ"]) +const SAS_DATE_FORMATS = Set(["DATE", "WEEKDATE", "WEEKDATX", "WEEKDAY", "MMDDYY", + "DDMMYY", "YYMMDD", "DDMMYYB", "DDMMYYC", "DDMMYYD", "DDMMYYN", "DDMMYYP", + "DDMMYYS", "MMDDYYB", "MMDDYYC", "MMDDYYD", "MMDDYYN", "MMDDYYP", "MMDDYYS", + "YYMMDDB", "YYMMDDD", "YYMMDDN", "YYMMDDP", "YYMMDDS", "MONNAME", "MONTH", + "MONYY", "QTR", "QTRR", "YEAR", "DAY", "DOWNAME", "JULIAN", "E8601DA"]) +const SAS_TIME_FORMATS = Set(["TIME", "HHMM", "HOUR", "MMSS", "E8601TM"]) + +# SPSS format base names. +const SPSS_DATETIME_FORMATS = Set(["DATETIME", "YMDHMS"]) +const SPSS_DATE_FORMATS = Set(["DATE", "ADATE", "EDATE", "JDATE", "SDATE"]) +const SPSS_TIME_FORMATS = Set(["TIME", "DTIME"]) + +# "DATETIME22.3" -> "DATETIME", "EDATE10" -> "EDATE", "E8601DA" -> "E8601DA". +_format_base(fmt::AbstractString) = + replace(replace(uppercase(fmt), r"\.\d*$" => ""), r"\d+$" => "") + +# Classify a format string into a converter function (applied to non-missing +# numeric cells) and a target element type, or `nothing` when the column is +# not a date/time. +function datetime_rule(fmt::AbstractString, producer::Symbol) + isempty(fmt) && return nothing + if producer === :dta + if startswith(fmt, "%tc") || startswith(fmt, "%tC") || + startswith(fmt, "%-tc") || startswith(fmt, "%-tC") + return (v -> STATA_EPOCH_DATETIME + Millisecond(round(Int64, Float64(v)))), DateTime + elseif startswith(fmt, "%td") || startswith(fmt, "%-td") || startswith(fmt, "%d") || + startswith(fmt, "%-d") + return (v -> STATA_EPOCH_DATE + Day(round(Int64, Float64(v)))), Date + elseif startswith(fmt, "%tw") + # Stata weeks reset every year: 52 fixed-length weeks per year. + return (v -> (i = round(Int64, Float64(v)); + Date(1960 + fld(i, 52), 1, 1) + Week(mod(i, 52)))), Date + elseif startswith(fmt, "%tm") + return (v -> STATA_EPOCH_DATE + Month(round(Int64, Float64(v)))), Date + elseif startswith(fmt, "%tq") + return (v -> STATA_EPOCH_DATE + Month(3 * round(Int64, Float64(v)))), Date + elseif startswith(fmt, "%th") + return (v -> STATA_EPOCH_DATE + Month(6 * round(Int64, Float64(v)))), Date + elseif startswith(fmt, "%ty") + return (v -> Date(round(Int64, Float64(v)), 1, 1)), Date + end + return nothing + elseif producer === :sas7bdat || producer === :xport + base = _format_base(fmt) + if base in SAS_DATETIME_FORMATS + return (v -> SAS_EPOCH_DATETIME + Millisecond(round(Int64, Float64(v) * 1000))), DateTime + elseif base in SAS_DATE_FORMATS + return (v -> SAS_EPOCH_DATE + Day(round(Int64, Float64(v)))), Date + elseif base in SAS_TIME_FORMATS + return (v -> HMS(Float64(v))), HMS + end + return nothing + elseif producer === :sav || producer === :por + base = _format_base(fmt) + if base in SPSS_DATETIME_FORMATS + return (v -> SPSS_EPOCH_DATETIME + Millisecond(round(Int64, Float64(v) * 1000))), DateTime + elseif base in SPSS_DATE_FORMATS + return (v -> SPSS_EPOCH_DATE + Day(fld(round(Int64, Float64(v)), 86400))), Date + elseif base in SPSS_TIME_FORMATS + return (v -> HMS(Float64(v))), HMS + end + return nothing + end + return nothing +end + +# Convert a numeric column to Date/DateTime/HMS behind a function barrier. +function convert_datetime_column(col::DataValueVector{T}, f::F, ::Type{S}) where {T<:Number,F,S} + n = length(col) + out = DataValueVector{S}(Vector{S}(undef, n), fill(true, n)) + @inbounds for i in 1:n + v = col[i] + DataValues.isna(v) || (out[i] = f(get(v))) + end + return out +end diff --git a/src/values/labeled.jl b/src/values/labeled.jl new file mode 100644 index 0000000..19d590e --- /dev/null +++ b/src/values/labeled.jl @@ -0,0 +1,104 @@ +# A lazily labeled view of a value-labeled column. +# +# Stat packages store "categorical" columns as raw codes plus a (possibly +# partial) label set. A LabeledArray keeps the raw code column untouched and +# looks labels up only on access, so nothing is lost (codes, gaps, unlabeled +# values, ordering) and nothing is paid when the view is not used. This is +# deliberately a partial dictionary encoding, not a categorical/factor type: +# unlabeled codes remain first-class values that merely display as +# `string(code)`. + +""" + LabeledValue{T} + +A single value from a value-labeled column: the raw code of type `T` plus a +reference to the column's shared label dictionary. It displays as its label +(falling back to `string(code)` when the code has no label) but computes as +its code: `==`, `isless`, and `hash` all act on the code, so predicates, +joins, and grouping behave exactly as on the raw column. Comparing against +an `AbstractString` compares the label instead. + +Access the parts with [`unwrap`](@ref) (the code) and [`valuelabel`](@ref) +(the label). +""" +struct LabeledValue{T} + value::T + labels::ValueLabelDict +end + +""" + unwrap(lv::LabeledValue{T}) -> T + +The raw code stored in a labeled value. +""" +unwrap(lv::LabeledValue) = lv.value + +_labelkey(v::Integer) = Int32(v) +_labelkey(v::AbstractFloat) = Float64(v) +_labelkey(v::AbstractString) = String(v) + +""" + valuelabel(lv::LabeledValue) -> String + +The label of a labeled value, or `string(unwrap(lv))` when its code has no +label in the column's label set. +""" +valuelabel(lv::LabeledValue) = get(() -> string(lv.value), lv.labels, _labelkey(lv.value)) + +Base.:(==)(a::LabeledValue, b::LabeledValue) = a.value == b.value +Base.:(==)(a::LabeledValue, b::Number) = a.value == b +Base.:(==)(a::Number, b::LabeledValue) = a == b.value +Base.:(==)(a::LabeledValue, b::AbstractString) = valuelabel(a) == b +Base.:(==)(a::AbstractString, b::LabeledValue) = a == valuelabel(b) +Base.isequal(a::LabeledValue, b::LabeledValue) = isequal(a.value, b.value) +Base.isless(a::LabeledValue, b::LabeledValue) = isless(a.value, b.value) +Base.isless(a::LabeledValue, b::Number) = isless(a.value, b) +Base.isless(a::Number, b::LabeledValue) = isless(a, b.value) +Base.hash(lv::LabeledValue, h::UInt) = hash(lv.value, h) + +Base.show(io::IO, lv::LabeledValue) = print(io, valuelabel(lv)) +function Base.show(io::IO, ::MIME"text/plain", lv::LabeledValue) + print(io, valuelabel(lv), " (", lv.value, ")") +end + +""" + LabeledArray{T} <: AbstractVector{DataValue{LabeledValue{T}}} + +A lazy labeled view of a value-labeled column: wraps the raw +`DataValueVector` of codes plus the shared label dictionary, constructing +[`LabeledValue`](@ref)s only on access. NA cells stay NA. Obtain one with +[`labeled`](@ref) or by reading with `apply_value_labels=true`; get the raw +column back with [`rawvalues`](@ref) (zero-copy) and the dictionary with +`getvaluelabels`. +""" +struct LabeledArray{T} <: AbstractVector{DataValues.DataValue{LabeledValue{T}}} + values::DataValueVector{T} + labels::ValueLabelDict +end + +""" + rawvalues(a::LabeledArray) -> DataValueVector + +The raw code column underlying a labeled view (zero-copy). +""" +rawvalues(a::LabeledArray) = a.values + +""" + getvaluelabels(a::LabeledArray) -> ValueLabelDict + +The label dictionary shared by the elements of a labeled view. +""" +getvaluelabels(a::LabeledArray) = a.labels + +Base.size(a::LabeledArray) = size(a.values) +Base.IndexStyle(::Type{<:LabeledArray}) = IndexLinear() + +function Base.getindex(a::LabeledArray{T}, i::Int) where {T} + v = a.values[i] + DataValues.isna(v) ? DataValues.DataValue{LabeledValue{T}}() : + DataValues.DataValue(LabeledValue{T}(get(v), a.labels)) +end + +Base.setindex!(a::LabeledArray{T}, v, i::Int) where {T} = (a.values[i] = v; a) +Base.setindex!(a::LabeledArray{T}, v::LabeledValue, i::Int) where {T} = + (a.values[i] = unwrap(v); a) diff --git a/src/writer/write.jl b/src/writer/write.jl new file mode 100644 index 0000000..aa7e6d3 --- /dev/null +++ b/src/writer/write.jl @@ -0,0 +1,346 @@ +# High-level writing: columns plus names (Queryverse-style, not a table +# protocol) or a ReadStatTable, canonicalized onto the six storage types the +# formats support, with metadata, value labels, tagged missings, and +# date/time re-encoding. + +const _WRITE_PRODUCER = Dict(:dta => :dta, :sav => :spss, :por => :spss, + :sas7bdat => :sas, :xport => :sas) + +# A column ready for the row loop: canonical storage values, NA mask access +# through the DataValueVector API, per-row tags, and variable attributes. +struct PreparedColumn + data::AbstractVector # DataValueVector{S} with S one of the 6 storage types + type::ReadStatType + storage_width::Int + format::String # derived format ("" = keep caller's hint) + tags::Union{Nothing,Vector{Char}} +end + +_plain_to_dv(v::AbstractVector{T}) where {T} = DataValueVector{T}(collect(v), fill(false, length(v))) + +function _prepare_column(col::AbstractVector, producer::Symbol, convert_datetime::Bool, + format::Symbol, tags::Union{Nothing,Vector{Char}}) + col isa LabeledArray && return _prepare_column(rawvalues(col), producer, convert_datetime, + format, tags) + col isa DataValueVector || (col = _plain_to_dv(col)) + T = eltype(eltype(col)) + n = length(col) + + # The por writer only handles doubles (the format is text-based), so + # every numeric column becomes Float64 there. + if T <: Real && format === :por && T !== Float64 + out = DataValueVector{Float64}(Vector{Float64}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = Float64(get(col[i]))) + end + return PreparedColumn(out, READSTAT_TYPE_DOUBLE, 0, "", tags) + elseif T === Bool + out = DataValueVector{Int8}(Vector{Int8}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = Int8(get(col[i]))) + end + return PreparedColumn(out, READSTAT_TYPE_INT8, 0, "", tags) + elseif T === Int8 || T === Int16 || T === Int32 || T === Float32 || T === Float64 + width = format === :xport && T === Float64 ? 8 : 0 + return PreparedColumn(col, rstype(T), width, "", tags) + elseif T <: Integer + # Narrow to Int32 when every value fits, otherwise go through Float64. + fits = all(i -> DataValues.isna(col[i]) || + typemin(Int32) <= get(col[i]) <= typemax(Int32), 1:n) + S = fits ? Int32 : Float64 + out = DataValueVector{S}(Vector{S}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = S(get(col[i]))) + end + return PreparedColumn(out, rstype(S), 0, "", tags) + elseif T <: Real + out = DataValueVector{Float64}(Vector{Float64}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = Float64(get(col[i]))) + end + return PreparedColumn(out, READSTAT_TYPE_DOUBLE, 0, "", tags) + elseif T <: AbstractString + out = DataValueVector{String}(fill("", n), fill(true, n)) + width = 1 + for i in 1:n + if !DataValues.isna(col[i]) + s = String(get(col[i])) + out[i] = s + width = max(width, sizeof(s)) + end + end + return PreparedColumn(out, READSTAT_TYPE_STRING, width, "", tags) + elseif convert_datetime && (T === Date || T === DateTime || T === HMS || T === Time) + f, S, fmt = _dt_encode(producer, T) + out = DataValueVector{S}(Vector{S}(undef, n), fill(true, n)) + for i in 1:n + DataValues.isna(col[i]) || (out[i] = f(get(col[i]))) + end + width = format === :xport && S === Float64 ? 8 : 0 + return PreparedColumn(out, rstype(S), width, fmt, tags) + else + throw(ArgumentError("cannot write a column with element type $T" * + (T in (Date, DateTime, HMS, Time) ? " with convert_datetime=false" : ""))) + end +end + +function _dt_encode(producer::Symbol, ::Type{Date}) + if producer === :dta + (d -> Int32(Dates.value(d - STATA_EPOCH_DATE)), Int32, "%td") + elseif producer === :spss + (d -> Float64(Dates.value(d - SPSS_EPOCH_DATE)) * 86400.0, Float64, "EDATE10") + else + (d -> Float64(Dates.value(d - SAS_EPOCH_DATE)), Float64, "YYMMDD10") + end +end + +function _dt_encode(producer::Symbol, ::Type{DateTime}) + if producer === :dta + (dt -> Float64(Dates.value(dt - STATA_EPOCH_DATETIME)), Float64, "%tc") + elseif producer === :spss + (dt -> Float64(Dates.value(dt - SPSS_EPOCH_DATETIME)) / 1000.0, Float64, "DATETIME20") + else + (dt -> Float64(Dates.value(dt - SAS_EPOCH_DATETIME)) / 1000.0, Float64, "DATETIME19") + end +end + +_seconds(t::HMS) = unwrap(t) +_seconds(t::Time) = Dates.value(t) / 1.0e9 + +function _dt_encode(producer::Symbol, ::Type{T}) where {T<:Union{HMS,Time}} + if producer === :dta + # Stata has no pure time type; times of day become %tc datetimes on + # the epoch day, matching common Stata practice. + (t -> _seconds(t) * 1000.0, Float64, "%tcHH:MM:SS") + else + (t -> Float64(_seconds(t)), Float64, "TIME8") + end +end + +# Infer the C key type of a label set from its dictionary keys (Char keys +# label tagged missings and do not constrain the type). +function _label_set_type(dict::AbstractDict) + keytypes = [typeof(k) for k in keys(dict) if !(k isa Char)] + any(t -> t <: AbstractString, keytypes) && return READSTAT_TYPE_STRING + any(t -> t <: AbstractFloat, keytypes) && return READSTAT_TYPE_DOUBLE + return READSTAT_TYPE_INT32 +end + +function _write_label_sets!(w::Writer, value_labels) + sets = Dict{Symbol,LabelSet}() + for (name, dict) in value_labels + ls = add_label_set!(w, _label_set_type(dict), name) + for (k, v) in dict + label!(ls, k, v) + end + sets[Symbol(name)] = ls + end + return sets +end + +_getvec(v::Nothing, i) = nothing +_getvec(v::AbstractVector, i) = v[i] + +function write_data_file(dest::Union{AbstractString,IO}, format::Symbol, + columns::AbstractVector, names::AbstractVector{Symbol}; + labels::Union{Nothing,AbstractVector{<:AbstractString}}=nothing, + formats::Union{Nothing,AbstractVector{<:AbstractString}}=nothing, + value_labels=Dict{Symbol,ValueLabelDict}(), + vallabels::Union{Nothing,AbstractVector{Symbol}}=nothing, + measures::Union{Nothing,AbstractVector{ReadStatMeasure}}=nothing, + alignments::Union{Nothing,AbstractVector{ReadStatAlignment}}=nothing, + display_widths::Union{Nothing,AbstractVector{<:Integer}}=nothing, + missing_ranges::Union{Nothing,AbstractVector}=nothing, + tags::Union{Nothing,AbstractVector}=nothing, + file_label::AbstractString="", table_name::AbstractString="", + notes::AbstractVector{<:AbstractString}=String[], + timestamp::Union{Nothing,DateTime}=nothing, + version::Union{Nothing,Integer}=nothing, + compress::Symbol=:none, is_64bit::Bool=true, + fweight::Union{Nothing,Symbol}=nothing, + convert_datetime::Bool=true) + ncols = length(columns) + ncols == length(names) || + throw(ArgumentError("got $(length(columns)) columns for $(length(names)) names")) + for (kw, v) in (("labels", labels), ("formats", formats), ("vallabels", vallabels), + ("measures", measures), ("alignments", alignments), + ("display_widths", display_widths), ("missing_ranges", missing_ranges), + ("tags", tags)) + v === nothing || length(v) == ncols || + throw(ArgumentError("$kw must have one entry per column")) + end + nrows = ncols == 0 ? 0 : length(columns[1]) + all(c -> length(c) == nrows, columns) || + throw(ArgumentError("all columns must have the same length")) + + # SPSS portable is an uppercase-only format; the C writer silently writes + # no data under lowercase names, so normalize them here (a por read + # returns uppercase names regardless). + format === :por && (names = [Symbol(uppercase(String(n))) for n in names]) + + producer = _WRITE_PRODUCER[format] + prepared = PreparedColumn[ + _prepare_column(columns[i], producer, convert_datetime, format, _getvec(tags, i)) + for i in 1:ncols] + + w = Writer(dest) + try + sets = _write_label_sets!(w, value_labels) + vars = Vector{WriterVariable}(undef, ncols) + for i in 1:ncols + p = prepared[i] + # LabeledArray columns bring their own label set when none is named. + vallabel = _getvec(vallabels, i) + if vallabel === nothing && columns[i] isa LabeledArray + vallabel = Symbol(:__auto_labels_, i) + d = getvaluelabels(columns[i]) + ls = add_label_set!(w, _label_set_type(d), vallabel) + for (k, v) in d + label!(ls, k, v) + end + sets[vallabel] = ls + end + fmt = p.format + isempty(fmt) && formats !== nothing && (fmt = formats[i]) + ranges = missing_ranges === nothing ? () : + Tuple((r isa Tuple ? r : (r, r)) for r in missing_ranges[i]) + vars[i] = add_variable!(w, names[i], p.type; + storage_width=p.storage_width, + label=labels === nothing ? "" : labels[i], + format=fmt, + label_set=vallabel === nothing || vallabel === Symbol("") ? nothing : + get(sets, vallabel, nothing), + measure=measures === nothing ? READSTAT_MEASURE_UNKNOWN : measures[i], + alignment=alignments === nothing ? READSTAT_ALIGNMENT_UNKNOWN : alignments[i], + display_width=display_widths === nothing ? 0 : display_widths[i], + missing_ranges=ranges) + end + isempty(file_label) || file_label!(w, file_label) + isempty(table_name) || table_name!(w, table_name) + timestamp === nothing || timestamp!(w, timestamp) + version === nothing || format_version!(w, version) + compress === :none || compression!(w, compress) + format in (:sas7bdat, :xport) && is_64bit!(w, is_64bit) + for note in notes + add_note!(w, note) + end + if fweight !== nothing + j = findfirst(==(fweight), names) + j === nothing && throw(ArgumentError("fweight column $fweight not found")) + fweight!(w, vars[j]) + end + + begin_writing!(w, format, nrows) + for r in 1:nrows + begin_row!(w) + for i in 1:ncols + _insert_cell!(w, vars[i], prepared[i], r) + end + end_row!(w) + end + end_writing!(w) + finally + close(w) + end + return dest +end + +function _insert_cell!(w::Writer, var::WriterVariable, p::PreparedColumn, r::Int) + t = p.tags + if t !== nothing && t[r] != '\0' + insert_tagged_missing!(w, var, t[r]) + return + end + _insert_cell!(w, var, p.data, r) +end + +function _insert_cell!(w::Writer, var::WriterVariable, col::DataValueVector{T}, r::Int) where {T} + v = col[r] + DataValues.isna(v) ? insert_missing!(w, var) : insert_value!(w, var, get(v)) + return +end + +const _WRITE_KWARGS_DOC = """ +`columns` is a vector of columns (`DataValueVector`s, [`LabeledArray`](@ref)s +— which bring their value labels along — or plain vectors) and `names` the +matching column names. Keyword arguments, all optional: + +- `labels`, `formats`, `measures`, `alignments`, `display_widths`: one + variable attribute per column. +- `value_labels::Dict{Symbol,ValueLabelDict}` plus `vallabels` (one label-set + name per column, `Symbol("")` for none): explicit value-label sets. +- `missing_ranges`: per column, a collection of SPSS user-missing values or + `(lo, hi)` range tuples. +- `tags`: per column, `nothing` or a `Vector{Char}` marking tagged missing + values (`'a'`-`'z'`, `'\\0'` elsewhere; Stata/SAS formats). +- `file_label`, `notes`, `timestamp`, `version`, `table_name` (XPORT), + `is_64bit` (SAS), `compress` (`:rows` for sas7bdat/sav, `:binary` for + zsav), `fweight` (name of the frequency-weight column). +- `convert_datetime=true`: encode `Date`/`DateTime`/[`HMS`](@ref)/`Time` + columns into the format's native representation with a matching display + format. + +Instead of a path, `dest` may be any writable `IO`. +""" + +for (fn, fmt) in ((:write_dta, :dta), (:write_sav, :sav), (:write_por, :por), + (:write_sas7bdat, :sas7bdat), (:write_xport, :xport)) + @eval begin + """ + $($(string(fn)))(dest, columns, names::Vector{Symbol}; kwargs...) + $($(string(fn)))(dest, tbl::ReadStatTable; kwargs...) + + Write columns to $($(string(fmt))) format. + + $_WRITE_KWARGS_DOC + """ + $fn(dest::Union{AbstractString,IO}, columns::AbstractVector, + names::AbstractVector{Symbol}; kwargs...) = + write_data_file(dest, $(QuoteNode(fmt)), columns, names; kwargs...) + $fn(dest::Union{AbstractString,IO}, tbl::ReadStatTable; kwargs...) = + _write_table(dest, $(QuoteNode(fmt)), tbl; kwargs...) + end +end + +""" + write_sas7bcat(dest, value_labels::Dict{Symbol,<:AbstractDict}) + +Write a SAS value-label catalog holding the given label sets (as read by +[`read_sas7bcat`](@ref)). + +!!! warning + The catalog writer in readstat 1.1.9 only handles numeric label sets + correctly; string-keyed sets do not survive a round trip. +""" +function write_sas7bcat(dest::Union{AbstractString,IO}, value_labels::AbstractDict) + w = Writer(dest) + try + _write_label_sets!(w, value_labels) + begin_writing!(w, :sas7bcat) + end_writing!(w) + finally + close(w) + end + return dest +end + +# Write a ReadStatTable, carrying its metadata across; explicit keyword +# arguments override what the table provides. +function _write_table(dest, format::Symbol, tbl::ReadStatTable; kwargs...) + meta = filemetadata(tbl) + colmeta = getfield(tbl, :colmeta) + columns = getfield(tbl, :cols) + provided = (; + labels=[m.label for m in colmeta], + value_labels=meta.value_labels, + vallabels=[m.vallabel for m in colmeta], + measures=[m.measure for m in colmeta], + alignments=[m.alignment for m in colmeta], + tags=getfield(tbl, :tags), + file_label=meta.file_label, + notes=meta.notes, + fweight=meta.fweight === Symbol("") ? nothing : meta.fweight, + ) + return write_data_file(dest, format, columns, names(tbl); + merge(provided, NamedTuple(kwargs))...) +end diff --git a/src/writer/writer.jl b/src/writer/writer.jl new file mode 100644 index 0000000..d89ab8d --- /dev/null +++ b/src/writer/writer.jl @@ -0,0 +1,383 @@ +# Low-level writer: a thin, safe Julia layer over the C writer API. It +# mirrors the C call sequence one-to-one — define label sets, define +# variables, begin writing, insert rows, end — and streams all output bytes +# into any Julia IO through the C data-writer callback. The high-level +# write_dta/write_sav/... functions in writer/write.jl are built on top; +# reach for this layer directly to control every detail the C API exposes. + +""" + Writer(io::IO) + Writer(path::AbstractString) + +A handle for writing a stat-package file through the C writer API. Follow +the C call order: [`add_label_set!`](@ref)/[`label!`](@ref) for value +labels, [`add_variable!`](@ref) for each column, the file-level setters +([`file_label!`](@ref), [`timestamp!`](@ref), [`compression!`](@ref), ...), +then [`begin_writing!`](@ref), one [`begin_row!`](@ref)/`insert_*`/ +[`end_row!`](@ref) cycle per observation, and finally `close`. Output +bytes stream into the given IO (a path is opened for writing and closed +again by `close`). +""" +mutable struct Writer + ptr::WriterPtr + io::IO + owns_io::Bool + err::Union{Nothing,Tuple{Any,Any}} + began::Bool + ended::Bool +end + +struct WriterVariable + ptr::VariablePtr +end + +struct LabelSet + ptr::LabelSetPtr + type::ReadStatType +end + +struct StringRef + ptr::StringRefPtr +end + +function Writer(io::IO) + ptr = readstat_writer_init() + ptr == C_NULL && error("readstat_writer_init failed") + w = Writer(ptr, io, false, nothing, false, false) + _writer_check(w, readstat_set_data_writer(ptr, CF_DATA_WRITER[])) + finalizer(_writer_finalize, w) + return w +end + +Writer(path::AbstractString) = (w = Writer(open(path, "w")); w.owns_io = true; w) + +function _writer_finalize(w::Writer) + if w.ptr != C_NULL + readstat_writer_free(w.ptr) + w.ptr = C_NULL + end + return +end + +function handle_data_write(data::Ptr{Cvoid}, len::Csize_t, ctx::Ptr{Cvoid})::Cssize_t + w = unsafe_pointer_to_objref(ctx)::Writer + try + unsafe_write(w.io, Ptr{UInt8}(data), UInt(len)) + return Cssize_t(len) + catch e + w.err = (e, catch_backtrace()) + return Cssize_t(-1) + end +end + +const CF_DATA_WRITER = Ref(C_NULL) + +function _init_writer_cfunctions() + CF_DATA_WRITER[] = @cfunction(handle_data_write, Cssize_t, (Ptr{Cvoid}, Csize_t, Ptr{Cvoid})) + return nothing +end + +function _writer_check(w::Writer, err::ReadStatError) + if w.err !== nothing + e, _ = w.err + throw(e) + end + err == READSTAT_OK || error("readstat writer error: $(readstat_error_message(err))") + return w +end + +_ptr(w::Writer) = (w.ptr == C_NULL && error("Writer is closed"); w.ptr) + +# The Julia storage type used for each writable ReadStatType. +function rstype(::Type{T}) where {T} + T === Int8 || T === Bool ? READSTAT_TYPE_INT8 : + T === Int16 ? READSTAT_TYPE_INT16 : + T <: Integer ? READSTAT_TYPE_INT32 : + T === Float32 ? READSTAT_TYPE_FLOAT : + T <: Real ? READSTAT_TYPE_DOUBLE : + T <: Union{AbstractString,Char} ? READSTAT_TYPE_STRING : + throw(ArgumentError("no readstat storage type for $T")) +end + +""" + add_label_set!(w::Writer, type, name) -> LabelSet + +Create a named value-label set. `type` is a `ReadStatType` or a Julia type +(`Int32`, `Float64`, `String`); add entries with [`label!`](@ref) and attach +the set to variables via `add_variable!(...; label_set=...)`. +""" +add_label_set!(w::Writer, type::ReadStatType, name::Union{Symbol,AbstractString}) = + LabelSet(readstat_add_label_set(_ptr(w), type, string(name)), type) +add_label_set!(w::Writer, ::Type{T}, name::Union{Symbol,AbstractString}) where {T} = + add_label_set!(w, rstype(T), name) + +""" + label!(ls::LabelSet, value, label::AbstractString) + +Add one entry to a value-label set. Numeric values go to the set's numeric +key type, strings to string keys, and a `Char` labels a tagged missing value +(`.a`-`.z`, Stata/SAS only). +""" +function label!(ls::LabelSet, value::Real, label::AbstractString) + if ls.type == READSTAT_TYPE_DOUBLE || ls.type == READSTAT_TYPE_FLOAT + readstat_label_double_value(ls.ptr, Float64(value), label) + else + readstat_label_int32_value(ls.ptr, Int32(value), label) + end + return ls +end +label!(ls::LabelSet, value::AbstractString, label::AbstractString) = + (readstat_label_string_value(ls.ptr, value, label); ls) +label!(ls::LabelSet, tag::Char, label::AbstractString) = + (readstat_label_tagged_value(ls.ptr, tag, label); ls) + +""" + add_variable!(w::Writer, name, type; storage_width=0, label="", format="", + label_set=nothing, measure=READSTAT_MEASURE_UNKNOWN, + alignment=READSTAT_ALIGNMENT_UNKNOWN, display_width=0, + missing_values=(), missing_ranges=()) -> WriterVariable + +Define the next variable. `type` is a `ReadStatType` or a Julia type; +`storage_width` matters for strings (all formats) and doubles (XPORT). +`missing_values`/`missing_ranges` declare SPSS user-defined missing values +(numbers or strings; ranges as 2-tuples). +""" +function add_variable!(w::Writer, name::Union{Symbol,AbstractString}, type::ReadStatType; + storage_width::Integer=0, label::AbstractString="", + format::AbstractString="", label_set::Union{Nothing,LabelSet}=nothing, + measure::ReadStatMeasure=READSTAT_MEASURE_UNKNOWN, + alignment::ReadStatAlignment=READSTAT_ALIGNMENT_UNKNOWN, + display_width::Integer=0, missing_values=(), missing_ranges=()) + ptr = readstat_add_variable(_ptr(w), string(name), type, storage_width) + ptr == C_NULL && error("readstat_add_variable failed for $name") + isempty(label) || readstat_variable_set_label(ptr, label) + isempty(format) || readstat_variable_set_format(ptr, format) + label_set === nothing || readstat_variable_set_label_set(ptr, label_set.ptr) + measure == READSTAT_MEASURE_UNKNOWN || readstat_variable_set_measure(ptr, measure) + alignment == READSTAT_ALIGNMENT_UNKNOWN || readstat_variable_set_alignment(ptr, alignment) + display_width == 0 || readstat_variable_set_display_width(ptr, display_width) + for v in missing_values + _writer_check(w, v isa AbstractString ? + readstat_variable_add_missing_string_value(ptr, v) : + readstat_variable_add_missing_double_value(ptr, Float64(v))) + end + for (lo, hi) in missing_ranges + _writer_check(w, lo isa AbstractString ? + readstat_variable_add_missing_string_range(ptr, lo, hi) : + readstat_variable_add_missing_double_range(ptr, Float64(lo), Float64(hi))) + end + return WriterVariable(ptr) +end +add_variable!(w::Writer, name::Union{Symbol,AbstractString}, ::Type{T}; kwargs...) where {T} = + add_variable!(w, name, rstype(T); kwargs...) + +""" + add_note!(w::Writer, note) + +Add a file note (SPSS Document Record line — at most 80 characters there — +or a Stata note). +""" +add_note!(w::Writer, note::AbstractString) = (readstat_add_note(_ptr(w), note); w) + +""" + add_string_ref!(w::Writer, s) -> StringRef + +Intern a string for a `READSTAT_TYPE_STRING_REF` (Stata strL) column; +insert it into rows with [`insert_string_ref!`](@ref). Refs can be shared +across columns and rows. +""" +add_string_ref!(w::Writer, s::AbstractString) = StringRef(readstat_add_string_ref(_ptr(w), s)) + +""" + file_label!(w::Writer, s) + +Set the file label. +""" +file_label!(w::Writer, s::AbstractString) = + _writer_check(w, readstat_writer_set_file_label(_ptr(w), s)) + +""" + timestamp!(w::Writer, t::DateTime) + +Set the file timestamp. +""" +timestamp!(w::Writer, t::DateTime) = + _writer_check(w, readstat_writer_set_file_timestamp(_ptr(w), + round(Int64, Dates.datetime2unix(t)))) + +""" + fweight!(w::Writer, var::WriterVariable) + +Mark a variable as the frequency weight. +""" +fweight!(w::Writer, var::WriterVariable) = + _writer_check(w, readstat_writer_set_fweight_variable(_ptr(w), var.ptr)) + +""" + format_version!(w::Writer, v) + +Set the file format version (e.g. 104-119 for `.dta`, 5 or 8 for XPORT, +2 or 3 for `.sav`). +""" +format_version!(w::Writer, v::Integer) = + _writer_check(w, readstat_writer_set_file_format_version(_ptr(w), v)) + +""" + table_name!(w::Writer, s) + +Set the table name (used by XPORT files; defaults to `DATASET`). +""" +table_name!(w::Writer, s::AbstractString) = + _writer_check(w, readstat_writer_set_table_name(_ptr(w), s)) + +""" + is_64bit!(w::Writer, b::Bool) + +Set whether a SAS file is written in 64-bit layout (default true). +""" +is_64bit!(w::Writer, b::Bool) = + _writer_check(w, readstat_writer_set_file_format_is_64bit(_ptr(w), b)) + +const COMPRESSION_BY_NAME = Dict(:none => READSTAT_COMPRESS_NONE, + :rows => READSTAT_COMPRESS_ROWS, :binary => READSTAT_COMPRESS_BINARY) + +""" + compression!(w::Writer, c) + +Set the output compression: `:none`, `:rows` (sas7bdat and sav), or +`:binary` (sav only — produces a zsav file). +""" +function compression!(w::Writer, c::Symbol) + haskey(COMPRESSION_BY_NAME, c) || + throw(ArgumentError("compression must be :none, :rows, or :binary")) + _writer_check(w, readstat_writer_set_compression(_ptr(w), COMPRESSION_BY_NAME[c])) +end + +""" + begin_writing!(w::Writer, format::Symbol, row_count) + +Start writing the file body for `format` (`:dta`, `:sav`, `:por`, +`:sas7bdat`, `:xport`, or `:sas7bcat`, which takes no rows). All label +sets, variables, and file-level metadata must be defined beforehand. +""" +function begin_writing!(w::Writer, format::Symbol, row_count::Integer=0) + ptr = _ptr(w) + ctx = pointer_from_objref(w) + err = if format === :dta + readstat_begin_writing_dta(ptr, ctx, row_count) + elseif format === :sav + readstat_begin_writing_sav(ptr, ctx, row_count) + elseif format === :por + readstat_begin_writing_por(ptr, ctx, row_count) + elseif format === :sas7bdat + readstat_begin_writing_sas7bdat(ptr, ctx, row_count) + elseif format === :xport + readstat_begin_writing_xport(ptr, ctx, row_count) + elseif format === :sas7bcat + readstat_begin_writing_sas7bcat(ptr, ctx) + else + throw(ArgumentError("unknown format $format")) + end + _writer_check(w, err) + w.began = true + return w +end + +""" + validate_metadata(w::Writer) + +Run the format's file-level validation (call after [`begin_writing!`](@ref)). +""" +validate_metadata(w::Writer) = _writer_check(w, readstat_validate_metadata(_ptr(w))) + +""" + validate_variable(w::Writer, var::WriterVariable) + +Run the format's per-variable validation (call after [`begin_writing!`](@ref)). +""" +validate_variable(w::Writer, var::WriterVariable) = + _writer_check(w, readstat_validate_variable(_ptr(w), var.ptr)) + +""" + begin_row!(w::Writer) + +Start one observation; insert a value for every variable, then call +[`end_row!`](@ref). +""" +begin_row!(w::Writer) = _writer_check(w, readstat_begin_row(_ptr(w))) + +""" + end_row!(w::Writer) + +Finish the current observation. +""" +end_row!(w::Writer) = _writer_check(w, readstat_end_row(_ptr(w))) + +""" + insert_value!(w::Writer, var::WriterVariable, v) + +Insert one cell into the current row: an `Int8`/`Int16`/`Int32`/`Float32`/ +`Float64` or a string, matching the variable's storage type. See also +[`insert_missing!`](@ref), [`insert_tagged_missing!`](@ref), and +[`insert_string_ref!`](@ref). +""" +insert_value!(w::Writer, var::WriterVariable, v::Int8) = + _writer_check(w, readstat_insert_int8_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::Int16) = + _writer_check(w, readstat_insert_int16_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::Int32) = + _writer_check(w, readstat_insert_int32_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::Float32) = + _writer_check(w, readstat_insert_float_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::Float64) = + _writer_check(w, readstat_insert_double_value(_ptr(w), var.ptr, v)) +insert_value!(w::Writer, var::WriterVariable, v::AbstractString) = + _writer_check(w, readstat_insert_string_value(_ptr(w), var.ptr, v)) + +""" + insert_missing!(w::Writer, var::WriterVariable) + +Insert a (system) missing cell into the current row. +""" +insert_missing!(w::Writer, var::WriterVariable) = + _writer_check(w, readstat_insert_missing_value(_ptr(w), var.ptr)) + +""" + insert_tagged_missing!(w::Writer, var::WriterVariable, tag::Char) + +Insert a tagged missing cell (`'a'`-`'z'`; Stata and SAS formats only). +""" +insert_tagged_missing!(w::Writer, var::WriterVariable, tag::Char) = + _writer_check(w, readstat_insert_tagged_missing_value(_ptr(w), var.ptr, tag)) + +""" + insert_string_ref!(w::Writer, var::WriterVariable, ref::StringRef) + +Insert an interned string (from [`add_string_ref!`](@ref)) into a +`READSTAT_TYPE_STRING_REF` (Stata strL) column. +""" +insert_string_ref!(w::Writer, var::WriterVariable, ref::StringRef) = + _writer_check(w, readstat_insert_string_ref(_ptr(w), var.ptr, ref.ptr)) + +""" + end_writing!(w::Writer) + +Finish the file body. Called automatically by `close(w)` when writing has +begun and not yet ended. +""" +function end_writing!(w::Writer) + w.ended && return w + GC.@preserve w _writer_check(w, readstat_end_writing(_ptr(w))) + w.ended = true + return w +end + +function Base.close(w::Writer) + if w.ptr != C_NULL + w.began && !w.ended && end_writing!(w) + readstat_writer_free(w.ptr) + w.ptr = C_NULL + end + w.owns_io && close(w.io) + return nothing +end diff --git a/test/data/README.md b/test/data/README.md new file mode 100644 index 0000000..823126e --- /dev/null +++ b/test/data/README.md @@ -0,0 +1,27 @@ +# Test fixtures + +Sourced fixtures (verified against the upstream git blob SHAs when copied): + +- `alltypes.dta`, `stringtypes.dta`, `sample.dta`, `sample.sav`, `sample.por`, + `sample.sas7bdat`, `sample.xpt`, `string_labeled_value.sav`, + `datetime13.xpt` are copied from + [ReadStatTables.jl](https://github.com/junyuan-chen/ReadStatTables.jl) + (`data/`, MIT license). `alltypes.dta` and `stringtypes.dta` are generated + by the `alltypes.do`/`stringtypes.do` scripts kept in that repository; + the `sample.*` files there originally come from pyreadstat (see below), + with `sample.dta` slightly modified for wider test coverage. +- `sample_missing.sav`, `sample_missing.csv`, `sample_missing_user.csv`, and + `sample.zsav` are copied from + [pyreadstat](https://github.com/Roche/pyreadstat) (`test_data/basic/`, + Apache-2.0 license). The two CSV files hold the expected values of + `sample_missing.sav` with user-defined missing values collapsed to missing + and kept as data, respectively. +- `test_data_linux.sas7bdat`, `test_formats_linux.sas7bcat`, and + `sas_formatted.csv` (the expected labeled values) are copied from + pyreadstat's `test_data/sas_catalog/`. +- `schema.dct` and `fixed.txt` are hand-written fixtures for the + fixed-width-text schema path. + +The `types.{dta,sav,sas7bdat,xpt,por}` fixtures in the parent directory are +the original ReadStat.jl fixtures (`types.por` was copied from +StatFiles.jl, which ships the same table in SPSS portable form). diff --git a/test/data/alltypes.dta b/test/data/alltypes.dta new file mode 100644 index 0000000..fdad025 Binary files /dev/null and b/test/data/alltypes.dta differ diff --git a/test/data/datetime13.xpt b/test/data/datetime13.xpt new file mode 100644 index 0000000..2a33ba8 Binary files /dev/null and b/test/data/datetime13.xpt differ diff --git a/test/data/fixed.txt b/test/data/fixed.txt new file mode 100644 index 0000000..91fa727 --- /dev/null +++ b/test/data/fixed.txt @@ -0,0 +1,3 @@ + 1Alice 3.50 + 42Bob 12.25 +100Carol .75 diff --git a/test/data/sample.dta b/test/data/sample.dta new file mode 100644 index 0000000..0289ddf Binary files /dev/null and b/test/data/sample.dta differ diff --git a/test/data/sample.por b/test/data/sample.por new file mode 100644 index 0000000..bee0011 --- /dev/null +++ b/test/data/sample.por @@ -0,0 +1,14 @@ +ÁâÃÉÉ@â×ââ@×ÖÙã@ÆÉÓÅ@@@@@@@@@@@@@@@@@@@@ASCII SPSS PORT FILE +00000-0000-0000-0000--------------------!3#))0303300/240&),%00000000000000000000 +0200002'220'&)3000#0000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrst +uvwxyz .<(+0&[]!$*);^-/|,%_>?`:#@'="000000~000000000000000000000{}\0000000000000 +00000000000000000000000000000000000000000000000000000000SPSSPORTA8/201812166/172 +8211O/IBM SPSS Statistics 25.047/5B/71/6/MYCHAR1/1/0/1/1/0/C9/character70/5/MYNU +M5/8/2/5/8/2/C7/numeric70/6/MYDATE40/A/0/40/A/0/C4/date70/5/DTIME3E/K/0/3E/K/0/C +8/datetime70/6/MYLABL5/8/2/5/8/2/C7/labeled70/5/MYORD5/8/2/5/8/2/C7/ordinal70/6/ +MYTIME3D/8/0/3D/8/0/C4/timeD1/6/MYLABL2/1/4/Male2/6/FemaleD1/5/MYORD3/1/3/low2/6 +/medium3/4/highE4/N/some test text as notesO/ (Entered 15-Aug-2018)J/some othe +r commentsO/ (Entered 15-Aug-2018)F1/a1.3/IPJ2+3/IPJ3AKA/1/1/1AKA/1/b1.6/CQCMC ++2/CQCNMKA/2/2/32KA/1/c-13A.9/G9Q+4/G9Q+4/1/3/0/1/d-1.C/8CO+2/8CO+2/2/1/24KA/1/e +13A.9/*.*.1/1/*.ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ diff --git a/test/data/sample.sas7bdat b/test/data/sample.sas7bdat new file mode 100644 index 0000000..cba665f Binary files /dev/null and b/test/data/sample.sas7bdat differ diff --git a/test/data/sample.sav b/test/data/sample.sav new file mode 100644 index 0000000..20d0c5c Binary files /dev/null and b/test/data/sample.sav differ diff --git a/test/data/sample.xpt b/test/data/sample.xpt new file mode 100644 index 0000000..da15bd3 Binary files /dev/null and b/test/data/sample.xpt differ diff --git a/test/data/sample.zsav b/test/data/sample.zsav new file mode 100644 index 0000000..092c78e Binary files /dev/null and b/test/data/sample.zsav differ diff --git a/test/data/sample_missing.csv b/test/data/sample_missing.csv new file mode 100644 index 0000000..54eec2f --- /dev/null +++ b/test/data/sample_missing.csv @@ -0,0 +1,8 @@ +mychar,mynum,mydate,dtime,mylabl,myord,mytime +a,1.10,2018-05-06,2018-05-06T10:10:10.000000,1.00,1.00,10:10:10.000000 +b,1.20,1880-05-06,1880-05-06T10:10:10.000000,2.00,2.00,23:10:10.000000 +c,-1000.30,1960-01-01,1960-01-01T00:00:00.000000,1.00,3.00,0:00:00.000000 +d,-1.40,1583-01-01,1583-01-01T00:00:00.000000,2.00,1.00,16:10:10.000000 +e,1000.30,#NULL!,#NULL!,1.00,1.00,#NULL! +Z,#NULL!,#NULL!,#NULL!,#NULL!,#NULL!,#NULL! +,#NULL!,#NULL!,#NULL!,#NULL!,#NULL!,#NULL! diff --git a/test/data/sample_missing.sav b/test/data/sample_missing.sav new file mode 100644 index 0000000..b2c1243 Binary files /dev/null and b/test/data/sample_missing.sav differ diff --git a/test/data/sample_missing_user.csv b/test/data/sample_missing_user.csv new file mode 100644 index 0000000..3544782 --- /dev/null +++ b/test/data/sample_missing_user.csv @@ -0,0 +1,8 @@ +mychar,mynum,mydate,dtime,mylabl,myord,mytime +a,1.10,2018-05-06,2018-05-06T10:10:10.000000,1.00,1.00,10:10:10.000000 +b,1.20,1880-05-06,1880-05-06T10:10:10.000000,2.00,2.00,23:10:10.000000 +c,-1000.30,1960-01-01,1960-01-01T00:00:00.000000,1.00,3.00,0:00:00.000000 +d,-1.40,1583-01-01,1583-01-01T00:00:00.000000,2.00,1.00,16:10:10.000000 +e,1000.30,#NULL!,#NULL!,1.00,1.00,#NULL! +Z,-1.00,#NULL!,#NULL!,-1.00,-1.00,#NULL! +,2500.00,#NULL!,#NULL!,#NULL!,-3.00,#NULL! diff --git a/test/data/sas_formatted.csv b/test/data/sas_formatted.csv new file mode 100644 index 0000000..2f382c7 --- /dev/null +++ b/test/data/sas_formatted.csv @@ -0,0 +1,4 @@ +ID,SEXA,SEXB +ID1,Male,Male +ID2,Female,Female +ID3,Male,Male \ No newline at end of file diff --git a/test/data/schema.dct b/test/data/schema.dct new file mode 100644 index 0000000..b827199 --- /dev/null +++ b/test/data/schema.dct @@ -0,0 +1,5 @@ +infile dictionary { + _column(1) int id %3f "ID number" + _column(4) str8 name %8s "Name" + _column(12) double wage %6f "Wage" +} diff --git a/test/data/string_labeled_value.sav b/test/data/string_labeled_value.sav new file mode 100644 index 0000000..a5bb89e Binary files /dev/null and b/test/data/string_labeled_value.sav differ diff --git a/test/data/stringtypes.dta b/test/data/stringtypes.dta new file mode 100644 index 0000000..fbb31c3 Binary files /dev/null and b/test/data/stringtypes.dta differ diff --git a/test/data/test_data_linux.sas7bdat b/test/data/test_data_linux.sas7bdat new file mode 100644 index 0000000..5d88454 Binary files /dev/null and b/test/data/test_data_linux.sas7bdat differ diff --git a/test/data/test_formats_linux.sas7bcat b/test/data/test_formats_linux.sas7bcat new file mode 100644 index 0000000..d405880 Binary files /dev/null and b/test/data/test_formats_linux.sas7bcat differ diff --git a/test/runtests.jl b/test/runtests.jl index 391ebaa..c1e9078 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,16 @@ using TestItemRunner -include("test_readstat.jl") +include("test_abi.jl") +include("test_read.jl") +include("test_kwargs.jl") +include("test_missing.jl") +include("test_labels.jl") +include("test_datetime.jl") +include("test_io.jl") +include("test_threads.jl") +include("test_source.jl") +include("test_write.jl") +include("test_schema.jl") +include("test_deprecated.jl") @run_package_tests diff --git a/test/test_abi.jl b/test/test_abi.jl new file mode 100644 index 0000000..21d9c3e --- /dev/null +++ b/test/test_abi.jl @@ -0,0 +1,56 @@ +@testitem "C ABI: ReadStatValue layout" begin + using ReadStat: CAPI + using ReadStat_jll: libreadstat + + # readstat_value_t: 8-byte value union, Cint type at 8, char tag at 12, + # then two 1-bit bitfields. SysV GCC packs the bitfields into byte 13 + # (size 16); MinGW GCC uses MS bitfield layout, placing a fresh unsigned + # int unit at offset 16 (size 24). + @test sizeof(CAPI.ReadStatValue) == (Sys.iswindows() ? 24 : 16) + @test fieldoffset(CAPI.ReadStatValue, findfirst(==(:type), fieldnames(CAPI.ReadStatValue))) == 8 + @test fieldoffset(CAPI.ReadStatValue, findfirst(==(:tag), fieldnames(CAPI.ReadStatValue))) == 12 + @test fieldoffset(CAPI.ReadStatValue, findfirst(==(:bits), fieldnames(CAPI.ReadStatValue))) == + (Sys.iswindows() ? 16 : 13) + + # Behavioral check of the by-value ABI in both directions. Missing ranges + # declared on a writer variable come back as readstat_value_t BY VALUE from + # readstat_variable_get_missing_range_lo/_hi, and are then passed BY VALUE + # into the accessor functions. If the struct layout disagreed with the C + # library, these reads would come back as garbage rather than the exact + # doubles inserted below. + writer = @ccall libreadstat.readstat_writer_init()::CAPI.WriterPtr + @test writer != C_NULL + try + var = @ccall libreadstat.readstat_add_variable( + writer::CAPI.WriterPtr, "x"::Cstring, + CAPI.READSTAT_TYPE_DOUBLE::CAPI.ReadStatType, 8::Csize_t)::CAPI.VariablePtr + @test var != C_NULL + + @test (@ccall libreadstat.readstat_variable_add_missing_double_range( + var::CAPI.VariablePtr, (-99.5)::Cdouble, (-90.25)::Cdouble)::CAPI.ReadStatError) == + CAPI.READSTAT_OK + @test (@ccall libreadstat.readstat_variable_add_missing_double_value( + var::CAPI.VariablePtr, 999.0::Cdouble)::CAPI.ReadStatError) == CAPI.READSTAT_OK + + @test CAPI.readstat_variable_get_missing_ranges_count(var) == 2 + + lo = CAPI.readstat_variable_get_missing_range_lo(var, 0) + hi = CAPI.readstat_variable_get_missing_range_hi(var, 0) + @test CAPI.readstat_value_type(lo) == CAPI.READSTAT_TYPE_DOUBLE + @test CAPI.readstat_double_value(lo) == -99.5 + @test CAPI.readstat_double_value(hi) == -90.25 + @test CAPI.readstat_value_tag(lo) == '\0' + @test !CAPI.readstat_value_is_system_missing(lo) + @test !CAPI.readstat_value_is_tagged_missing(lo) + + # A singleton "range" (lo == hi) carrying the flag bits through the + # by-value path: 999.0 is defined missing for this variable. + v = CAPI.readstat_variable_get_missing_range_lo(var, 1) + @test CAPI.readstat_double_value(v) == 999.0 + @test CAPI.readstat_value_is_defined_missing(v, var) + @test CAPI.readstat_value_is_missing(v, var) + @test !CAPI.readstat_value_is_missing(v, C_NULL) + finally + @ccall libreadstat.readstat_writer_free(writer::CAPI.WriterPtr)::Cvoid + end +end diff --git a/test/test_datetime.jl b/test/test_datetime.jl new file mode 100644 index 0000000..68228bd --- /dev/null +++ b/test/test_datetime.jl @@ -0,0 +1,95 @@ +@testitem "date/time conversion" begin + using DataValues + using Dates + + dir = joinpath(@__DIR__, "data") + + expected_dates = DataValueArray{Date}( + [Date(2018, 5, 6), Date(1880, 5, 6), Date(1960, 1, 1), Date(1583, 1, 1), NA]) + expected_dtimes = DataValueArray{DateTime}( + [DateTime(2018, 5, 6, 10, 10, 10), DateTime(1880, 5, 6, 10, 10, 10), + DateTime(1960, 1, 1), DateTime(1583, 1, 1), NA]) + expected_times = [36610.0, 83410.0, 0.0, 58210.0] # seconds of day, row 5 NA + + @testset "$file" for (file, datecol, dtimecol, timecol) in + (("sample.dta", :mydate, :dtime, nothing), # mytime in .dta is %tc, tested below + ("sample.sav", :mydate, :dtime, :mytime), + ("sample.por", :MYDATE, :DTIME, :MYTIME), + ("sample.sas7bdat", :mydate, :dtime, :mytime), + ("sample.xpt", :MYDATE, :DTIME, :MYTIME)) + + tbl = readstat(joinpath(dir, file)) + @test tbl[datecol] == expected_dates + @test tbl[dtimecol] == expected_dtimes + if timecol !== nothing + col = tbl[timecol] + @test eltype(eltype(col)) === HMS + @test [unwrap(get(x)) for x in col[1:4]] == expected_times + @test isna(col[5]) + end + + # Raw numbers when conversion is off. + raw = readstat(joinpath(dir, file); convert_datetime=false) + @test eltype(eltype(raw[datecol])) <: Number + end + + @testset "Stata %tc time-of-day and DATETIME13" begin + tbl = read_dta(joinpath(dir, "sample.dta")) + # mytime uses %tcHH:MM:SS — a datetime anchored at the epoch day. + @test get(tbl[:mytime][1]) == DateTime(1960, 1, 1, 10, 10, 10) + + x = readstat(joinpath(dir, "datetime13.xpt")) + @test get(x[:DTTEST][1]) == DateTime(1960, 1, 2, 10, 17, 36) + + a = read_dta(joinpath(dir, "alltypes.dta")) + @test get(a[:vdate][1]) == Date(1960, 1, 2) # %td, value 1 + @test get(a[:vtime][1]) == DateTime(1960, 1, 1, 0, 0, 0, 1) # %tc, value 1 + @test isna(a[:vdate][2]) && isna(a[:vdate][3]) + end + + @testset "HMS" begin + t = HMS(36610.0) + @test unwrap(t) == 36610.0 + @test Dates.hour(t) == 10 + @test Dates.minute(t) == 10 + @test Dates.second(t) == 10 + @test sprint(show, t) == "10:10:10" + @test Time(t) == Time(10, 10, 10) + + # Durations beyond a day and negative durations are representable. + long = HMS(30 * 3600.0) + @test Dates.hour(long) == 30 + @test sprint(show, long) == "30:00:00" + @test_throws ArgumentError Time(long) + @test sprint(show, HMS(-3661.0)) == "-1:01:01" + @test Dates.hour(HMS(-3661.0)) == -1 + + @test sprint(show, HMS(0.5)) == "0:00:00.500" + @test HMS(1.0) < HMS(2.0) + @test HMS(1.0) == HMS(1.0) + @test hash(HMS(1.0)) == hash(HMS(1.0)) + end + + @testset "format classification" begin + @test ReadStat.datetime_rule("%tm", :dta) !== nothing + f, T = ReadStat.datetime_rule("%tm", :dta) + @test T === Date && f(1) == Date(1960, 2, 1) + f, T = ReadStat.datetime_rule("%tq", :dta) + @test f(1) == Date(1960, 4, 1) + f, T = ReadStat.datetime_rule("%th", :dta) + @test f(1) == Date(1960, 7, 1) + f, T = ReadStat.datetime_rule("%ty", :dta) + @test f(1987) == Date(1987, 1, 1) + f, T = ReadStat.datetime_rule("%tw", :dta) + @test f(0) == Date(1960, 1, 1) + @test f(52) == Date(1961, 1, 1) + + @test ReadStat.datetime_rule("F8.2", :sav) === nothing + @test ReadStat.datetime_rule("BEST12", :sas7bdat) === nothing + @test ReadStat.datetime_rule("", :dta) === nothing + @test ReadStat.datetime_rule("DATETIME22.3", :sas7bdat) !== nothing + @test ReadStat.datetime_rule("MMDDYYS10", :xport) !== nothing + @test ReadStat.datetime_rule("SDATE10", :por) !== nothing + @test ReadStat.datetime_rule("DTIME10", :sav) !== nothing + end +end diff --git a/test/test_deprecated.jl b/test/test_deprecated.jl new file mode 100644 index 0000000..1ecbe50 --- /dev/null +++ b/test/test_deprecated.jl @@ -0,0 +1,32 @@ +@testitem "deprecated 1.x API" begin + using DataValues + using Dates + + df = read_dta(joinpath(@__DIR__, "types.dta")) + @test df isa ReadStatDataFrame + @test ReadStatDataFrame === ReadStatTable + + data = df.data + @test data isa Vector{Any} + @test length(data) == 6 + @test data[1] == DataValueArray{Float32}([3.14, 7.0, NA]) + @test data[5] == DataValueArray{Int8}([2, 7, NA]) + + @test df.headers == [:vfloat, :vdouble, :vlong, :vint, :vbyte, :vstring] + @test df.types == [Float32, Float64, Int32, Int16, Int8, String] + @test df.types_as_int == Int32[4, 5, 3, 2, 1, 0] + @test df.labels == fill("", 6) + @test df.formats isa Vector{String} + @test df.storagewidths isa Vector{Csize_t} + @test df.measures == zeros(Int32, 6) + @test df.alignments == Int32[3, 3, 3, 3, 3, 3] + @test df.val_label_keys == fill("", 6) + @test df.val_label_dict isa Dict{String,Dict{Any,String}} + @test isempty(df.val_label_dict) + @test df.rows == 3 + @test df.columns == 6 + @test df.filelabel == "" + @test df.timestamp isa DateTime + @test df.format isa Clong + @test df.hasmissings == fill(false, 6) +end diff --git a/test/test_io.jl b/test/test_io.jl new file mode 100644 index 0000000..9a9603d --- /dev/null +++ b/test/test_io.jl @@ -0,0 +1,47 @@ +@testitem "reading from IO" begin + using DataValues + + dir = @__DIR__ + + # A minimal read-only, non-seekable stream to exercise the buffering path. + struct NoSeekIO <: IO + inner::IOBuffer + end + Base.read(io::NoSeekIO, ::Type{UInt8}) = read(io.inner, UInt8) + Base.eof(io::NoSeekIO) = eof(io.inner) + + @testset "$file" for (reader, file) in + ((read_dta, "types.dta"), (read_sav, "types.sav"), + (read_sas7bdat, "types.sas7bdat"), (read_xport, "types.xpt")) + path = joinpath(dir, file) + expected = reader(path) + + for source in (open(path), IOBuffer(read(path)), NoSeekIO(IOBuffer(read(path)))) + tbl = reader(source) + @test names(tbl) == names(expected) + @test size(tbl) == size(expected) + for i in 1:size(tbl, 2) + @test tbl[i] == expected[i] + end + source isa IOStream && close(source) + end + end + + @testset "kwargs compose with IO input" begin + path = joinpath(dir, "types.dta") + tbl = read_dta(IOBuffer(read(path)); usecols=[:vlong], row_limit=2) + @test names(tbl) == [:vlong] + @test tbl[:vlong] == DataValueArray{Int32}([2, 7]) + end + + @testset "dispatcher and read_meta over IO" begin + path = joinpath(dir, "types.sav") + io = IOBuffer(read(path)) + @test size(readstat(io; format=:sav)) == (3, 6) + @test_throws ArgumentError readstat(IOBuffer(read(path))) + + m = read_meta(IOBuffer(read(path)); format=:sav) + @test size(m) == (0, 6) + @test filemetadata(m).row_count == 3 + end +end diff --git a/test/test_kwargs.jl b/test/test_kwargs.jl new file mode 100644 index 0000000..a0cdeb0 --- /dev/null +++ b/test/test_kwargs.jl @@ -0,0 +1,88 @@ +@testitem "reader kwargs" begin + using DataValues + + dta = joinpath(@__DIR__, "types.dta") + xpt = joinpath(@__DIR__, "types.xpt") + + @testset "usecols" begin + tbl = read_dta(dta; usecols=[:vint, :vstring]) + @test names(tbl) == [:vint, :vstring] + @test size(tbl) == (3, 2) + @test tbl[:vint] == DataValueArray{Int16}([2, 7, NA]) + @test varmetadata(tbl, :vstring).name === :vstring + + @test names(read_dta(dta; usecols=1)) == [:vfloat] + @test names(read_dta(dta; usecols=:vbyte)) == [:vbyte] + @test names(read_dta(dta; usecols=[2, 4])) == [:vdouble, :vint] + @test names(read_dta(dta; usecols=r"^vd")) == [:vdouble] + @test names(read_dta(dta; usecols=n -> startswith(string(n), "vs"))) == [:vstring] + @test size(read_dta(dta; usecols=Symbol[])) == (0, 0) + + # Column selection composes with row selection on the C side. + tbl = read_dta(dta; usecols=[:vlong], row_limit=2) + @test size(tbl) == (2, 1) + @test tbl[:vlong] == DataValueArray{Int32}([2, 7]) + end + + @testset "row_limit / row_offset" begin + tbl = read_dta(dta; row_limit=2) + @test size(tbl) == (2, 6) + @test tbl[:vlong] == DataValueArray{Int32}([2, 7]) + + tbl = read_dta(dta; row_offset=1) + @test size(tbl) == (2, 6) + @test tbl[:vlong] == DataValueArray{Int32}([7, NA]) + + tbl = read_dta(dta; row_offset=1, row_limit=1) + @test size(tbl) == (1, 6) + @test tbl[:vlong] == DataValueArray{Int32}([7]) + + @test size(read_dta(dta; row_limit=0)) == (0, 6) + + # Formats with unknown row counts grow their buffers row by row. + tbl = read_xport(xpt; row_limit=2) + @test size(tbl) == (2, 6) + tbl = read_xport(xpt; row_offset=1) + @test size(tbl) == (2, 6) + @test tbl[:vlong] == DataValueArray{Int32}([7, NA]) + + @test_throws ArgumentError read_dta(dta; row_offset=-1) + @test_throws ArgumentError read_dta(dta; row_limit=-1) + end + + @testset "read_meta" begin + tbl = read_meta(dta) + @test size(tbl) == (0, 6) + @test names(tbl) == [:vfloat, :vdouble, :vlong, :vint, :vbyte, :vstring] + @test filemetadata(tbl).row_count == 3 + @test varmetadata(tbl, :vint).name === :vint + + @test filemetadata(read_meta(joinpath(@__DIR__, "types.sav"))).row_count == 3 + @test filemetadata(read_meta(xpt)).row_count == -1 + end + + @testset "readstat dispatcher" begin + for file in ("types.dta", "types.sav", "types.sas7bdat", "types.xpt") + @test size(readstat(joinpath(@__DIR__, file)), 2) == 6 + end + @test size(readstat(dta; format=:dta), 1) == 3 + @test_throws ArgumentError readstat("data.unknown") + end + + @testset "progress" begin + fractions = Float64[] + tbl = read_dta(dta; progress=p -> (push!(fractions, p); true)) + @test size(tbl) == (3, 6) + @test !isempty(fractions) + @test all(0.0 .<= fractions .<= 1.0) + + # Returning false stops the parse; the rows read so far come back. + tbl = read_dta(dta; progress=p -> false) + @test size(tbl, 1) <= 3 + end + + @testset "encodings" begin + tbl = read_dta(dta; handler_encoding="UTF-8") + @test tbl[:vstring] == DataValueArray{String}(["2", "7", ""]) + end +end diff --git a/test/test_labels.jl b/test/test_labels.jl new file mode 100644 index 0000000..f617f8c --- /dev/null +++ b/test/test_labels.jl @@ -0,0 +1,92 @@ +@testitem "value labels" begin + using DataValues + + dir = joinpath(@__DIR__, "data") + + @testset "LabeledValue / LabeledArray semantics" begin + tbl = read_sav(joinpath(dir, "sample.sav")) + d = valuelabels(tbl, :mylabl) + @test d == Dict(1.0 => "Male", 2.0 => "Female") + + la = labeled(tbl, :mylabl) + @test la isa LabeledArray + @test length(la) == 5 + @test rawvalues(la) === tbl[:mylabl] + @test getvaluelabels(la) === d + + x = la[1] + @test x isa DataValue{LabeledValue{Float64}} + lv = get(x) + @test unwrap(lv) == 1.0 + @test valuelabel(lv) == "Male" + + # Computes as the code, compares against strings as the label. + @test lv == 1.0 + @test 1.0 == lv + @test lv == "Male" + @test lv != "Female" + @test hash(lv) == hash(1.0) + lv2 = get(la[2]) + @test lv != lv2 + @test isless(lv, lv2) + @test sort([lv2, lv]) == [lv, lv2] + + # Displays as the label. + @test sprint(print, lv) == "Male" + @test sprint(show, MIME"text/plain"(), lv) == "Male (1.0)" + + # Partial label sets: unlabeled codes fall back to string(code). + @test valuelabel(LabeledValue(9.0, d)) == "9.0" + + @test_throws ArgumentError labeled(tbl, :mychar) + end + + @testset "apply_value_labels" begin + tbl = read_sav(joinpath(dir, "sample.sav"); apply_value_labels=true) + @test tbl[:mylabl] isa LabeledArray + @test tbl[:myord] isa LabeledArray + @test !(tbl[:mynum] isa LabeledArray) + @test labeled(tbl, :mylabl) === tbl[:mylabl] + @test [sprint(print, get(x)) for x in tbl[:mylabl]] == + ["Male", "Female", "Male", "Female", "Male"] + @test rawvalues(tbl[:mylabl]) == DataValueArray{Float64}([1, 2, 1, 2, 1]) + @test occursin("Male", sprint(show, MIME"text/plain"(), tbl)) + end + + @testset "string-keyed labels" begin + tbl = read_sav(joinpath(dir, "string_labeled_value.sav"); apply_value_labels=true) + la = tbl[:v611] + @test la isa LabeledArray + @test unwrap(get(la[1])) == "BE33" + @test valuelabel(get(la[1])) == "Prov. Liege" + end + + @testset "sas7bcat catalogs" begin + catpath = joinpath(dir, "test_formats_linux.sas7bcat") + cat = read_sas7bcat(catpath) + @test cat[Symbol("\$A")]["1"] == "Male" + @test cat[Symbol("\$A")]["2"] == "Female" + + tbl = read_sas7bdat(joinpath(dir, "test_data_linux.sas7bdat"); + catalog=catpath, apply_value_labels=true) + @test varmetadata(tbl, :SEXA).vallabel == Symbol("\$A") + @test tbl[:SEXA] isa LabeledArray + # Expected labels from pyreadstat's sas_formatted.csv. + @test [valuelabel(get(x)) for x in tbl[:SEXA]] == ["Male", "Female", "Male"] + @test [valuelabel(get(x)) for x in tbl[:SEXB]] == ["Male", "Female", "Male"] + + # Without the catalog the label sets are simply absent. + plain = read_sas7bdat(joinpath(dir, "test_data_linux.sas7bdat")) + @test valuelabels(plain, :SEXA) === nothing + + @test_throws ArgumentError read_dta(joinpath(@__DIR__, "types.dta"); + catalog=catpath) + end + + @testset "labeled view over tagged missings (alltypes.dta)" begin + tbl = read_dta(joinpath(dir, "alltypes.dta")) + la = labeled(tbl, :vbyte) + @test valuelabel(get(la[1])) == "A" + @test isna(la[2]) + end +end diff --git a/test/test_missing.jl b/test/test_missing.jl new file mode 100644 index 0000000..9ca0b8d --- /dev/null +++ b/test/test_missing.jl @@ -0,0 +1,78 @@ +@testitem "missing values" begin + using DataValues + + @testset "tagged missings (alltypes.dta)" begin + tbl = read_dta(joinpath(@__DIR__, "data", "alltypes.dta")) + @test size(tbl, 1) == 3 + + # Rows: 1 = the value 1, 2 = .a (tagged missing), 3 = system missing. + for col in (:vbyte, :vint, :vlong, :vfloat, :vdouble) + c = tbl[col] + @test !isna(c[1]) && get(c[1]) == 1 + @test isna(c[2]) + @test isna(c[3]) + @test missingtags(tbl, col) == ['\0', 'a', '\0'] + end + @test missingtags(tbl, :vstr) === nothing + + # The label set covers a value and a tagged missing value. + d = valuelabels(tbl, :vbyte) + @test d !== nothing + @test d[Int32(1)] == "A" + @test d['a'] == "Tagged missing" + + # strL values surface as Strings. + @test occursin("long string!", get(tbl[:vstrL][1])) + end + + @testset "SPSS user-defined missings (sample_missing.sav)" begin + path = joinpath(@__DIR__, "data", "sample_missing.sav") + + # Default: user-defined missing values collapse to NA + # (expected values from pyreadstat's sample_missing.csv). + tbl = read_sav(path) + @test tbl[:mychar] == DataValueArray{String}(["a", "b", "c", "d", "e", "Z", ""]) + @test tbl[:mynum] == DataValueArray{Float64}([1.1, 1.2, -1000.3, -1.4, 1000.3, NA, NA]) + @test isna.(tbl[:mydate]) == [false, false, false, false, true, true, true] + @test !isempty(varmetadata(tbl, :mynum).missing_ranges) + + # :keep returns the user-missing codes as data + # (expected values from pyreadstat's sample_missing_user.csv). + kept = read_sav(path; user_missing=:keep) + @test kept[:mynum] == + DataValueArray{Float64}([1.1, 1.2, -1000.3, -1.4, 1000.3, -1.0, 2500.0]) + @test kept[:mylabl] == DataValueArray{Float64}([1, 2, 1, 2, 1, -1, NA]) + @test kept[:myord] == DataValueArray{Float64}([1, 2, 3, 1, 1, -1, -3]) + # System-missing cells stay NA either way. + @test isna.(kept[:mydate]) == [false, false, false, false, true, true, true] + + @test_throws ArgumentError read_sav(path; user_missing=:bogus) + end +end + +@testitem "read_por" begin + using DataValues + + tbl = read_por(joinpath(@__DIR__, "data", "sample.por")) + @test size(tbl) == (5, 7) + @test filemetadata(tbl).row_count == -1 + @test names(tbl) == [:MYCHAR, :MYNUM, :MYDATE, :DTIME, :MYLABL, :MYORD, :MYTIME] + @test tbl[:MYCHAR] == DataValueArray{String}(["a", "b", "c", "d", "e"]) + @test tbl[:MYNUM] == DataValueArray{Float64}([1.1, 1.2, -1000.3, -1.4, 1000.3]) + @test isna(tbl[:MYDATE][5]) + @test varmetadata(tbl, :MYDATE).format == "EDATE10" + d = valuelabels(tbl, :MYLABL) + @test d !== nothing && d[1.0] == "Male" && d[2.0] == "Female" + + # The types.por fixture inherited from StatFiles.jl has an invalid + # timestamp that readstat 1.1.9 rejects; the error must surface cleanly + # (queryverse/ReadStat.jl#96, queryverse/StatFiles.jl#32). + err = try + read_por(joinpath(@__DIR__, "types.por")) + nothing + catch e + e + end + @test err isa ErrorException + @test occursin("timestamp", err.msg) +end diff --git a/test/test_read.jl b/test/test_read.jl new file mode 100644 index 0000000..ed4c8b8 --- /dev/null +++ b/test/test_read.jl @@ -0,0 +1,59 @@ +@testitem "read basics" begin + using DataValues + using Dates + using ReadStat: CAPI + + # Expected alignments/measures are the raw enum values as stored in the + # fixtures; every fixture reports measure UNKNOWN (0). + @testset "types.$ext" for (reader, ext, alignments, has_row_count) in + ((read_dta, "dta", [3, 3, 3, 3, 3, 3], true), + (read_sav, "sav", [0, 0, 0, 0, 0, 0], true), + (read_sas7bdat, "sas7bdat", [0, 0, 0, 0, 0, 0], true), + (read_xport, "xpt", [3, 3, 3, 3, 3, 1], false)) + + tbl = reader(joinpath(@__DIR__, "types.$ext")) + + @test size(tbl) == (3, 6) + @test size(tbl, 1) == 3 + @test size(tbl, 2) == 6 + @test names(tbl) == [:vfloat, :vdouble, :vlong, :vint, :vbyte, :vstring] + + @test tbl[:vfloat] == DataValueArray{Float32}([3.14, 7.0, NA]) + @test tbl[:vdouble] == DataValueArray{Float64}([3.14, 7.0, NA]) + @test tbl[:vlong] == DataValueArray{Int32}([2, 7, NA]) + @test tbl[:vint] == DataValueArray{Int16}([2, 7, NA]) + @test tbl[:vbyte] == DataValueArray{Int8}([2, 7, NA]) + # None of these formats has a distinct missing state for strings — + # Stata's missing string literally is the empty string — so the C + # library delivers "" as a regular value and the cell is not NA. + @test tbl[:vstring] == DataValueArray{String}(["2", "7", ""]) + @test tbl[1] === tbl[:vfloat] + @test_throws ArgumentError tbl[:nonexistent] + + meta = filemetadata(tbl) + @test meta.var_count == 6 + @test meta.row_count == (has_row_count ? 3 : -1) + @test meta.modified_time isa DateTime + @test meta.modified_time > DateTime(2000) + @test isempty(meta.notes) + @test isempty(meta.value_labels) + + @test [Int(varmetadata(tbl, i).alignment) for i in 1:6] == alignments + @test all(varmetadata(tbl, i).measure == CAPI.READSTAT_MEASURE_UNKNOWN for i in 1:6) + @test varmetadata(tbl, :vstring).type == CAPI.READSTAT_TYPE_STRING + # Only Stata has a distinct float storage type; SPSS and SAS report + # doubles (the equality checks above compare numerically). + @test varmetadata(tbl, :vfloat).type == + (ext == "dta" ? CAPI.READSTAT_TYPE_FLOAT : CAPI.READSTAT_TYPE_DOUBLE) + @test varmetadata(tbl, :vbyte).name === :vbyte + @test all(isempty(varmetadata(tbl, i).missing_ranges) for i in 1:6) + @test valuelabels(tbl, :vfloat) === nothing + + rendered = sprint(show, MIME"text/plain"(), tbl; context=:limit => true) + @test occursin("3x6 ReadStatTable", rendered) + @test occursin("vstring", rendered) + @test occursin("NA", rendered) + end + + @test_throws ArgumentError read_dta(joinpath(@__DIR__, "no_such_file.dta")) +end diff --git a/test/test_readstat.jl b/test/test_readstat.jl deleted file mode 100644 index d9fcee6..0000000 --- a/test/test_readstat.jl +++ /dev/null @@ -1,35 +0,0 @@ -@testitem "ReadStat" begin - using DataValues - - # Expected alignments are readstat_alignment_t values: - # 0 = UNKNOWN, 1 = LEFT, 2 = CENTER, 3 = RIGHT. - @testset "ReadStat: $ext files" for (reader, ext, alignments) in - ((read_dta, "dta", Int32[3, 3, 3, 3, 3, 3]), - (read_sav, "sav", Int32[0, 0, 0, 0, 0, 0]), - (read_sas7bdat, "sas7bdat", Int32[0, 0, 0, 0, 0, 0]), - (read_xport, "xpt", Int32[3, 3, 3, 3, 3, 1])) - - dtafile = joinpath(@__DIR__, "types.$ext") - rsdf = reader(dtafile) - data = rsdf.data - - @test length(data) == 6 - @test rsdf.headers == [:vfloat, :vdouble, :vlong, :vint, :vbyte, :vstring] - @test data[1] == DataValueArray{Float32}([3.14, 7., NA]) - @test data[2] == DataValueArray{Float64}([3.14, 7., NA]) - @test data[3] == DataValueArray{Int32}([2, 7, NA]) - @test data[4] == DataValueArray{Int16}([2, 7, NA]) - @test data[5] == DataValueArray{Int8}([2, 7., NA]) - @test data[6] == DataValueArray{String}(["2", "7", ""]) - - # Alignments must come from readstat_variable_get_alignment, not from - # the measure accessor sitting next to it in the C API. Every fixture - # reports measure UNKNOWN, so reading the wrong one yields all zeros. - @test rsdf.alignments == alignments - @test rsdf.measures == Int32[0, 0, 0, 0, 0, 0] - - # Every readstat_type_t the readers emit must map to a concrete Julia - # type; a gap in that mapping used to surface as Nothing. - @test all(!=(Nothing), rsdf.types) - end -end diff --git a/test/test_schema.jl b/test/test_schema.jl new file mode 100644 index 0000000..74ae672 --- /dev/null +++ b/test/test_schema.jl @@ -0,0 +1,27 @@ +@testitem "fixed-width text via schema files" begin + using DataValues + + dir = joinpath(@__DIR__, "data") + data = joinpath(dir, "fixed.txt") + dct = joinpath(dir, "schema.dct") + + tbl = read_txt(data, dct; schema_format=:stata_dictionary) + @test size(tbl) == (3, 3) + @test names(tbl) == [:id, :name, :wage] + @test tbl[:id] == DataValueArray{Int16}([1, 42, 100]) + @test [get(x) for x in tbl[:name]] == ["Alice", "Bob", "Carol"] + @test tbl[:wage] == DataValueArray{Float64}([3.5, 12.25, 0.75]) + @test varmetadata(tbl, :id).label == "ID number" + + # The reader kwargs push down as usual. + part = read_txt(data, dct; schema_format=:stata_dictionary, + usecols=[:name], row_limit=2, row_offset=1) + @test size(part) == (2, 1) + @test [get(x) for x in part[:name]] == ["Bob", "Carol"] + + @test_throws ArgumentError read_txt(data, dct; schema_format=:nonsense) + @test_throws ArgumentError read_txt("no_such.txt", dct; + schema_format=:stata_dictionary) + # A data file is not a valid schema file. + @test_throws Exception read_txt(data, data; schema_format=:stata_dictionary) +end diff --git a/test/test_source.jl b/test/test_source.jl new file mode 100644 index 0000000..629e53d --- /dev/null +++ b/test/test_source.jl @@ -0,0 +1,99 @@ +@testitem "ReadStatSource and chunks" begin + using DataValues + using Dates + + dir = joinpath(@__DIR__, "data") + path = joinpath(dir, "sample.dta") + + @testset "planning surface" begin + src = ReadStatSource(path; apply_value_labels=true) + s = schema(src) + @test size(s, 1) == 0 + @test colnames(src) == [:mychar, :mynum, :mydate, :dtime, :mylabl, :myord, :mytime] + @test nrows(src) == 5 + ct = coltypes(src) + @test ct[1] == DataValue{String} + @test ct[3] == DataValue{Date} + @test ct[4] == DataValue{DateTime} + @test supports(src, :projection) + @test supports(src, :row_range) + @test supports(src, :row_count) + @test supports(src, :parallel) + @test !supports(src, :frobnicate) + + xsrc = ReadStatSource(joinpath(dir, "sample.xpt")) + @test ismissing(nrows(xsrc)) + @test !supports(xsrc, :row_count) + @test !supports(xsrc, :parallel) + + # The schema's column types match what a read produces. + full = read(src) + @test size(full) == (5, 7) + @test [eltype(full[i]) for i in 1:7] == ct + @test full[:mylabl] isa LabeledArray + + part = read(src; rows=2:4, usecols=[:mynum]) + @test size(part) == (3, 1) + @test part[:mynum] == read_dta(path)[:mynum][2:4] + @test_throws ArgumentError read(src; rows=0:3) + + # An IO-backed source serves schema and repeated reads off one stream. + io_src = ReadStatSource(open(path); format=:dta) + @test nrows(io_src) == 5 + @test size(read(io_src)) == (5, 7) + @test size(read(io_src; rows=1:2)) == (2, 7) + @test_throws ArgumentError ReadStatSource(IOBuffer(UInt8[])) + end + + @testset "chunks ≡ full read" begin + src = ReadStatSource(path; apply_value_labels=true) + full = read(src) + pieces = collect(chunks(src; chunksize=2)) + @test length(pieces) == 3 + @test [size(p, 1) for p in pieces] == [2, 2, 1] + for j in 1:size(full, 2) + @test isequal(reduce(vcat, [collect(p[j]) for p in pieces]), collect(full[j])) + end + # Metadata and labels are complete from the first chunk on. + @test pieces[1][:mylabl] isa LabeledArray + @test valuelabel(get(pieces[1][:mylabl][1])) in ("Male", "Female") + @test names(pieces[1]) == names(full) + + # A chunk size beyond the file yields one chunk. + @test length(collect(chunks(src; chunksize=100))) == 1 + end + + @testset "chunks with pushdown and unknown row counts" begin + src = ReadStatSource(path) + expected = read(src; rows=2:5, usecols=[:mynum, :mydate]) + pieces = collect(chunks(src; rows=2:5, usecols=[:mynum, :mydate], chunksize=3)) + @test [size(p, 1) for p in pieces] == [3, 1] + @test names(pieces[1]) == [:mynum, :mydate] + for j in 1:2 + @test isequal(reduce(vcat, [collect(p[j]) for p in pieces]), collect(expected[j])) + end + + # Formats without a recorded row count stream fine. + xsrc = ReadStatSource(joinpath(dir, "sample.xpt")) + pieces = collect(chunks(xsrc; chunksize=2)) + @test [size(p, 1) for p in pieces] == [2, 2, 1] + end + + @testset "early termination" begin + src = ReadStatSource(path) + it = chunks(src; chunksize=1) + step = iterate(it) + @test step !== nothing + @test size(step[1], 1) == 1 + close(it) + # Chunks already buffered when the iterator was closed still drain + # (bounded channel of 2); after that, iteration ends and the parse + # task has been released rather than blocking forever. + drained = collect(it) + @test length(drained) <= 2 + @test iterate(it) === nothing + end + + # zsav coverage rides along here. + @test size(read_sav(joinpath(dir, "sample.zsav"))) == (5, 7) +end diff --git a/test/test_threads.jl b/test/test_threads.jl new file mode 100644 index 0000000..89aa97c --- /dev/null +++ b/test/test_threads.jl @@ -0,0 +1,63 @@ +@testitem "multi-task reads" begin + using DataValues + + dir = joinpath(@__DIR__, "data") + + function tables_equal(a, b) + names(a) == names(b) || return false + size(a) == size(b) || return false + for i in 1:size(a, 2) + typeof(a[i]) == typeof(b[i]) || return false + a[i] isa LabeledArray ? (rawvalues(a[i]) == rawvalues(b[i]) || return false) : + (a[i] == b[i] || return false) + missingtags(a, i) == missingtags(b, i) || return false + end + fa, fb = filemetadata(a), filemetadata(b) + return fa.value_labels == fb.value_labels && fa.notes == fb.notes + end + + @testset "$file ≡ serial" for file in + ("sample.dta", "sample.sav", "sample.sas7bdat", "alltypes.dta") + path = joinpath(dir, file) + serial = readstat(path; ntasks=1) + for T in (2, 3, 8) + @test tables_equal(readstat(path; ntasks=T), serial) + end + end + + @testset "kwargs compose with ntasks" begin + path = joinpath(dir, "sample.dta") + serial = read_dta(path; row_offset=1, row_limit=3) + threaded = read_dta(path; row_offset=1, row_limit=3, ntasks=2) + @test tables_equal(threaded, serial) + # The pre-pass reports the file's true row count even under a limit. + @test filemetadata(threaded).row_count == 5 + + @test tables_equal(read_dta(path; usecols=[:mynum, :mylabl], ntasks=2), + read_dta(path; usecols=[:mynum, :mylabl])) + @test tables_equal( + read_sav(joinpath(dir, "sample_missing.sav"); user_missing=:keep, ntasks=2), + read_sav(joinpath(dir, "sample_missing.sav"); user_missing=:keep)) + @test tables_equal( + read_sav(joinpath(dir, "sample.sav"); apply_value_labels=true, ntasks=3), + read_sav(joinpath(dir, "sample.sav"); apply_value_labels=true)) + end + + @testset "tags survive the chunk merge" begin + tbl = read_dta(joinpath(dir, "alltypes.dta"); ntasks=3) + @test missingtags(tbl, :vbyte) == ['\0', 'a', '\0'] + @test valuelabels(tbl, :vbyte)['a'] == "Tagged missing" + end + + @testset "fallbacks" begin + # Unknown row count: xport cannot split and parses serially. + xpt = joinpath(dir, "sample.xpt") + @test tables_equal(readstat(xpt; ntasks=4), readstat(xpt)) + # More tasks than rows. + @test size(read_dta(joinpath(dir, "sample.dta"); ntasks=100), 1) == 5 + # Zero-row reads keep full metadata. + tbl = read_sav(joinpath(dir, "sample.sav"); row_limit=0, ntasks=2) + @test size(tbl) == (0, 7) + @test !isempty(filemetadata(tbl).value_labels) + end +end diff --git a/test/test_write.jl b/test/test_write.jl new file mode 100644 index 0000000..b31fc64 --- /dev/null +++ b/test/test_write.jl @@ -0,0 +1,182 @@ +@testitem "writer round trips" begin + using DataValues + using Dates + using ReadStat: Writer, add_variable!, add_string_ref!, begin_writing!, + begin_row!, insert_string_ref!, end_row! + using ReadStat: CAPI + + tmp = mktempdir() + cols = Any[ + DataValueArray{Int8}([1, 2, NA]), + DataValueArray{Int16}([10, 20, NA]), + DataValueArray{Int32}([100, 200, NA]), + DataValueArray{Float32}([1.5f0, 2.5f0, NA]), + DataValueArray{Float64}([1.25, 2.75, NA]), + DataValueArray{String}(["ab", "cdef", NA]), + DataValueArray{Date}([Date(2020, 1, 2), Date(1959, 12, 31), NA]), + DataValueArray{DateTime}([DateTime(2020, 1, 2, 3, 4, 5), DateTime(1960, 1, 1), NA]), + ] + cnames = [:vbyte, :vint, :vlong, :vfloat, :vdouble, :vstr, :vdate, :vdt] + writers = (dta=write_dta, sav=write_sav, por=write_por, + sas7bdat=write_sas7bdat, xport=write_xport) + + @testset "all types via $fmt" for fmt in (:dta, :sav, :por, :sas7bdat, :xport) + path = joinpath(tmp, "types_out.$(fmt === :xport ? "xpt" : fmt)") + writers[fmt](path, cols, cnames; file_label="roundtrip") + tbl = readstat(path; format=fmt) + + @test size(tbl) == (3, 8) + expected_names = fmt === :por ? + [Symbol(uppercase(String(n))) for n in cnames] : cnames + @test names(tbl) == expected_names + for i in 1:5 + @test tbl[i] == cols[i] # numeric equality across widened storage + end + @test get(tbl[6][1]) == "ab" && get(tbl[6][2]) == "cdef" + # A missing string cell comes back as the format's missing string, + # which is the empty string. + @test isna(tbl[6][3]) || get(tbl[6][3]) == "" + @test tbl[7] == cols[7] + @test tbl[8] == cols[8] + @test filemetadata(tbl).file_label == "roundtrip" + + # Writing into an IO works the same. + io = IOBuffer() + writers[fmt](io, cols, cnames) + @test size(readstat(IOBuffer(take!(io)); format=fmt)) == (3, 8) + end + + @testset "labels, tags, notes, attributes (dta)" begin + path = joinpath(tmp, "labels_out.dta") + write_dta(path, Any[DataValueArray{Int32}([1, 2, NA])], [:grade]; + value_labels=Dict(:gl => ReadStat.ValueLabelDict( + Int32(1) => "low", Int32(2) => "high", 'a' => "refused")), + vallabels=[:gl], + tags=[['\0', '\0', 'a']], + labels=["the grade"], + notes=["a note", "another note"], + file_label="labeled", + timestamp=DateTime(2020, 6, 1, 12, 0, 0)) + tbl = read_dta(path) + @test missingtags(tbl, :grade) == ['\0', '\0', 'a'] + d = valuelabels(tbl, :grade) + @test d[Int32(1)] == "low" && d[Int32(2)] == "high" && d['a'] == "refused" + @test varmetadata(tbl, :grade).label == "the grade" + @test filemetadata(tbl).notes == ["a note", "another note"] + @test year(filemetadata(tbl).modified_time) == 2020 + end + + @testset "LabeledArray columns carry their labels" begin + path = joinpath(tmp, "la_out.dta") + la = LabeledArray(DataValueArray{Int32}([1, 2, 1]), + ReadStat.ValueLabelDict(Int32(1) => "one", Int32(2) => "two")) + write_dta(path, Any[la], [:k]) + tbl = read_dta(path) + d = valuelabels(tbl, :k) + @test d !== nothing && d[Int32(1)] == "one" && d[Int32(2)] == "two" + @test tbl[:k] == DataValueArray{Int32}([1, 2, 1]) + end + + @testset "SPSS user-defined missing values (sav)" begin + path = joinpath(tmp, "um_out.sav") + write_sav(path, Any[DataValueArray{Float64}([1.0, -1.0, 99.0])], [:x]; + missing_ranges=[[-1.0, (90.0, 100.0)]]) + tbl = read_sav(path) + @test isna.(tbl[:x]) == [false, true, true] + kept = read_sav(path; user_missing=:keep) + @test kept[:x] == DataValueArray{Float64}([1.0, -1.0, 99.0]) + ranges = varmetadata(tbl, :x).missing_ranges + @test (90.0, 100.0) in ranges && (-1.0, -1.0) in ranges + end + + @testset "sas7bcat" begin + path = joinpath(tmp, "cat_out.sas7bcat") + write_sas7bcat(path, Dict(:NUMFMT => ReadStat.ValueLabelDict(1.0 => "yes", 2.0 => "no"))) + cat = read_sas7bcat(path) + @test cat[:NUMFMT][1.0] == "yes" && cat[:NUMFMT][2.0] == "no" + end + + @testset "strL through the low-level writer" begin + path = joinpath(tmp, "strl_out.dta") + w = Writer(path) + ref = add_string_ref!(w, "a very long shared string") + v = add_variable!(w, :s, CAPI.READSTAT_TYPE_STRING_REF) + begin_writing!(w, :dta, 2) + for _ in 1:2 + begin_row!(w) + insert_string_ref!(w, v, ref) + end_row!(w) + end + close(w) + tbl = read_dta(path) + @test [get(x) for x in tbl[:s]] == fill("a very long shared string", 2) + end + + @testset "table round trips" begin + src = read_dta(joinpath(@__DIR__, "data", "alltypes.dta")) + path = joinpath(tmp, "tbl_out.dta") + write_dta(path, src) + tbl = read_dta(path) + @test names(tbl) == names(src) + @test missingtags(tbl, :vbyte) == missingtags(src, :vbyte) + @test valuelabels(tbl, :vbyte) == valuelabels(src, :vbyte) + @test tbl[:vdate] == src[:vdate] + @test tbl[:vstrL] == src[:vstrL] + + sav = read_sav(joinpath(@__DIR__, "data", "sample.sav")) + path2 = joinpath(tmp, "tbl_out.sav") + write_sav(path2, sav) + tbl2 = read_sav(path2) + for i in 1:size(sav, 2) + @test isequal(collect(tbl2[i]), collect(sav[i])) + end + @test valuelabels(tbl2, :mylabl) == valuelabels(sav, :mylabl) + end + + @testset "compression" begin + path = joinpath(tmp, "z_out.zsav") + write_sav(path, Any[DataValueArray{Float64}([1.0, 2.0])], [:x]; compress=:binary) + @test size(read_sav(path), 1) == 2 + path2 = joinpath(tmp, "c_out.sas7bdat") + write_sas7bdat(path2, Any[DataValueArray{Float64}([1.0, 2.0])], [:x]; compress=:rows) + @test size(read_sas7bdat(path2), 1) == 2 + end + + @testset "validation errors" begin + one = Any[DataValueArray{Float64}([1.0])] + @test_throws ErrorException write_sav(joinpath(tmp, "badnote.sav"), one, [:x]; + notes=[repeat("x", 200)]) # SPSS notes are capped at 80 characters + @test_throws ArgumentError write_dta(joinpath(tmp, "mismatch.dta"), one, [:x, :y]) + @test_throws ArgumentError write_dta(joinpath(tmp, "raggedy.dta"), + Any[DataValueArray{Float64}([1.0]), DataValueArray{Float64}([1.0, 2.0])], [:x, :y]) + @test_throws ArgumentError write_dta(joinpath(tmp, "nofw.dta"), one, [:x]; fweight=:nope) + end +end + +@testitem "large write/read round trip with ntasks and chunks" begin + using DataValues + + n = 50_000 + isna_mask = [i % 997 == 0 for i in 1:n] + cols = Any[ + DataValueVector{Int32}(collect(Int32, 1:n), copy(isna_mask)), + DataValueVector{Float64}(collect(1:n) ./ 3, fill(false, n)), + DataValueVector{String}(string.(mod.(1:n, 100)), copy(isna_mask)), + ] + path = joinpath(mktempdir(), "big.dta") + write_dta(path, cols, [:id, :val, :bucket]) + + serial = read_dta(path; ntasks=1) + @test size(serial) == (n, 3) + @test isna.(serial[:id]) == isna_mask + @test get(serial[:val][n]) ≈ n / 3 + + threaded = read_dta(path; ntasks=4) + for i in 1:3 + @test isequal(collect(threaded[i]), collect(serial[i])) + end + + reassembled = reduce(vcat, + [collect(p[:id]) for p in chunks(ReadStatSource(path); chunksize=7_000)]) + @test isequal(reassembled, collect(serial[:id])) +end diff --git a/test/types.por b/test/types.por new file mode 100644 index 0000000..cd4b8b3 --- /dev/null +++ b/test/types.por @@ -0,0 +1,9 @@ +ASCII SPSS PORT FILE ASCII SPSS PORT FILE +ASCII SPSS PORT FILE ASCII SPSS PORT FILE +ASCII SPSS PORT FILE 0000000000000000000000000000000000000000 +0000000000000000000000000123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrst +uvwxyz .<(+0&[]!$*);^-/|,%_>?`:#@'="000000~000000000000000000000{}\0000000000000 +00000000000000000000000000000000000000000000000000000000SPSSPORTA8/201705106/ + 1D/STAT/TRANSFER46/5A/70/6/VFLOAT5/9/2/5/9/2/70/7/VDOUBLE5/9/2/5/9/2/70/5/VLO +NG5/9/2/5/9/2/70/4/VINT5/9/2/5/9/2/70/5/VBYTE5/9/2/5/9/2/73/7/VSTRING1/3/0/1/3/0 +/F3.46002GE7J/3.46/2/2/2/1/27/7/7/7/7/1/7*1*1*1*1*10/ZZZZZZZZZZZZZZZZZZZZZZZZZZZ \ No newline at end of file