diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4e00cd8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "julia" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" \ No newline at end of file diff --git a/.github/workflows/jlpkgbutler-butler-workflow.yml b/.github/workflows/jlpkgbutler-butler-workflow.yml deleted file mode 100644 index 70544cc..0000000 --- a/.github/workflows/jlpkgbutler-butler-workflow.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Run the Julia Package Butler - -on: - push: - branches: - - main - - master - schedule: - - cron: '0 */1 * * *' - workflow_dispatch: - -jobs: - butler: - name: "Run Package Butler" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: davidanthoff/julia-pkgbutler@releases/v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - ssh-private-key: ${{ secrets.JLPKGBUTLER_TOKEN }} - channel: stable diff --git a/.github/workflows/jlpkgbutler-ci-master-workflow.yml b/.github/workflows/jlpkgbutler-ci-master-workflow.yml deleted file mode 100644 index dd8c64d..0000000 --- a/.github/workflows/jlpkgbutler-ci-master-workflow.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Run CI on main - -on: - push: - branches: - - main - - master - workflow_dispatch: - -jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - matrix: - julia-version: ['1.0', '1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '1.9', '1.10'] - julia-arch: [x64, x86] - os: [ubuntu-latest, windows-latest, macOS-latest] - exclude: - - os: macOS-latest - julia-arch: x86 - - steps: - - uses: actions/checkout@v3 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.julia-version }} - arch: ${{ matrix.julia-arch }} - - uses: julia-actions/julia-buildpkg@v1 - env: - PYTHON: "" - - uses: julia-actions/julia-runtest@v1 - env: - PYTHON: "" - - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v3 - with: - files: ./lcov.info - flags: unittests - token: ${{ secrets.CODECOV_TOKEN }} - \ No newline at end of file diff --git a/.github/workflows/jlpkgbutler-ci-pr-workflow.yml b/.github/workflows/jlpkgbutler-ci-pr-workflow.yml deleted file mode 100644 index 08f833c..0000000 --- a/.github/workflows/jlpkgbutler-ci-pr-workflow.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Run CI on PR - -on: - pull_request: - types: [opened, synchronize, reopened] - -jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - matrix: - julia-version: ['1.0', '1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '1.9', '1.10'] - julia-arch: [x64, x86] - os: [ubuntu-latest, windows-latest, macOS-latest] - exclude: - - os: macOS-latest - julia-arch: x86 - - steps: - - uses: actions/checkout@v3 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.julia-version }} - arch: ${{ matrix.julia-arch }} - - uses: julia-actions/julia-buildpkg@v1 - env: - PYTHON: "" - - uses: julia-actions/julia-runtest@v1 - env: - PYTHON: "" - - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v3 - with: - files: ./lcov.info - flags: unittests - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/jlpkgbutler-codeformat-pr-workflow.yml b/.github/workflows/jlpkgbutler-codeformat-pr-workflow.yml deleted file mode 100644 index 411bed4..0000000 --- a/.github/workflows/jlpkgbutler-codeformat-pr-workflow.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Code Formatting - -on: - push: - branches: - - main - - master - workflow_dispatch: - -jobs: - format: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: julia-actions/julia-codeformat@releases/v1 - - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: Format files using DocumentFormat - title: '[AUTO] Format files using DocumentFormat' - body: '[DocumentFormat.jl](https://github.com/julia-vscode/DocumentFormat.jl) would suggest these formatting changes' - labels: no changelog diff --git a/.github/workflows/jlpkgbutler-compathelper-workflow.yml b/.github/workflows/jlpkgbutler-compathelper-workflow.yml deleted file mode 100644 index b315831..0000000 --- a/.github/workflows/jlpkgbutler-compathelper-workflow.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Run CompatHelper - -on: - schedule: - - cron: '00 * * * *' - issues: - types: [opened, reopened] - workflow_dispatch: - -jobs: - CompatHelper: - name: "Run CompatHelper.jl" - runs-on: ubuntu-latest - steps: - - name: Pkg.add("CompatHelper") - run: julia -e 'using Pkg; Pkg.add("CompatHelper")' - - name: CompatHelper.main() - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/.github/workflows/jlpkgbutler-docdeploy-workflow.yml b/.github/workflows/jlpkgbutler-docdeploy-workflow.yml deleted file mode 100644 index e517ba1..0000000 --- a/.github/workflows/jlpkgbutler-docdeploy-workflow.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Deploy documentation - -on: - push: - branches: - - main - - master - tags: - - v* - workflow_dispatch: - -jobs: - docdeploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: julia-actions/julia-buildpkg@v1 - env: - PYTHON: "" - - uses: julia-actions/julia-docdeploy@latest - env: - DOCUMENTER_KEY: ${{ secrets.JLPKGBUTLER_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/jlpkgbutler-tagbot-workflow.yml b/.github/workflows/jlpkgbutler-tagbot-workflow.yml deleted file mode 100644 index d3ca956..0000000 --- a/.github/workflows/jlpkgbutler-tagbot-workflow.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: TagBot -on: - issue_comment: - types: - - created - workflow_dispatch: - -jobs: - TagBot: - if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' - runs-on: ubuntu-latest - steps: - - uses: JuliaRegistries/TagBot@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - ssh: ${{ secrets.JLPKGBUTLER_TOKEN }} - branches: true diff --git a/.github/workflows/juliaci.yml b/.github/workflows/juliaci.yml new file mode 100644 index 0000000..6aa24b5 --- /dev/null +++ b/.github/workflows/juliaci.yml @@ -0,0 +1,17 @@ +name: Julia CI + +on: + push: {branches: [main,master]} + pull_request: {types: [opened,synchronize,reopened,ready_for_review,converted_to_draft]} + issue_comment: {types: [created]} + workflow_dispatch: {inputs: {feature: {type: choice, description: What to run, options: [DocDeploy,LintAndTest,TagBot]}}} + +jobs: + julia-ci: + uses: julia-testitems/testitem-workflow/.github/workflows/juliaci.yml@v2 + with: + include-all-compatible-minor-versions: true + include-rc-versions: true + permissions: write-all + secrets: + codecov_token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 49a815a..d6f25ca 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ *.jl.mem Manifest.toml .vscode +docs/build/ +docs/Manifest.toml +test-output*.* diff --git a/.jlpkgbutler.toml b/.jlpkgbutler.toml deleted file mode 100644 index b72304f..0000000 --- a/.jlpkgbutler.toml +++ /dev/null @@ -1 +0,0 @@ -template = "bach" diff --git a/LICENSE.md b/LICENSE.md index 1738dfc..db8adc6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ The ExcelFiles.jl package is licensed under the MIT "Expat" License: -> Copyright (c) 2017-2019: David Anthoff. +> Copyright (c) 2017-2026: David Anthoff. > > > Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/Project.toml b/Project.toml index 68ecaaa..d158594 100644 --- a/Project.toml +++ b/Project.toml @@ -10,28 +10,29 @@ FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" IterableTables = "1c8ee90f-4401-5389-894e-7a04a3dc0f4d" IteratorInterfaceExtensions = "82899510-4779-5014-852e-03e436cf321d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" -PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" TableShowUtils = "5e66a065-1f0a-5976-b372-e0b8c017ca10" TableTraits = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" TableTraitsUtils = "382cd787-c1b6-5bf2-a167-d5b971a19bda" XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0" [compat] -DataValues = "0.4.11" -ExcelReaders = "0.11" +DataValues = "0.4.11, 0.5, 1" +Dates = "1" +ExcelReaders = "0.13, 1" FileIO = "1" IterableTables = "0.8.3, 0.9, 0.10, 0.11, 1" IteratorInterfaceExtensions = "0.1.1, 1" -PyCall = "1.90" -TableShowUtils = "0.2" +Printf = "1" +TableShowUtils = "0.2, 0.3, 1" TableTraits = "0.3.1, 0.4, 1" TableTraitsUtils = "0.3, 0.4, 1" -XLSX = "0.4.1, 0.5, 0.6, 0.7, 0.8, 0.9" +XLSX = "0.12" julia = "1" [extras] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" [targets] -test = ["Test", "DataFrames"] +test = ["Test", "DataFrames", "TestItemRunner"] diff --git a/README.md b/README.md index f175400..749d70e 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,21 @@ # ExcelFiles [![Project Status: Active - The project has reached a stable, usable state and is being actively developed.](http://www.repostatus.org/badges/latest/active.svg)](http://www.repostatus.org/#active) -[![Build Status](https://travis-ci.org/queryverse/ExcelFiles.jl.svg?branch=master)](https://travis-ci.org/queryverse/ExcelFiles.jl) -[![Build status](https://ci.appveyor.com/api/projects/status/wfx5avj0s2m0x94w/branch/master?svg=true)](https://ci.appveyor.com/project/queryverse/excelfiles-jl/branch/master) +[![Build Status](https://github.com/queryverse/ExcelFiles.jl/actions/workflows/juliaci.yml/badge.svg?branch=main)](https://github.com/queryverse/ExcelFiles.jl/actions/workflows/juliaci.yml) [![codecov.io](http://codecov.io/github/queryverse/ExcelFiles.jl/coverage.svg?branch=master)](http://codecov.io/github/queryverse/ExcelFiles.jl?branch=master) ## Overview This package provides load support for Excel files under the -[FileIO.jl](https://github.com/JuliaIO/FileIO.jl) package. +[FileIO.jl](https://github.com/JuliaIO/FileIO.jl) package. Both modern xlsx +files and legacy xls files can be loaded (the format is detected from the +content of the file); saving is supported for xlsx files. + +Note that the central FileIO registry routes Excel files to +[XLSX.jl](https://github.com/JuliaData/XLSX.jl) and does not cover xls files. +Once ExcelFiles is loaded (``using ExcelFiles``), its loader takes priority, +which restores range-based loading (``"Sheet1!A1:C4"``), the lazy iterable +table interface, and support for xls files. ## Installation diff --git a/docs/Project.toml b/docs/Project.toml index f2a273e..1814eb3 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -2,4 +2,4 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" [compat] -Documenter = "~0.24" +Documenter = "1" diff --git a/docs/make.jl b/docs/make.jl index 3402bf5..dab34a6 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -2,7 +2,8 @@ using Documenter, ExcelFiles makedocs(modules=[ExcelFiles], sitename="ExcelFiles.jl", - analytics="UA-132838790-1", + format = Documenter.HTML(analytics = "UA-132838790-1"), + warnonly = [:missing_docs], pages=[ "Introduction" => "index.md" ]) diff --git a/src/ExcelFiles.jl b/src/ExcelFiles.jl index 9b7eb6a..fa19dd6 100644 --- a/src/ExcelFiles.jl +++ b/src/ExcelFiles.jl @@ -7,6 +7,30 @@ import IterableTables export load, save, File, @format_str +function __init__() + # Since FileIO PR #439 (July 2026) the central FileIO registry routes + # format"Excel" to XLSX.jl's own FileIO extension, and no longer covers + # legacy xls files at all. Loading ExcelFiles puts its loader first + # again, and registers a format for xls files. Users who don't load + # ExcelFiles are unaffected. + try + loaders = get!(Vector{FileIO.ActionSource}, FileIO.sym2loader, :Excel) + filter!(x -> x !== ExcelFiles, loaders) + pushfirst!(loaders, ExcelFiles) + savers = get!(Vector{FileIO.ActionSource}, FileIO.sym2saver, :Excel) + filter!(x -> x !== ExcelFiles, savers) + pushfirst!(savers, ExcelFiles) + + if !haskey(FileIO.sym2info, :ExcelLegacy) + FileIO.add_format(FileIO.format"ExcelLegacy", (), [".xls"], + [:ExcelFiles => Base.UUID("89b67f3b-d1aa-5f6f-9ca4-282e8d98620d"), FileIO.LOAD]) + end + catch err + @warn "ExcelFiles could not register itself with FileIO. Loading Excel files via FileIO.load may not use ExcelFiles." exception = (err, catch_backtrace()) + end + return nothing +end + struct ExcelFile filename::String range::String @@ -29,7 +53,9 @@ end Base.Multimedia.showable(::MIME"application/vnd.dataresource+json", source::ExcelFile) = true -function fileio_load(f::FileIO.File{FileIO.format"Excel"}, range; keywords...) +const ExcelFileFormat = Union{FileIO.File{FileIO.format"Excel"},FileIO.File{FileIO.format"ExcelLegacy"}} + +function fileio_load(f::ExcelFileFormat, range; keywords...) return ExcelFile(f.filename, range, keywords) end @@ -38,6 +64,10 @@ function fileio_save(f::FileIO.File{FileIO.format"Excel"}, data; sheetname::Abst return XLSX.writetable(f.filename, cols, colnames; sheetname=sheetname) end +function fileio_save(f::FileIO.File{FileIO.format"ExcelLegacy"}, data; kwargs...) + error("Writing legacy xls files is not supported. Save to an xlsx file instead.") +end + IteratorInterfaceExtensions.isiterable(x::ExcelFile) = true TableTraits.isiterabletable(x::ExcelFile) = true @@ -112,7 +142,7 @@ function IteratorInterfaceExtensions.getiterator(file::ExcelFile) _readxl(excelfile, sheetname, startrow, startcol, endrow, endcol; file.keywords...) else excelfile = openxl(file.filename) - sheet = excelfile.workbook.sheet_by_name(file.range) + sheet = ExcelReaders.sheet_handle(excelfile, file.range) keywords = filter(i -> !(i[1] in (:header, :colnames)), file.keywords) startrow, startcol, endrow, endcol = ExcelReaders.convert_args_to_row_col(sheet; keywords...) diff --git a/test/runtests.jl b/test/runtests.jl index d1d0372..b9e874d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,149 +1,3 @@ -using ExcelFiles -using ExcelReaders -using IteratorInterfaceExtensions -using TableTraits -using TableTraitsUtils -using Dates -using DataValues -using DataFrames -using Test +using TestItemRunner -@testset "ExcelFiles" begin - - filename = normpath(dirname(pathof(ExcelReaders)), "..", "test", "TestData.xlsx") - - efile = load(filename, "Sheet1") - - @test sprint((stream, data) -> show(stream, "text/html", data), efile) == "
Some Float64sSome StringsSome BoolsMixed columnMixed with NAFloat64 with NAString with NABool with NASome datesDates with NASome errorsErrors with NAColumn with NULL and then mixed
1.0"A"true2.09.03.0"FF"#NA2015-03-03T00:00:001965-04-03T00:00:00#DIV/0!#DIV/0!#NA
1.5"BB"false"EEEEE""III"#NA#NAtrue2015-02-04T10:14:001950-08-09T18:40:00#N/A#N/A3.4
2.0"CCC"falsefalse#NA3.5"GGG"#NA1988-04-09T00:00:0019:00:00#REF!#NAME?"HKEJW"
2.5"DDDD"true1.5true4.0"HHHH"false15:02:00#NA#NAME?#NA#NA
" - - @test sprint((stream, data) -> show(stream, "application/vnd.dataresource+json", data), efile) == "{\"schema\":{\"fields\":[{\"name\":\"Some Float64s\",\"type\":\"number\"},{\"name\":\"Some Strings\",\"type\":\"string\"},{\"name\":\"Some Bools\",\"type\":\"boolean\"},{\"name\":\"Mixed column\",\"type\":\"string\"},{\"name\":\"Mixed with NA\",\"type\":\"string\"},{\"name\":\"Float64 with NA\",\"type\":\"number\"},{\"name\":\"String with NA\",\"type\":\"string\"},{\"name\":\"Bool with NA\",\"type\":\"boolean\"},{\"name\":\"Some dates\",\"type\":\"string\"},{\"name\":\"Dates with NA\",\"type\":\"string\"},{\"name\":\"Some errors\",\"type\":\"string\"},{\"name\":\"Errors with NA\",\"type\":\"string\"},{\"name\":\"Column with NULL and then mixed\",\"type\":\"string\"}]},\"data\":[{\"Some Float64s\":1.0,\"Some Strings\":\"A\",\"Some Bools\":true,\"Mixed column\":2.0,\"Mixed with NA\":9.0,\"Float64 with NA\":3.0,\"String with NA\":\"FF\",\"Bool with NA\":null,\"Some dates\":\"2015-03-03T00:00:00\",\"Dates with NA\":\"1965-04-03T00:00:00\",\"Some errors\":{\"errorcode\":7},\"Errors with NA\":{\"errorcode\":7},\"Column with NULL and then mixed\":null},{\"Some Float64s\":1.5,\"Some Strings\":\"BB\",\"Some Bools\":false,\"Mixed column\":\"EEEEE\",\"Mixed with NA\":\"III\",\"Float64 with NA\":null,\"String with NA\":null,\"Bool with NA\":true,\"Some dates\":\"2015-02-04T10:14:00\",\"Dates with NA\":\"1950-08-09T18:40:00\",\"Some errors\":{\"errorcode\":42},\"Errors with NA\":{\"errorcode\":42},\"Column with NULL and then mixed\":3.4},{\"Some Float64s\":2.0,\"Some Strings\":\"CCC\",\"Some Bools\":false,\"Mixed column\":false,\"Mixed with NA\":null,\"Float64 with NA\":3.5,\"String with NA\":\"GGG\",\"Bool with NA\":null,\"Some dates\":\"1988-04-09T00:00:00\",\"Dates with NA\":\"19:00:00\",\"Some errors\":{\"errorcode\":23},\"Errors with NA\":{\"errorcode\":29},\"Column with NULL and then mixed\":\"HKEJW\"},{\"Some Float64s\":2.5,\"Some Strings\":\"DDDD\",\"Some Bools\":true,\"Mixed column\":1.5,\"Mixed with NA\":true,\"Float64 with NA\":4.0,\"String with NA\":\"HHHH\",\"Bool with NA\":false,\"Some dates\":\"15:02:00\",\"Dates with NA\":null,\"Some errors\":{\"errorcode\":29},\"Errors with NA\":null,\"Column with NULL and then mixed\":null}]}" - - @test sprint(show, efile) == "4x13 Excel file\nSome Float64s │ Some Strings │ Some Bools │ Mixed column │ Mixed with NA\n──────────────┼──────────────┼────────────┼──────────────┼──────────────\n1.0 │ A │ true │ 2.0 │ 9.0 \n1.5 │ BB │ false │ \"EEEEE\" │ \"III\" \n2.0 │ CCC │ false │ false │ #NA \n2.5 │ DDDD │ true │ 1.5 │ true \n... with 8 more columns: Float64 with NA, String with NA, Bool with NA, Some dates, Dates with NA, Some errors, Errors with NA, Column with NULL and then mixed" - - @test TableTraits.isiterabletable(efile) == true - @test IteratorInterfaceExtensions.isiterable(efile) == true - @test showable("text/html", efile) == true - @test showable("application/vnd.dataresource+json", efile) == true - - @test isiterable(efile) == true - - full_dfs = [create_columns_from_iterabletable(load(filename, "Sheet1!C3:O7")), create_columns_from_iterabletable(load(filename, "Sheet1"))] - for (df, names) in full_dfs - @test length(df) == 13 - @test length(df[1]) == 4 - - @test df[1] == [1., 1.5, 2., 2.5] - @test df[2] == ["A", "BB", "CCC", "DDDD"] - @test df[3] == [true, false, false, true] - @test df[4] == [2, "EEEEE", false, 1.5] - @test df[5] == [9., "III", NA, true] - @test df[6] == [3., NA, 3.5, 4] - @test df[7] == ["FF", NA, "GGG", "HHHH"] - @test df[8] == [NA, true, NA, false] - @test df[9] == [Date(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), Date(1988, 4, 9), Dates.Time(15, 2, 0)] - @test df[10] == [Date(1965, 4, 3), DateTime(1950, 8, 9, 18, 40), Dates.Time(19, 0, 0), NA] - @test eltype(df[11]) == ExcelReaders.ExcelErrorCell - @test df[12][1][] isa ExcelReaders.ExcelErrorCell - @test df[12][2][] isa ExcelReaders.ExcelErrorCell - @test df[12][3][] isa ExcelReaders.ExcelErrorCell - @test df[12][4] == NA - @test df[13] == [NA, 3.4, "HKEJW", NA] - end - - df, names = create_columns_from_iterabletable(load(filename, "Sheet1!C4:O7", header=false)) - @test names == [:x1,:x2,:x3,:x4,:x5,:x6,:x7,:x8,:x9,:x10,:x11,:x12,:x13] - @test length(df[1]) == 4 - @test length(df) == 13 - @test df[1] == [1., 1.5, 2., 2.5] - @test df[2] == ["A", "BB", "CCC", "DDDD"] - @test df[3] == [true, false, false, true] - @test df[4] == [2, "EEEEE", false, 1.5] - @test df[5] == [9., "III", NA, true] - @test df[6] == [3, NA, 3.5, 4] - @test df[7] == ["FF", NA, "GGG", "HHHH"] - @test df[8] == [NA, true, NA, false] - @test df[9] == [Date(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), DateTime(1988, 4, 9), Dates.Time(15, 2, 0)] - @test df[10] == [Date(1965, 4, 3), DateTime(1950, 8, 9, 18, 40), Dates.Time(19, 0, 0), NA] - @test isa(df[11][1], ExcelReaders.ExcelErrorCell) - @test isa(df[11][2], ExcelReaders.ExcelErrorCell) - @test isa(df[11][3], ExcelReaders.ExcelErrorCell) - @test isa(df[11][4], ExcelReaders.ExcelErrorCell) - @test isa(df[12][1][], ExcelReaders.ExcelErrorCell) - @test isa(df[12][2][], ExcelReaders.ExcelErrorCell) - @test isa(df[12][3][], ExcelReaders.ExcelErrorCell) - @test DataValues.isna(df[12][4]) - @test df[13] == [NA, 3.4, "HKEJW", NA] - - good_colnames = [:c1, :c2, :c3, :c4, :c5, :c6, :c7, :c8, :c9, :c10, :c11, :c12, :c13] - - df, names = create_columns_from_iterabletable(load(filename, "Sheet1!C4:O7", header=false, colnames=good_colnames)) - @test names == good_colnames - @test length(df[1]) == 4 - @test length(df) == 13 - @test df[1] == [1., 1.5, 2., 2.5] - @test df[2] == ["A", "BB", "CCC", "DDDD"] - @test df[3] == [true, false, false, true] - @test df[4] == [2, "EEEEE", false, 1.5] - @test df[5] == [9., "III", NA, true] - @test df[6] == [3, NA, 3.5, 4] - @test df[7] == ["FF", NA, "GGG", "HHHH"] - @test df[8] == [NA, true, NA, false] - @test df[9] == [Date(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), DateTime(1988, 4, 9), Dates.Time(15, 2, 0)] - @test df[10] == [Date(1965, 4, 3), DateTime(1950, 8, 9, 18, 40), Dates.Time(19, 0, 0), NA] - @test isa(df[11][1], ExcelReaders.ExcelErrorCell) - @test isa(df[11][2], ExcelReaders.ExcelErrorCell) - @test isa(df[11][3], ExcelReaders.ExcelErrorCell) - @test isa(df[11][4], ExcelReaders.ExcelErrorCell) - @test isa(df[12][1][], ExcelReaders.ExcelErrorCell) - @test isa(df[12][2][], ExcelReaders.ExcelErrorCell) - @test isa(df[12][3][], ExcelReaders.ExcelErrorCell) - @test DataValues.isna(df[12][4]) - @test df[13] == [NA, 3.4, "HKEJW", NA] - -# Test for saving DataFrame to XLSX - input = (Day = ["Nov. 27","Nov. 28","Nov. 29"], Highest = [78,79,75]) |> DataFrame - file = save("file.xlsx", input) - output = load("file.xlsx", "Sheet1") |> DataFrame - @test input == output - rm("file.xlsx") - -# Test for saving DataFrame to XLSX with sheetname keyword - input = (Day = ["Nov. 27","Nov. 28","Nov. 29"], Highest = [78,79,75]) |> DataFrame - file = save("file.xlsx", input, sheetname="SheetName") - output = load("file.xlsx", "SheetName") |> DataFrame - @test input == output - rm("file.xlsx") - - df, names = create_columns_from_iterabletable(load(filename, "Sheet1", colnames=good_colnames)) - @test names == good_colnames - @test length(df[1]) == 4 - @test length(df) == 13 - @test df[1] == [1., 1.5, 2., 2.5] - @test df[2] == ["A", "BB", "CCC", "DDDD"] - @test df[3] == [true, false, false, true] - @test df[4] == [2, "EEEEE", false, 1.5] - @test df[5] == [9., "III", NA, true] - @test df[6] == [3, NA, 3.5, 4] - @test df[7] == ["FF", NA, "GGG", "HHHH"] - @test df[8] == [NA, true, NA, false] - @test df[9] == [Date(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), DateTime(1988, 4, 9), Dates.Time(15, 2, 0)] - @test df[10] == [Date(1965, 4, 3), DateTime(1950, 8, 9, 18, 40), Dates.Time(19, 0, 0), NA] - @test isa(df[11][1], ExcelReaders.ExcelErrorCell) - @test isa(df[11][2], ExcelReaders.ExcelErrorCell) - @test isa(df[11][3], ExcelReaders.ExcelErrorCell) - @test isa(df[11][4], ExcelReaders.ExcelErrorCell) - @test isa(df[12][1][], ExcelReaders.ExcelErrorCell) - @test isa(df[12][2][], ExcelReaders.ExcelErrorCell) - @test isa(df[12][3][], ExcelReaders.ExcelErrorCell) - @test DataValues.isna(df[12][4]) - @test df[13] == [NA, 3.4, "HKEJW", NA] - -# Too few colnames - @test_throws ErrorException create_columns_from_iterabletable(load(filename, "Sheet1!C4:O7", header=true, colnames=[:c1, :c2, :c3, :c4])) - -# Test for constructing DataFrame with empty header cell - data, names = create_columns_from_iterabletable(load(filename, "Sheet2!C5:E7")) - @test names == [:Col1, :x1, :Col3] - - -end +@run_package_tests diff --git a/test/test_excelfiles.jl b/test/test_excelfiles.jl new file mode 100644 index 0000000..5a45b15 --- /dev/null +++ b/test/test_excelfiles.jl @@ -0,0 +1,168 @@ +@testitem "ExcelFiles" begin + using ExcelReaders + using IteratorInterfaceExtensions + using TableTraits + using TableTraitsUtils + using Dates + using DataValues + using DataFrames + + filename = normpath(dirname(pathof(ExcelReaders)), "..", "test", "TestData.xlsx") + + efile = load(filename, "Sheet1") + + @test sprint((stream, data) -> show(stream, "text/html", data), efile) == "
Some Float64sSome StringsSome BoolsMixed columnMixed with NAFloat64 with NAString with NABool with NASome datesDates with NASome errorsErrors with NAColumn with NULL and then mixed
1.0"A"true2.09.03.0"FF"#NA2015-03-03T00:00:001965-04-03T00:00:00#DIV/0!#DIV/0!#NA
1.5"BB"false"EEEEE""III"#NA#NAtrue2015-02-04T10:14:001950-08-09T18:40:00#N/A#N/A3.4
2.0"CCC"falsefalse#NA3.5"GGG"#NA1988-04-09T00:00:0019:00:00#REF!#NAME?"HKEJW"
2.5"DDDD"true1.5true4.0"HHHH"false15:02:00#NA#NAME?#NA#NA
" + + @test sprint((stream, data) -> show(stream, "application/vnd.dataresource+json", data), efile) == "{\"schema\":{\"fields\":[{\"name\":\"Some Float64s\",\"type\":\"number\"},{\"name\":\"Some Strings\",\"type\":\"string\"},{\"name\":\"Some Bools\",\"type\":\"boolean\"},{\"name\":\"Mixed column\",\"type\":\"string\"},{\"name\":\"Mixed with NA\",\"type\":\"string\"},{\"name\":\"Float64 with NA\",\"type\":\"number\"},{\"name\":\"String with NA\",\"type\":\"string\"},{\"name\":\"Bool with NA\",\"type\":\"boolean\"},{\"name\":\"Some dates\",\"type\":\"string\"},{\"name\":\"Dates with NA\",\"type\":\"string\"},{\"name\":\"Some errors\",\"type\":\"string\"},{\"name\":\"Errors with NA\",\"type\":\"string\"},{\"name\":\"Column with NULL and then mixed\",\"type\":\"string\"}]},\"data\":[{\"Some Float64s\":1.0,\"Some Strings\":\"A\",\"Some Bools\":true,\"Mixed column\":2.0,\"Mixed with NA\":9.0,\"Float64 with NA\":3.0,\"String with NA\":\"FF\",\"Bool with NA\":null,\"Some dates\":\"2015-03-03T00:00:00\",\"Dates with NA\":\"1965-04-03T00:00:00\",\"Some errors\":{\"errorcode\":7},\"Errors with NA\":{\"errorcode\":7},\"Column with NULL and then mixed\":null},{\"Some Float64s\":1.5,\"Some Strings\":\"BB\",\"Some Bools\":false,\"Mixed column\":\"EEEEE\",\"Mixed with NA\":\"III\",\"Float64 with NA\":null,\"String with NA\":null,\"Bool with NA\":true,\"Some dates\":\"2015-02-04T10:14:00\",\"Dates with NA\":\"1950-08-09T18:40:00\",\"Some errors\":{\"errorcode\":42},\"Errors with NA\":{\"errorcode\":42},\"Column with NULL and then mixed\":3.4},{\"Some Float64s\":2.0,\"Some Strings\":\"CCC\",\"Some Bools\":false,\"Mixed column\":false,\"Mixed with NA\":null,\"Float64 with NA\":3.5,\"String with NA\":\"GGG\",\"Bool with NA\":null,\"Some dates\":\"1988-04-09T00:00:00\",\"Dates with NA\":\"19:00:00\",\"Some errors\":{\"errorcode\":23},\"Errors with NA\":{\"errorcode\":29},\"Column with NULL and then mixed\":\"HKEJW\"},{\"Some Float64s\":2.5,\"Some Strings\":\"DDDD\",\"Some Bools\":true,\"Mixed column\":1.5,\"Mixed with NA\":true,\"Float64 with NA\":4.0,\"String with NA\":\"HHHH\",\"Bool with NA\":false,\"Some dates\":\"15:02:00\",\"Dates with NA\":null,\"Some errors\":{\"errorcode\":29},\"Errors with NA\":null,\"Column with NULL and then mixed\":null}]}" + + @test sprint(show, efile) == "4x13 Excel file\nSome Float64s │ Some Strings │ Some Bools │ Mixed column │ Mixed with NA\n──────────────┼──────────────┼────────────┼──────────────┼──────────────\n1.0 │ A │ true │ 2.0 │ 9.0 \n1.5 │ BB │ false │ \"EEEEE\" │ \"III\" \n2.0 │ CCC │ false │ false │ #NA \n2.5 │ DDDD │ true │ 1.5 │ true \n... with 8 more columns: Float64 with NA, String with NA, Bool with NA, Some dates, Dates with NA, Some errors, Errors with NA, Column with NULL and then mixed" + + @test TableTraits.isiterabletable(efile) == true + @test IteratorInterfaceExtensions.isiterable(efile) == true + @test showable("text/html", efile) == true + @test showable("application/vnd.dataresource+json", efile) == true + + @test isiterable(efile) == true + + full_dfs = [create_columns_from_iterabletable(load(filename, "Sheet1!C3:O7")), create_columns_from_iterabletable(load(filename, "Sheet1"))] + for (df, names) in full_dfs + @test length(df) == 13 + @test length(df[1]) == 4 + + @test df[1] == [1., 1.5, 2., 2.5] + @test df[2] == ["A", "BB", "CCC", "DDDD"] + @test df[3] == [true, false, false, true] + @test df[4] == [2, "EEEEE", false, 1.5] + @test df[5] == [9., "III", NA, true] + @test df[6] == [3., NA, 3.5, 4] + @test df[7] == ["FF", NA, "GGG", "HHHH"] + @test df[8] == [NA, true, NA, false] + @test df[9] == [Date(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), Date(1988, 4, 9), Dates.Time(15, 2, 0)] + @test df[10] == [Date(1965, 4, 3), DateTime(1950, 8, 9, 18, 40), Dates.Time(19, 0, 0), NA] + @test eltype(df[11]) == ExcelReaders.ExcelErrorCell + @test df[12][1][] isa ExcelReaders.ExcelErrorCell + @test df[12][2][] isa ExcelReaders.ExcelErrorCell + @test df[12][3][] isa ExcelReaders.ExcelErrorCell + @test df[12][4] == NA + @test df[13] == [NA, 3.4, "HKEJW", NA] + end + + df, names = create_columns_from_iterabletable(load(filename, "Sheet1!C4:O7", header=false)) + @test names == [:x1,:x2,:x3,:x4,:x5,:x6,:x7,:x8,:x9,:x10,:x11,:x12,:x13] + @test length(df[1]) == 4 + @test length(df) == 13 + @test df[1] == [1., 1.5, 2., 2.5] + @test df[2] == ["A", "BB", "CCC", "DDDD"] + @test df[3] == [true, false, false, true] + @test df[4] == [2, "EEEEE", false, 1.5] + @test df[5] == [9., "III", NA, true] + @test df[6] == [3, NA, 3.5, 4] + @test df[7] == ["FF", NA, "GGG", "HHHH"] + @test df[8] == [NA, true, NA, false] + @test df[9] == [Date(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), DateTime(1988, 4, 9), Dates.Time(15, 2, 0)] + @test df[10] == [Date(1965, 4, 3), DateTime(1950, 8, 9, 18, 40), Dates.Time(19, 0, 0), NA] + @test isa(df[11][1], ExcelReaders.ExcelErrorCell) + @test isa(df[11][2], ExcelReaders.ExcelErrorCell) + @test isa(df[11][3], ExcelReaders.ExcelErrorCell) + @test isa(df[11][4], ExcelReaders.ExcelErrorCell) + @test isa(df[12][1][], ExcelReaders.ExcelErrorCell) + @test isa(df[12][2][], ExcelReaders.ExcelErrorCell) + @test isa(df[12][3][], ExcelReaders.ExcelErrorCell) + @test DataValues.isna(df[12][4]) + @test df[13] == [NA, 3.4, "HKEJW", NA] + + good_colnames = [:c1, :c2, :c3, :c4, :c5, :c6, :c7, :c8, :c9, :c10, :c11, :c12, :c13] + + df, names = create_columns_from_iterabletable(load(filename, "Sheet1!C4:O7", header=false, colnames=good_colnames)) + @test names == good_colnames + @test length(df[1]) == 4 + @test length(df) == 13 + @test df[1] == [1., 1.5, 2., 2.5] + @test df[2] == ["A", "BB", "CCC", "DDDD"] + @test df[3] == [true, false, false, true] + @test df[4] == [2, "EEEEE", false, 1.5] + @test df[5] == [9., "III", NA, true] + @test df[6] == [3, NA, 3.5, 4] + @test df[7] == ["FF", NA, "GGG", "HHHH"] + @test df[8] == [NA, true, NA, false] + @test df[9] == [Date(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), DateTime(1988, 4, 9), Dates.Time(15, 2, 0)] + @test df[10] == [Date(1965, 4, 3), DateTime(1950, 8, 9, 18, 40), Dates.Time(19, 0, 0), NA] + @test isa(df[11][1], ExcelReaders.ExcelErrorCell) + @test isa(df[11][2], ExcelReaders.ExcelErrorCell) + @test isa(df[11][3], ExcelReaders.ExcelErrorCell) + @test isa(df[11][4], ExcelReaders.ExcelErrorCell) + @test isa(df[12][1][], ExcelReaders.ExcelErrorCell) + @test isa(df[12][2][], ExcelReaders.ExcelErrorCell) + @test isa(df[12][3][], ExcelReaders.ExcelErrorCell) + @test DataValues.isna(df[12][4]) + @test df[13] == [NA, 3.4, "HKEJW", NA] + +# Test for saving DataFrame to XLSX + input = (Day = ["Nov. 27","Nov. 28","Nov. 29"], Highest = [78,79,75]) |> DataFrame + file = save("file.xlsx", input) + output = load("file.xlsx", "Sheet1") |> DataFrame + @test input == output + rm("file.xlsx") + +# Test for saving DataFrame to XLSX with sheetname keyword + input = (Day = ["Nov. 27","Nov. 28","Nov. 29"], Highest = [78,79,75]) |> DataFrame + file = save("file.xlsx", input, sheetname="SheetName") + output = load("file.xlsx", "SheetName") |> DataFrame + @test input == output + rm("file.xlsx") + + df, names = create_columns_from_iterabletable(load(filename, "Sheet1", colnames=good_colnames)) + @test names == good_colnames + @test length(df[1]) == 4 + @test length(df) == 13 + @test df[1] == [1., 1.5, 2., 2.5] + @test df[2] == ["A", "BB", "CCC", "DDDD"] + @test df[3] == [true, false, false, true] + @test df[4] == [2, "EEEEE", false, 1.5] + @test df[5] == [9., "III", NA, true] + @test df[6] == [3, NA, 3.5, 4] + @test df[7] == ["FF", NA, "GGG", "HHHH"] + @test df[8] == [NA, true, NA, false] + @test df[9] == [Date(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), DateTime(1988, 4, 9), Dates.Time(15, 2, 0)] + @test df[10] == [Date(1965, 4, 3), DateTime(1950, 8, 9, 18, 40), Dates.Time(19, 0, 0), NA] + @test isa(df[11][1], ExcelReaders.ExcelErrorCell) + @test isa(df[11][2], ExcelReaders.ExcelErrorCell) + @test isa(df[11][3], ExcelReaders.ExcelErrorCell) + @test isa(df[11][4], ExcelReaders.ExcelErrorCell) + @test isa(df[12][1][], ExcelReaders.ExcelErrorCell) + @test isa(df[12][2][], ExcelReaders.ExcelErrorCell) + @test isa(df[12][3][], ExcelReaders.ExcelErrorCell) + @test DataValues.isna(df[12][4]) + @test df[13] == [NA, 3.4, "HKEJW", NA] + +# Too few colnames + @test_throws ErrorException create_columns_from_iterabletable(load(filename, "Sheet1!C4:O7", header=true, colnames=[:c1, :c2, :c3, :c4])) + +# Test for constructing DataFrame with empty header cell + data, names = create_columns_from_iterabletable(load(filename, "Sheet2!C5:E7")) + @test names == [:Col1, :x1, :Col3] +end + +@testitem "ExcelFiles xls loading" begin + using ExcelReaders + using TableTraitsUtils + using Dates + using DataValues + using DataFrames + + # Loading the legacy xls variant of the test data must produce the same + # table as the xlsx variant (which has one additional column). + filename_xls = normpath(dirname(pathof(ExcelReaders)), "..", "test", "TestData.xls") + + df = DataFrame(load(filename_xls, "Sheet1!C3:N7")) + @test size(df) == (4, 12) + @test df[!, 1] == [1., 1.5, 2., 2.5] + @test df[!, 2] == ["A", "BB", "CCC", "DDDD"] + @test df[!, 3] == [true, false, false, true] + @test df[!, 9] == [DateTime(2015, 3, 3), DateTime(2015, 2, 4, 10, 14), DateTime(1988, 4, 9), Dates.Time(15, 2, 0)] + @test eltype(df[!, 11]) == ExcelReaders.ExcelErrorCell + + df2 = DataFrame(load(filename_xls, "Sheet1")) + @test size(df2) == (4, 13) + @test df2[!, 1] == [1., 1.5, 2., 2.5] +end