diff --git a/CHANGELOG.md b/CHANGELOG.md index 39289d4..b438ea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,35 @@ All notable changes to DatabentoAPI.jl are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.3] - 2026-09-17 + +### Added +- **`get_job_details(client; job_id)`** wraps the new + `GET /v0/batch.get_job_details` endpoint and returns the full job object + (request parameters, `record_count`, `billed_size`, `actual_size`, + `package_size`, `cost_usd`, `state`, `progress`, and the `ts_*` lifecycle + timestamps). +- **`list_jobs(...; short=true)`** requests the new condensed response (`id`, + `state`, `ts_received` only). Per Databento's August 2026 batch API notice, + `list_jobs` is moving to that shape in phases: `short` is opt-in today, + becomes the server default next, and is finally removed along with the legacy + full response. Code that reads any other field from `list_jobs` should switch + to `get_job_details`. The default `short=nothing` omits the parameter so the + server default applies through the transition. + +### Notes +- Databento's July 2026 CME (`GLBX.MDP3`) normalization changes — one + definition record per strategy leg, a standalone `F_LAST` MBO record with + `action = NONE` / `price = UNDEF_PRICE` / `size = 0`, price-limit statistics + (`stat_type` 17/18), implied-matching status events, and + `instrument_class = X` for FX spot — need no code changes in this package: + record decoding is delegated to DatabentoBinaryEncoding, which handles these + shapes and gains `StatType`/`TradingEvent` enums, `InstrumentClass.INDEX`, + and `F_*` flag constants in its 0.1.7 release. Until this package + re-exports them they are reachable as `DatabentoAPI.DBN.StatType` and + `DatabentoAPI.DBN.TradingEvent`. Downstream code that builds books from + MBO, or keys definitions by `instrument_id`, should review the notice. + ## [0.3.2] - 2026-09-10 ### Changed @@ -311,6 +340,9 @@ Initial public release. Registered in the General registry. `SymbologyResolution`, `RollRule`, `SlowReaderBehavior`. [Unreleased]: https://github.com/tbeason/DatabentoAPI.jl/compare/v0.3.0...HEAD +[0.3.3]: https://github.com/tbeason/DatabentoAPI.jl/compare/v0.3.2...v0.3.3 +[0.3.2]: https://github.com/tbeason/DatabentoAPI.jl/compare/v0.3.1...v0.3.2 +[0.3.1]: https://github.com/tbeason/DatabentoAPI.jl/compare/v0.3.0...v0.3.1 [0.3.0]: https://github.com/tbeason/DatabentoAPI.jl/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/tbeason/DatabentoAPI.jl/compare/v0.1.2...v0.2.0 [0.1.2]: https://github.com/tbeason/DatabentoAPI.jl/compare/v0.1.1...v0.1.2 diff --git a/Project.toml b/Project.toml index da3a66d..0d7a961 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "DatabentoAPI" uuid = "feb0aed8-3291-460b-9688-521dcb17a4bf" -version = "0.3.2" +version = "0.3.3" authors = ["Tyler Beason "] [deps] diff --git a/docs/src/api/historical.md b/docs/src/api/historical.md index ea567ae..43a2163 100644 --- a/docs/src/api/historical.md +++ b/docs/src/api/historical.md @@ -37,6 +37,7 @@ get_cost ```@docs submit_job list_jobs +get_job_details list_files batch_download ``` diff --git a/docs/src/guide/historical.md b/docs/src/guide/historical.md index 33686ab..12467e8 100644 --- a/docs/src/guide/historical.md +++ b/docs/src/guide/historical.md @@ -160,11 +160,9 @@ job = submit_job(client; job_id = job["id"] -# Poll until done +# Poll until done — get_job_details returns the full job object for one id while true - jobs = list_jobs(client; states = [JobState.PROCESSING, JobState.DONE]) - j = first(filter(x -> x["id"] == job_id, jobs)) - j["state"] == "done" && break + get_job_details(client; job_id = job_id)["state"] == "done" && break sleep(30) end @@ -172,6 +170,12 @@ end paths = batch_download(client; job_id = job_id, output_dir = "./batch_out") ``` +To survey all your jobs, `list_jobs(client; short = true)` returns just `id`, +`state`, and `ts_received` per job. Databento is migrating `list_jobs` to that +condensed shape (opt-in today, then the default, then the only form), so read +every other per-job field — `record_count`, `cost_usd`, `ts_expiration`, the +original request parameters — through [`get_job_details`](@ref). + See [`submit_job`](@ref) for the full kwarg list — `split_duration`, `packaging`, `delivery`, etc. diff --git a/src/DatabentoAPI.jl b/src/DatabentoAPI.jl index 85a1853..0089d9a 100644 --- a/src/DatabentoAPI.jl +++ b/src/DatabentoAPI.jl @@ -109,7 +109,7 @@ export get_range, foreach_record export list_publishers, list_datasets, list_schemas, list_fields export list_unit_prices, get_dataset_range, get_dataset_condition export get_record_count, get_billable_size, get_cost -export submit_job, list_jobs, list_files, batch_download +export submit_job, list_jobs, get_job_details, list_files, batch_download export resolve # Live lifecycle diff --git a/src/enums.jl b/src/enums.jl index 45effc5..0b2bef8 100644 --- a/src/enums.jl +++ b/src/enums.jl @@ -39,7 +39,7 @@ behavior is selected via the `reconnect::Bool` kwarg on """ JobState -State of a batch job returned by [`list_jobs`](@ref): +State of a batch job returned by [`list_jobs`](@ref) / [`get_job_details`](@ref): - `RECEIVED` — accepted by the API. - `QUEUED` — waiting to start. diff --git a/src/historical/batch.jl b/src/historical/batch.jl index 0e2e288..a3db079 100644 --- a/src/historical/batch.jl +++ b/src/historical/batch.jl @@ -4,7 +4,7 @@ submit_job(client; dataset, symbols, schema, start_dt, end_dt=nothing, ...) -> Dict Submit an asynchronous batch job. The Historical API queues the request and -returns a `job_id`; poll [`list_jobs`](@ref) until the [`JobState`](@ref) is +returns a `job_id`; poll [`get_job_details`](@ref) until the [`JobState`](@ref) is `DONE`, then call [`list_files`](@ref) and [`batch_download`](@ref) to fetch the results. @@ -74,18 +74,29 @@ function submit_job(c::Historical; end """ - list_jobs(client; states=nothing, since=nothing) + list_jobs(client; states=nothing, since=nothing, short=nothing) List batch jobs submitted under the calling API key, optionally filtered to one or more [`JobState`](@ref) values (`states` accepts a single state, a vector, or a comma-separated string) and/or to jobs created after `since` (`DateTime`, ISO-8601 string, or unix-ns integer). Returns a JSON array of -job objects. +job objects sorted by `ts_received`. + +`short=true` requests the condensed response — only `id`, `state`, and +`ts_received` per job — which is all a polling loop needs. Databento is +migrating `list_jobs` to this shape in phases (August 2026 batch API notice): +currently the full job object is returned unless `short=true`; a later phase +flips the server default to condensed; the final phase removes the parameter +and the legacy response altogether. Read anything beyond those three fields +via [`get_job_details`](@ref), not from `list_jobs`. With `short=nothing` +(default) the parameter is omitted and the server default applies, so the +default behavior of this function tracks Databento's rollout. Wraps `GET /v0/batch.list_jobs`. """ function list_jobs(c::Historical; states::Union{Nothing,AbstractVector,JobState.T,AbstractString} = nothing, - since::Union{Nothing,DateTime,AbstractString,Integer} = nothing) + since::Union{Nothing,DateTime,AbstractString,Integer} = nothing, + short::Union{Nothing,Bool} = nothing) states_v = if states isa AbstractVector join((s isa JobState.T ? lowercase(String(Symbol(s))) : String(s) for s in states), ",") elseif states isa JobState.T @@ -96,9 +107,25 @@ function list_jobs(c::Historical; nothing end return get_json(c.http, hist_path("batch.list_jobs"); - query = (states = states_v, since = ts_str(since))) + query = (states = states_v, since = ts_str(since), short = short)) end +""" + get_job_details(client; job_id) -> Dict + +Fetch the full record for one batch job: the request parameters (`dataset`, +`symbols`, `schema`, `start`, `end`, `encoding`, `compression`, `split_*`, +`packaging`, `delivery`, `limit`, ...), sizing and cost (`record_count`, +`billed_size`, `actual_size`, `package_size`, `cost_usd`), and lifecycle +(`state`, `progress`, `ts_received`, `ts_queued`, `ts_process_start`, +`ts_process_done`, `ts_expiration`). This is the endpoint for per-job fields +now that [`list_jobs`](@ref) is converging on a condensed +`id`/`state`/`ts_received` response. +Wraps `GET /v0/batch.get_job_details`. +""" +get_job_details(c::Historical; job_id::AbstractString) = + get_json(c.http, hist_path("batch.get_job_details"); query = (; job_id = String(job_id))) + """ list_files(client; job_id) diff --git a/test/test_historical_batch.jl b/test/test_historical_batch.jl index f60a701..2df6aab 100644 --- a/test/test_historical_batch.jl +++ b/test/test_historical_batch.jl @@ -96,4 +96,50 @@ using HTTP @test gets_called[] == 0 end end + + @testset "list_jobs short parameter (Aug 2026 batch API change)" begin + seen = Ref(Dict{String,String}()) + function mock(method, url, headers, body; kwargs...) + @test method == "GET" + @test occursin("batch.list_jobs", url) + seen[] = Dict{String,String}(get(kwargs, :query, Pair{String,String}[])) + HTTP.Response(200; body = """[{"id":"GLBX-20220901-5DEFXVTMSM","state":"done","ts_received":"2022-09-01T00:00:00Z"}]""") + end + c = Historical("test-key"; gateway = "https://hist.test", dispatcher = mock) + + # Default: parameter omitted so the server default applies through the + # phased rollout (full response now, condensed later). + list_jobs(c) + @test !haskey(seen[], "short") + + jobs = list_jobs(c; short = true) + @test seen[]["short"] == "true" + @test jobs[1]["id"] == "GLBX-20220901-5DEFXVTMSM" + @test jobs[1]["state"] == "done" + @test jobs[1]["ts_received"] == "2022-09-01T00:00:00Z" + + list_jobs(c; short = false, states = JobState.DONE) + @test seen[]["short"] == "false" + @test seen[]["states"] == "done" + end + + @testset "get_job_details queries by job_id" begin + function mock(method, url, headers, body; kwargs...) + @test method == "GET" + @test occursin("batch.get_job_details", url) + d = Dict{String,String}(get(kwargs, :query, Pair{String,String}[])) + @test d["job_id"] == "GLBX-20220901-5DEFXVTMSM" + HTTP.Response(200; body = """{"id":"GLBX-20220901-5DEFXVTMSM","state":"done", + "dataset":"GLBX.MDP3","schema":"mbo","symbols":"ESM2","stype_in":"raw_symbol", + "record_count":12345,"billed_size":98765,"cost_usd":0.5,"progress":100, + "ts_received":"2022-09-01T00:00:00Z","ts_expiration":"2022-10-01T00:00:00Z"}""") + end + c = Historical("test-key"; gateway = "https://hist.test", dispatcher = mock) + job = get_job_details(c; job_id = "GLBX-20220901-5DEFXVTMSM") + @test job["id"] == "GLBX-20220901-5DEFXVTMSM" + @test job["state"] == "done" + @test job["dataset"] == "GLBX.MDP3" + @test job["record_count"] == 12345 + @test job["cost_usd"] == 0.5 + end end