From 53314d59109e3182e626269ca2e3ebd67929a227 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Wed, 23 Sep 2026 15:18:34 -0400 Subject: [PATCH 1/2] fix(massive): read financial statements from the v1 statement endpoints fetch_financials() called /stocks/financials/v1/financials, which Massive retired on 2026-06-22; every call failed with HTTP 404. It now reads income-statements, balance-sheets and cash-flow-statements, parses their flat record shape, sorts most recent period first and follows next_url up to limit. TTM is supported for income and cash flow. filed_at is Massive's filing_date, the most recent filing that included the period, and is documented as not point-in-time. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F6TR4nQDjiPtanA2n4F3Gw --- docs/providers/massive.md | 34 +++++++- src/ml4t/data/providers/polygon.py | 122 ++++++++++++++++++--------- tests/integration/test_polygon.py | 19 +++++ tests/test_fundamentals_providers.py | 113 ++++++++++++++++++++----- 4 files changed, 228 insertions(+), 60 deletions(-) diff --git a/docs/providers/massive.md b/docs/providers/massive.md index c9e2c67..062a169 100644 --- a/docs/providers/massive.md +++ b/docs/providers/massive.md @@ -106,6 +106,39 @@ or pass `asset_class="futures"` when calling `fetch_ohlcv()`. --- +## Fundamentals + +`fetch_financials()` reads Massive's financial statement endpoints and returns the shared +long-form statement schema, one row per period and line item: + +| `statement` | Endpoint | Periods | +|-------------|----------|---------| +| `income` | `/stocks/financials/v1/income-statements` | `annual`, `quarterly`, `ttm` | +| `balance` | `/stocks/financials/v1/balance-sheets` | `annual`, `quarterly` | +| `cashflow` | `/stocks/financials/v1/cash-flow-statements` | `annual`, `quarterly`, `ttm` | + +```python +income = provider.fetch_financials("AAPL", statement="income", period="quarterly", limit=8) +ratios = provider.fetch_company_metrics("AAPL") # /stocks/financials/v1/ratios +``` + +Periods come back most recent first; `limit` caps the number of periods and the provider +follows `next_url` until it is reached. Line items keep Massive's field names (`revenue`, +`total_assets`, `net_cash_from_operating_activities`). `fiscal_period` is `Q1`-`Q4`, `FY` +or `TTM`. + +`filed_at` is Massive's `filing_date`: the most recent SEC filing that included the period, +not the filing that first reported it. A quarter repeated as a comparative in a later +report carries that later date, and its values may reflect later restatements. Neither +column tells you what was known on a given date, so these statements are not point-in-time +data. + +The statement endpoints need Stocks Advanced or the Financials & Ratios expansion. Earlier +versions of this provider called a financials endpoint that Massive retired on 2026-06-22, +which now returns HTTP 404. + +--- + ## API Key Setup ```bash @@ -139,7 +172,6 @@ POLYGON_API_KEY=your_existing_polygon_key |---------|---------------|----------| | Options chains | Advanced | HIGH | | Options Greeks | Advanced | HIGH | -| Financials | Advanced | HIGH | | Trades (tick) | Developer | MEDIUM | | Quotes (NBBO) | Developer | MEDIUM | | WebSockets | Any | NOT PLANNED | diff --git a/src/ml4t/data/providers/polygon.py b/src/ml4t/data/providers/polygon.py index 6cf9adb..c638e30 100644 --- a/src/ml4t/data/providers/polygon.py +++ b/src/ml4t/data/providers/polygon.py @@ -72,17 +72,33 @@ class MassiveProvider(BaseProvider): "1minute": "minute", } - FINANCIAL_STATEMENT_SECTION_MAP: ClassVar[dict[StatementType, tuple[str, ...]]] = { - "income": ("income_statement", "income"), - "balance": ("balance_sheet", "balance"), - "cashflow": ("cash_flow_statement", "cash_flow", "cashflow"), + FINANCIAL_STATEMENT_PATHS: ClassVar[dict[StatementType, str]] = { + "income": "/stocks/financials/v1/income-statements", + "balance": "/stocks/financials/v1/balance-sheets", + "cashflow": "/stocks/financials/v1/cash-flow-statements", } FINANCIAL_PERIOD_MAP: ClassVar[dict[PeriodType, str]] = { "annual": "annual", "quarterly": "quarterly", + "ttm": "trailing_twelve_months", } + # Record fields that describe the filing rather than a statement line item. + FINANCIAL_RECORD_METADATA: ClassVar[frozenset[str]] = frozenset( + { + "cik", + "filing_date", + "fiscal_quarter", + "fiscal_year", + "period_end", + "tickers", + "timeframe", + } + ) + + FINANCIAL_PAGE_SIZE_MAX: ClassVar[int] = 50_000 + def __init__( self, api_key: str | None = None, @@ -393,45 +409,62 @@ def fetch_financials( period: str = "annual", limit: int = 100, ) -> pl.DataFrame: - """Fetch Massive stock financial statements in canonical long form.""" + """Fetch Massive stock financial statements in canonical long form. + + Reads the income-statements, balance-sheets and cash-flow-statements endpoints, + most recent period first, following ``next_url`` until ``limit`` periods are read. + Trailing-twelve-months periods exist for income and cash flow statements only. + + ``filed_at`` is Massive's ``filing_date``: the most recent SEC filing that included + the period, which is later than the original filing when a later report restated + or repeated it as a comparative. It is not a point-in-time availability date, and + values may reflect later restatements. + """ try: statement_type = normalize_statement_type(statement) period_type = normalize_period_type(period) except ValueError as err: raise DataValidationError(self.name, str(err)) from err - if period_type == "ttm": + if period_type == "ttm" and statement_type == "balance": raise DataValidationError( self.name, - "Massive financial statements support annual and quarterly periods", + "Massive balance sheets support annual and quarterly periods", field="period", value=period, ) + if limit < 1: + raise DataValidationError( + self.name, "limit must be a positive integer", field="limit", value=limit + ) - data = self._request_json( - "/stocks/financials/v1/financials", + path = self.FINANCIAL_STATEMENT_PATHS[statement_type] + records = self._paginate_results( + path, { - "ticker": symbol.upper(), + "tickers": symbol.upper(), "timeframe": self.FINANCIAL_PERIOD_MAP[period_type], - "limit": limit, + "sort": "period_end.desc", + "limit": min(limit, self.FINANCIAL_PAGE_SIZE_MAX), }, + max_results=limit, ) + rows: list[dict[str, Any]] = [] - for record in data.get("results", []): - if not isinstance(record, dict): - continue - financials = record.get("financials", {}) - if not isinstance(financials, dict): - continue - section = self._pick_statement_section(financials, statement_type) - if not section: - continue + for record in records: + line_items = { + key: value + for key, value in record.items() + if key not in self.FINANCIAL_RECORD_METADATA + } combined = { - **section, - "end_date": record.get("end_date"), + **line_items, + "end_date": record.get("period_end"), "filing_date": record.get("filing_date"), - "fiscal_period": record.get("fiscal_period"), "fiscal_year": record.get("fiscal_year"), + "fiscal_period": self._fiscal_period_label( + period_type, record.get("fiscal_quarter") + ), } rows.extend( records_to_financials_rows( @@ -440,11 +473,33 @@ def fetch_financials( provider=self.name, statement_type=statement_type, period_type=period_type, - source="stocks/financials/v1/financials", + source=path.lstrip("/"), ) ) return rows_to_financials_frame(rows) + @staticmethod + def _fiscal_period_label(period_type: PeriodType, fiscal_quarter: Any) -> str | None: + if period_type == "annual": + return "FY" + if period_type == "ttm": + return "TTM" + return f"Q{fiscal_quarter}" if fiscal_quarter is not None else None + + def _paginate_results( + self, path: str, params: dict[str, Any], *, max_results: int + ) -> list[dict[str, Any]]: + """Collect ``results`` records across ``next_url`` pages, up to ``max_results``.""" + records: list[dict[str, Any]] = [] + data = self._request_json(path, params) + while True: + records.extend(item for item in data.get("results", []) if isinstance(item, dict)) + next_url = data.get("next_url") + if len(records) >= max_results or not isinstance(next_url, str) or not next_url: + return records[:max_results] + # next_url carries the query as an opaque cursor; only the key is re-sent. + data = self._get_json(next_url, {}) + def fetch_company_metrics( self, symbol: str, @@ -475,8 +530,11 @@ def fetch_company_metrics( return rows_to_company_metrics_frame(rows) def _request_json(self, path: str, params: dict[str, Any]) -> dict[str, Any]: - """Fetch JSON from a Massive endpoint.""" - endpoint = f"{self.base_url}{path}" + """Fetch JSON from a Massive endpoint path.""" + return self._get_json(f"{self.base_url}{path}", params) + + def _get_json(self, endpoint: str, params: dict[str, Any]) -> dict[str, Any]: + """Fetch JSON from an absolute Massive URL, adding the API key.""" request_params = {**params, "apiKey": self.api_key} try: self.rate_limiter.acquire(blocking=True) @@ -515,18 +573,6 @@ def _request_json(self, path: str, params: dict[str, Any]) -> dict[str, Any]: except Exception as err: raise NetworkError(provider=self.name, message=f"Request failed: {endpoint}") from err - @classmethod - def _pick_statement_section( - cls, - financials: dict[str, Any], - statement_type: StatementType, - ) -> dict[str, Any]: - for key in cls.FINANCIAL_STATEMENT_SECTION_MAP[statement_type]: - section = financials.get(key) - if isinstance(section, dict): - return section - return {} - class PolygonProvider(MassiveProvider): """Deprecated compatibility alias for Polygon.io integrations. diff --git a/tests/integration/test_polygon.py b/tests/integration/test_polygon.py index 0bc86d8..b7489a6 100644 --- a/tests/integration/test_polygon.py +++ b/tests/integration/test_polygon.py @@ -220,3 +220,22 @@ def test_invalid_api_key(self): if __name__ == "__main__": pytest.main([__file__, "-v", "-s"]) + + +class TestMassiveFinancials: + """Financial statements endpoints (Stocks Advanced or Financials & Ratios expansion).""" + + def test_fetch_quarterly_income_statement(self, provider): + frame = provider.fetch_financials("AAPL", statement="income", period="quarterly", limit=2) + + assert frame["period_end"].n_unique() == 2 + assert "revenue" in frame["line_item"].to_list() + assert frame["filed_at"].null_count() == 0 + assert frame["fiscal_period"].str.starts_with("Q").all() + + def test_fetch_annual_balance_sheet(self, provider): + frame = provider.fetch_financials("AAPL", statement="balance", period="annual", limit=1) + + assert frame["period_end"].n_unique() == 1 + assert "total_assets" in frame["line_item"].to_list() + assert set(frame["fiscal_period"]) == {"FY"} diff --git a/tests/test_fundamentals_providers.py b/tests/test_fundamentals_providers.py index 99ef2f8..604250d 100644 --- a/tests/test_fundamentals_providers.py +++ b/tests/test_fundamentals_providers.py @@ -222,33 +222,104 @@ class TestMassiveFundamentals: def provider(self): return MassiveProvider(api_key="test_key", rate_limit=(100, 1.0)) - def test_fetch_financials(self, provider): + # Income-statement record as documented for GET /stocks/financials/v1/income-statements + # (Apple fiscal Q3 2025), trimmed to a few line items. + INCOME_RECORD = { + "tickers": ["AAPL"], + "cik": "0000320193", + "period_end": "2025-06-28", + "filing_date": "2025-08-01", + "fiscal_quarter": 3, + "fiscal_year": 2025, + "timeframe": "quarterly", + "revenue": 94036000000, + "consolidated_net_income_loss": 23434000000, + "diluted_earnings_per_share": 1.57, + } + + @staticmethod + def _response(payload): response = MagicMock() response.status_code = 200 - response.json.return_value = { - "results": [ - { - "end_date": "2024-12-31", - "filing_date": "2025-02-01", - "fiscal_period": "FY", - "fiscal_year": 2024, - "financials": { - "income_statement": { - "revenues": {"value": 100.0, "unit": "USD"}, - "net_income_loss": {"value": 20.0, "unit": "USD"}, - } - }, - } - ] + response.json.return_value = payload + return response + + def test_fetch_financials_parses_flat_statement_records(self, provider): + payload = {"status": "OK", "request_id": "r1", "results": [self.INCOME_RECORD]} + + with patch.object(provider.rate_limiter, "acquire"): + with patch.object(provider.session, "get", return_value=self._response(payload)) as get: + frame = provider.fetch_financials("aapl", statement="income", period="quarterly") + + assert get.call_args.args[0] == ( + "https://api.massive.com/stocks/financials/v1/income-statements" + ) + params = get.call_args.kwargs["params"] + assert params["tickers"] == "AAPL" + assert params["timeframe"] == "quarterly" + assert params["sort"] == "period_end.desc" + + assert set(frame["line_item"]) == { + "revenue", + "consolidated_net_income_loss", + "diluted_earnings_per_share", } + row = frame.filter(frame["line_item"] == "revenue").row(0, named=True) + assert row["value"] == 94036000000.0 + assert row["period_end"] == "2025-06-28" + assert row["filed_at"] == "2025-08-01" + assert row["fiscal_year"] == 2025 + assert row["fiscal_period"] == "Q3" + assert row["statement_type"] == "income" + assert row["period_type"] == "quarterly" + assert row["source"] == "stocks/financials/v1/income-statements" + + def test_fetch_financials_follows_next_url_until_limit(self, provider): + second = {**self.INCOME_RECORD, "period_end": "2025-03-29", "fiscal_quarter": 2} + third = {**self.INCOME_RECORD, "period_end": "2024-12-28", "fiscal_quarter": 1} + next_url = "https://api.massive.com/stocks/financials/v1/income-statements?cursor=abc" + pages = [ + self._response({"status": "OK", "results": [self.INCOME_RECORD], "next_url": next_url}), + self._response( + {"status": "OK", "results": [second, third], "next_url": next_url + "d"} + ), + ] with patch.object(provider.rate_limiter, "acquire"): - with patch.object(provider.session, "get", return_value=response) as get: - frame = provider.fetch_financials("AAPL") + with patch.object(provider.session, "get", side_effect=pages) as get: + frame = provider.fetch_financials("AAPL", period="quarterly", limit=2) + + assert get.call_count == 2 + assert get.call_args_list[1].args[0] == next_url + assert get.call_args_list[1].kwargs["params"] == {"apiKey": "test_key"} + assert sorted(set(frame["period_end"])) == ["2025-03-29", "2025-06-28"] + + @pytest.mark.parametrize( + ("statement", "period", "path", "timeframe", "fiscal_period"), + [ + ("balance", "annual", "balance-sheets", "annual", "FY"), + ("cashflow", "quarterly", "cash-flow-statements", "quarterly", "Q3"), + ("income", "ttm", "income-statements", "trailing_twelve_months", "TTM"), + ], + ) + def test_fetch_financials_routes_statement_and_period( + self, provider, statement, period, path, timeframe, fiscal_period + ): + record = {**self.INCOME_RECORD, "timeframe": timeframe} + payload = {"status": "OK", "results": [record]} - assert len(frame) == 2 - assert set(frame["line_item"]) == {"revenues", "net_income_loss"} - assert get.call_args.kwargs["params"]["timeframe"] == "annual" + with patch.object(provider.rate_limiter, "acquire"): + with patch.object(provider.session, "get", return_value=self._response(payload)) as get: + frame = provider.fetch_financials("AAPL", statement=statement, period=period) + + assert get.call_args.args[0].endswith(f"/stocks/financials/v1/{path}") + assert get.call_args.kwargs["params"]["timeframe"] == timeframe + assert set(frame["statement_type"]) == {statement} + assert set(frame["fiscal_period"]) == {fiscal_period} + + def test_fetch_financials_rejects_ttm_balance_sheet(self, provider): + with pytest.raises(DataValidationError, match="balance sheets"): + provider.fetch_financials("AAPL", statement="balance", period="ttm") def test_fetch_company_metrics(self, provider): response = MagicMock() From 87ca01f787d9e291ceb2992995d2deeb0dbf119c Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Wed, 23 Sep 2026 16:10:13 -0400 Subject: [PATCH 2/2] fix(massive): date company metrics from the ratios row's date field A live call to /stocks/financials/v1/ratios returns flat fields dated by 'date'; the provider read 'end_date', so every metric came back with as_of null. The unit test used an invented nested shape with end_date and could not see it. The test now uses the live row shape, and a live test asserts as_of is populated. Live financial tests move above the __main__ block. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F6TR4nQDjiPtanA2n4F3Gw --- src/ml4t/data/providers/polygon.py | 2 +- tests/integration/test_polygon.py | 14 ++++++++++---- tests/test_fundamentals_providers.py | 20 ++++++++++++-------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/ml4t/data/providers/polygon.py b/src/ml4t/data/providers/polygon.py index c638e30..5a05c6e 100644 --- a/src/ml4t/data/providers/polygon.py +++ b/src/ml4t/data/providers/polygon.py @@ -522,7 +522,7 @@ def fetch_company_metrics( symbol=symbol, provider=self.name, period=record.get("fiscal_period"), - as_of=record.get("end_date"), + as_of=record.get("date") or record.get("end_date"), source="stocks/financials/v1/ratios", metrics=metrics, ) diff --git a/tests/integration/test_polygon.py b/tests/integration/test_polygon.py index b7489a6..9a1e214 100644 --- a/tests/integration/test_polygon.py +++ b/tests/integration/test_polygon.py @@ -218,10 +218,6 @@ def test_invalid_api_key(self): # ... (all updater tests commented out) -if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) - - class TestMassiveFinancials: """Financial statements endpoints (Stocks Advanced or Financials & Ratios expansion).""" @@ -239,3 +235,13 @@ def test_fetch_annual_balance_sheet(self, provider): assert frame["period_end"].n_unique() == 1 assert "total_assets" in frame["line_item"].to_list() assert set(frame["fiscal_period"]) == {"FY"} + + def test_fetch_company_metrics_is_dated(self, provider): + frame = provider.fetch_company_metrics("AAPL") + + assert "market_cap" in frame["metric"].to_list() + assert frame["as_of"].null_count() == 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-s"]) diff --git a/tests/test_fundamentals_providers.py b/tests/test_fundamentals_providers.py index 604250d..49205d3 100644 --- a/tests/test_fundamentals_providers.py +++ b/tests/test_fundamentals_providers.py @@ -322,17 +322,19 @@ def test_fetch_financials_rejects_ttm_balance_sheet(self, provider): provider.fetch_financials("AAPL", statement="balance", period="ttm") def test_fetch_company_metrics(self, provider): + # Shape of a live /stocks/financials/v1/ratios row: flat fields, dated by `date`. response = MagicMock() response.status_code = 200 response.json.return_value = { "results": [ { "ticker": "AAPL", - "end_date": "2024-12-31", - "fiscal_period": "FY", - "fiscal_year": 2024, - "valuation": {"price_to_earnings": 30.0}, - "profitability": {"return_on_equity": 0.45}, + "cik": "0000320193", + "date": "2026-09-22", + "price": 339.75, + "market_cap": 4958372655000.0, + "price_to_earnings": 38.5, + "return_on_equity": 1.5, } ] } @@ -341,11 +343,13 @@ def test_fetch_company_metrics(self, provider): with patch.object(provider.session, "get", return_value=response): frame = provider.fetch_company_metrics("AAPL") - assert len(frame) == 2 assert set(frame["metric"]) == { - "valuation.price_to_earnings", - "profitability.return_on_equity", + "price", + "market_cap", + "price_to_earnings", + "return_on_equity", } + assert set(frame["as_of"]) == {"2026-09-22"} def test_fetch_company_metrics_provider_options_are_keyword_only(self, provider): signature = inspect.signature(provider.fetch_company_metrics)