diff --git a/AGENTS.md b/AGENTS.md index 6bdd70c..74644a7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,7 +47,7 @@ documentation, not in agent guides. directories. - Use shared exceptions, retry, rate-limit, and provider contracts rather than adapter-specific variants of the same behavior. -- Treat `CHANGELOG.md` and generated version files as generated artifacts; do not edit them by hand. +- Treat generated version files as generated artifacts; do not edit them by hand. - Keep release artifacts bound to one commit. Publishing is performed by the release workflow after qualification and documentation deployment, never by a local publish command. diff --git a/README.md b/README.md index 525c8ba..78fbe5d 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ emit `DeprecationWarning` and will not be removed before version 1.0. - [Documentation](https://www.ml4trading.io/docs/data/) - [Issue tracker](https://github.com/ml4t/data/issues) -- [Release notes](CHANGELOG.md) +- [Release notes](https://github.com/ml4t/data/releases) - [License](LICENSE) The library supplies data to [ml4t-engineer](https://github.com/ml4t/engineer) feature workflows and diff --git a/api.yaml b/api.yaml deleted file mode 100644 index 34505e8..0000000 --- a/api.yaml +++ /dev/null @@ -1,190 +0,0 @@ -name: ml4t-data -version: "1.0.0" -description: Unified market data acquisition and storage for quantitative finance - -providers: - equity: - - name: YahooFinanceProvider - import: ml4t.data.providers.yahoo - free_tier: unlimited - best_for: Quick start, US equities - - - name: EODHDProvider - import: ml4t.data.providers.eodhd - free_tier: 500/day - best_for: Global equities (60+ exchanges) - - - name: DatabentoProvider - import: ml4t.data.providers.databento - free_tier: trial - best_for: Institutional-grade data - - - name: PolygonProvider - import: ml4t.data.providers.polygon - free_tier: limited - best_for: US equities tick data - - - name: TiingoProvider - import: ml4t.data.providers.tiingo - free_tier: 500/day - best_for: US equities alternative - - - name: TwelveDataProvider - import: ml4t.data.providers.twelve_data - free_tier: 800/day - best_for: Multi-asset coverage - - - name: FinnhubProvider - import: ml4t.data.providers.finnhub - free_tier: limited - best_for: Real-time quotes - - crypto: - - name: BinanceProvider - import: ml4t.data.providers.binance - free_tier: generous - best_for: Spot and futures data - - - name: CoinGeckoProvider - import: ml4t.data.providers.coingecko - free_tier: 50/min - best_for: Market overview - - - name: CryptoCompareProvider - import: ml4t.data.providers.cryptocompare - free_tier: 250k/month - best_for: Historical crypto data - - - name: OKXProvider - import: ml4t.data.providers.okx - free_tier: unlimited - best_for: Exchange data - - forex: - - name: OandaProvider - import: ml4t.data.providers.oanda - free_tier: trial - best_for: Institutional forex - - factors: - - name: FamaFrenchProvider - import: ml4t.data.providers.fama_french - free_tier: unlimited - datasets: 50+ - best_for: Academic factors (FF3, FF5, Momentum) - - - name: AQRFactorProvider - import: ml4t.data.providers.aqr - free_tier: unlimited - datasets: 16 - best_for: Alternative factors (QMJ, BAB, TSMOM) - - - name: FREDProvider - import: ml4t.data.providers.fred - free_tier: unlimited - best_for: Economic indicators - - historical: - - name: WikiPricesProvider - import: ml4t.data.providers.wiki_prices - free_tier: local - coverage: US equities 1962-2018 - best_for: Historical fallback - -storage: - classes: - - name: HiveStorage - import: ml4t.data.storage.hive - pattern: provider/frequency/year=YYYY/month=MM/symbol/data.parquet - best_for: Production use (recommended) - - - name: FlatStorage - import: ml4t.data.storage.flat - pattern: symbol.parquet - best_for: Simple use cases - - - name: MetadataTracker - import: ml4t.data.storage.metadata_tracker - purpose: Track update timestamps and row counts - -core: - models: - - OHLCVBar - - SymbolSpec - - Timeframe - - ProviderCapabilities - - exceptions: - - ProviderError - - RateLimitError - - SymbolNotFoundError - - DataValidationError - - CircuitBreakerOpen - -utilities: - anomaly: - - name: AnomalyManager - import: ml4t.data.anomaly.manager - detectors: - - ReturnOutlierDetector - - VolumeSpikeDetector - - PriceStalenessDetector - - validation: - - name: OHLCVValidator - import: ml4t.data.validation.ohlcv - checks: - - high >= low - - high >= open, close - - low <= open, close - - volume >= 0 - -cli: - commands: - - name: fetch - description: Fetch data for a symbol - args: [--provider, --symbol, --start, --end, --frequency] - - - name: update-all - description: Update all datasets from config - args: [-c/--config, --dataset, --dry-run] - - - name: list - description: List stored data - args: [--storage, --provider, --symbol] - - - name: status - description: Show data status - args: [--symbol, --provider] - -protocols: - - name: OHLCVProvider - methods: - - fetch_ohlcv(symbol, start, end, frequency) -> DataFrame - - capabilities() -> ProviderCapabilities - - - name: AsyncOHLCVProvider - methods: - - fetch_ohlcv_async(symbol, start, end, frequency) -> DataFrame - - - name: FactorProvider - methods: - - fetch(dataset, frequency) -> DataFrame - - list_datasets() -> list[str] - -examples: - basic_fetch: | - from ml4t.data.providers import YahooFinanceProvider - provider = YahooFinanceProvider() - df = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-12-31") - - async_batch: | - from ml4t.data.managers.async_batch import async_batch_load - async with YahooFinanceProvider() as provider: - df = await async_batch_load(provider, ["AAPL", "MSFT"], "2024-01-01", "2024-12-31") - - with_storage: | - from ml4t.data.storage import HiveStorage, StorageConfig - storage = HiveStorage(config=StorageConfig(base_path="~/ml4t-data")) - storage.write(df, symbol="AAPL", provider="yahoo", frequency="daily") - df = storage.read(symbol="AAPL") diff --git a/docs/getting-started/migration.md b/docs/getting-started/migration.md index 3c68e50..90b1d78 100644 --- a/docs/getting-started/migration.md +++ b/docs/getting-started/migration.md @@ -1,6 +1,8 @@ -# Migrate former QLDM names +# Migrate compatibility names -The package preserves two former public names during the 0.x release series. Their use emits +## Names preserved through the 0.x series + +The package preserves two former QLDM names during the 0.x release series. Their use emits `DeprecationWarning`. They will not be removed before version 1.0. ## Data root @@ -33,3 +35,37 @@ from ml4t.data.core.exceptions import ML4TDataError The former exception name resolves to the same class during the migration period. Code that treats deprecation warnings as errors should migrate before upgrading. + +## Changes for version 0.2 + +Version 0.2 removes three compatibility interfaces that emitted `DeprecationWarning` in the 0.1 +series. Update callers and saved configuration before upgrading. + +### Massive provider name + +Replace `PolygonProvider` imports and the `polygon` registry name with `MassiveProvider` and +`massive`: + +```python +from ml4t.data.providers import MassiveProvider + +provider = MassiveProvider() +``` + +The service account can still use `POLYGON_API_KEY`; new environments should set +`MASSIVE_API_KEY`. + +### Negative-price validation + +Replace `check_negative_prices=True` with `negative_price_policy="forbid"`, and replace +`check_negative_prices=False` with `negative_price_policy="allow"`. Apply the same substitution in +saved `ValidationRuleConfig` YAML. + +Use `negative_price_policy="warn"` when negative prices should produce validation warnings without +failing validation. + +### Synthetic GARCH configuration + +Remove the `garch_omega` argument from `SyntheticProvider`. The provider derives the per-period +constant from `annual_volatility`, the requested frequency, `garch_alpha`, and `garch_beta` so the +configured unconditional variance is preserved. diff --git a/docs/providers/index.md b/docs/providers/index.md index 5a76718..6b4af85 100644 --- a/docs/providers/index.md +++ b/docs/providers/index.md @@ -8,8 +8,8 @@ For the wider vendor landscape, including sources the library does not wrap, see ## Provider Comparison -| Provider | Asset Class | Free Tier | Async | API Key | -|----------|-------------|-----------|-------|---------| +| Provider | Asset Class | Access | Async | API Key | +|----------|-------------|--------|-------|---------| | [Yahoo Finance](yahoo.md) | Stocks, ETFs, Crypto | Unlimited | Thread | No | | [CoinGecko](coingecko.md) | Crypto | 10K+ coins | Native | No | | [FRED](fred.md) | Economic Data | 120/min | Thread | Yes | @@ -25,7 +25,7 @@ For the wider vendor landscape, including sources the library does not wrap, see | [Tiingo](tiingo.md) | US Stocks | 1000/day | Thread | Yes | | [TwelveData](twelve_data.md) | Multi-asset | 800/day | Native | Yes | | [DataBento](databento.md) | Futures, Options | Free metadata; metered history | Thread | Yes | -| [Massive](massive.md) | Multi-asset | Free tier | Thread | Yes | +| [Massive](massive.md) | Multi-asset | Account-dependent | Thread | Yes | | [Finnhub](finnhub.md) | US quotes; premium OHLCV | 60 requests/minute | Thread | Yes | | [Binance](binance.md) | Crypto | Unlimited | Native | No | | [OKX](okx.md) | Crypto Perpetuals | No geo-limits | Native | No | diff --git a/docs/providers/massive.md b/docs/providers/massive.md index 062a169..7da5f06 100644 --- a/docs/providers/massive.md +++ b/docs/providers/massive.md @@ -1,185 +1,85 @@ -# Massive Provider +# Massive provider -**Provider**: `MassiveProvider` -**Legacy Alias**: `PolygonProvider` -**Website**: [massive.com](https://massive.com) -**API Key**: Required -**Free Tier**: 5 API calls/minute +`MassiveProvider` retrieves aggregate bars for stocks, options, futures, crypto, and foreign +exchange. It also retrieves stock financial statements and company ratios. A Massive account and +API key are required; accessible datasets, history, and request quotas depend on the account. ---- +## Configure access -## Overview +Set `MASSIVE_API_KEY` or pass the key directly: -Massive, formerly Polygon.io, provides comprehensive market data across stocks, -options, futures, forex, and crypto with institutional-quality tick, quote, -reference, and aggregate bar data. - -**Best For**: US equities, options research, futures reference data, tick data, -and multi-asset research workflows. - -**Compatibility**: Existing Polygon.io API keys remain valid. New code should use -`MassiveProvider` and `MASSIVE_API_KEY`; `PolygonProvider` and `POLYGON_API_KEY` -remain supported for backward compatibility. - -**Pricing**: -| Tier | Price | Features | -|------|-------|----------| -| Basic (Free) | $0/mo | 5 calls/min, 2yr history, EOD | -| Starter | $29/mo | Unlimited calls, 5yr history, 15min delayed | -| Developer | $79/mo | 10yr history, trades data | -| Advanced | $199/mo | 20yr+ history, real-time, quotes, financials | +```bash +export MASSIVE_API_KEY=your_api_key +``` ---- +Accounts created under the former Polygon.io name can continue to use `POLYGON_API_KEY`. If both +variables are set, `MASSIVE_API_KEY` takes precedence. -## Quick Start +## Fetch aggregate bars ```python -import os -os.environ["MASSIVE_API_KEY"] = "your_key_here" - from ml4t.data.providers import MassiveProvider provider = MassiveProvider() - -# Stocks -df = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-12-01", frequency="daily") - -# Options -options = provider.fetch_ohlcv( - "O:SPY240119C00480000", - "2024-01-01", - "2024-01-19", - frequency="daily", -) - -# Futures use Massive's futures endpoint; use a prefix or explicit asset class -futures = provider.fetch_ohlcv( - "F:ESM6", - "2024-01-01", - "2024-01-31", - frequency="daily", -) - -# Crypto and forex -btc = provider.fetch_ohlcv("X:BTCUSD", "2024-01-01", "2024-01-31", frequency="daily") -eurusd = provider.fetch_ohlcv("C:EURUSD", "2024-01-01", "2024-01-31", frequency="daily") - -provider.close() +try: + bars = provider.fetch_ohlcv( + "AAPL", + "2024-01-01", + "2024-01-31", + frequency="daily", + ) +finally: + provider.close() ``` ---- - -## Asset Classes - -| Asset Class | Symbol Format | REST Route | -|-------------|---------------|------------| -| Stocks | `AAPL` | aggregate bars | -| Options | `O:SPY240119C00480000` | aggregate bars | -| Futures | `F:ESM6` or `asset_class="futures"` | futures aggregate bars | -| Crypto | `X:BTCUSD` | aggregate bars | -| Forex | `C:EURUSD` | aggregate bars | +The provider accepts daily, weekly, monthly, hourly, and minute aliases. Symbols select the asset +route as follows: -Unprefixed futures tickers can be ambiguous with equities. Use the `F:` prefix -or pass `asset_class="futures"` when calling `fetch_ohlcv()`. +| Asset class | Symbol example | Routing rule | +|---|---|---| +| Stocks | `AAPL` | Default for an unprefixed symbol | +| Options | `O:SPY240119C00480000` | `O:` prefix | +| Futures | `F:ESM6` | `F:` or `FUT:` prefix | +| Crypto | `X:BTCUSD` | `X:` prefix | +| Foreign exchange | `C:EURUSD` | `C:` prefix | ---- +An unprefixed futures symbol is ambiguous with an equity symbol. Pass +`asset_class="futures"` or add the `F:` prefix. -## Supported Frequencies +## Fetch fundamentals -| Frequency | Free Tier | Paid Tier | -|-----------|-----------|-----------| -| `daily` | Yes (2yr) | Yes (20yr+) | -| `1h` | No | Yes | -| `1m` | No | Yes | -| Tick | No | Yes (Developer+) | - ---- - -## Coverage - -- **Stocks**: All US exchanges (NYSE, NASDAQ, etc.) -- **Options**: Full OPRA data (Advanced tier) -- **Futures**: CME, CBOT, COMEX, NYMEX contracts, products, schedules, and bars -- **Crypto**: Major cryptocurrencies -- **Forex**: Major pairs -- **Indices**: Major US indices - ---- - -## Fundamentals - -`fetch_financials()` reads Massive's financial statement endpoints and returns the shared -long-form statement schema, one row per period and line item: - -| `statement` | Endpoint | Periods | -|-------------|----------|---------| -| `income` | `/stocks/financials/v1/income-statements` | `annual`, `quarterly`, `ttm` | -| `balance` | `/stocks/financials/v1/balance-sheets` | `annual`, `quarterly` | -| `cashflow` | `/stocks/financials/v1/cash-flow-statements` | `annual`, `quarterly`, `ttm` | +`fetch_financials()` returns the shared long-form statement schema. Supported statements are +`income`, `balance`, and `cashflow`; supported periods are `annual` and `quarterly`. Income and cash +flow statements also accept `ttm`. ```python -income = provider.fetch_financials("AAPL", statement="income", period="quarterly", limit=8) -ratios = provider.fetch_company_metrics("AAPL") # /stocks/financials/v1/ratios -``` - -Periods come back most recent first; `limit` caps the number of periods and the provider -follows `next_url` until it is reached. Line items keep Massive's field names (`revenue`, -`total_assets`, `net_cash_from_operating_activities`). `fiscal_period` is `Q1`-`Q4`, `FY` -or `TTM`. - -`filed_at` is Massive's `filing_date`: the most recent SEC filing that included the period, -not the filing that first reported it. A quarter repeated as a comparative in a later -report carries that later date, and its values may reflect later restatements. Neither -column tells you what was known on a given date, so these statements are not point-in-time -data. - -The statement endpoints need Stocks Advanced or the Financials & Ratios expansion. Earlier -versions of this provider called a financials endpoint that Massive retired on 2026-06-22, -which now returns HTTP 404. - ---- - -## API Key Setup - -```bash -# Environment variable -export MASSIVE_API_KEY=your_api_key_here -``` - -Get your API key at [massive.com](https://massive.com). - -Existing users can continue to use: - -```bash -POLYGON_API_KEY=your_existing_polygon_key +provider = MassiveProvider() +try: + income = provider.fetch_financials( + "AAPL", + statement="income", + period="quarterly", + limit=8, + ) + ratios = provider.fetch_company_metrics("AAPL") +finally: + provider.close() ``` ---- - -## Rate Limits - -| Tier | Limit | -|------|-------| -| Basic | 5 calls/minute | -| Starter | Unlimited | -| Developer+ | Unlimited | - ---- - -## Not Yet Implemented +Results are ordered most recent first, and `limit` caps the number of statement periods. The +`filed_at` field is the most recent SEC filing that included a period. A later filing can repeat or +restate an earlier period, so this field does not establish when a value first became available. +Do not treat these results as point-in-time data without an independent filing-history check. -| Feature | Tier Required | Priority | -|---------|---------------|----------| -| Options chains | Advanced | HIGH | -| Options Greeks | Advanced | HIGH | -| Trades (tick) | Developer | MEDIUM | -| Quotes (NBBO) | Developer | MEDIUM | -| WebSockets | Any | NOT PLANNED | +## Rate limiting and failures ---- +The default client pace is five calls per minute. This is a conservative local setting, not a +statement about the service plan. Pass `rate_limit=(calls, period_seconds)` to use a lower pace +required by an account or workload. -## See Also +Authentication failures raise `AuthenticationError`; HTTP rate limits raise `RateLimitError` with +the reported retry delay; missing symbols raise `SymbolNotFoundError`; other transport and response +failures use the shared provider exception types. -- [Massive Pricing](https://massive.com/pricing) -- [Massive REST Docs](https://massive.com/docs) -- [Polygon compatibility note](polygon.md) +See the current [Massive API documentation](https://massive.com/docs) and +[account plans](https://massive.com/pricing) for service-side coverage and limits. diff --git a/docs/providers/polygon.md b/docs/providers/polygon.md deleted file mode 100644 index 32361bf..0000000 --- a/docs/providers/polygon.md +++ /dev/null @@ -1,17 +0,0 @@ -# Polygon Provider - -`PolygonProvider` remains available as a deprecated compatibility alias for -existing Polygon.io integrations. Polygon.io rebranded to Massive.com, and -existing Polygon API keys continue to work. - -New code should use: - -```python -from ml4t.data.providers import MassiveProvider -``` - -Set `MASSIVE_API_KEY` for new environments. Existing `POLYGON_API_KEY` values are -still supported. - -See the canonical [Massive provider](massive.md) documentation for current -coverage, examples, and limitations. diff --git a/examples/configs/modular/providers.yaml b/examples/configs/modular/providers.yaml index 9427d9b..0c359de 100644 --- a/examples/configs/modular/providers.yaml +++ b/examples/configs/modular/providers.yaml @@ -32,7 +32,7 @@ providers: # Premium providers (require API keys) - name: massive type: massive - api_key: ${POLYGON_API_KEY} + api_key: ${MASSIVE_API_KEY} rate_limit: 5.0 # Development/testing diff --git a/mkdocs.yml b/mkdocs.yml index cc9b7fe..d1409f5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -142,7 +142,6 @@ nav: - TwelveData: providers/twelve_data.md - DataBento: providers/databento.md - Massive: providers/massive.md - - Polygon Compatibility: providers/polygon.md - Finnhub: providers/finnhub.md - Binance: providers/binance.md - OKX: providers/okx.md diff --git a/pyproject.toml b/pyproject.toml index b0b7d03..004cb6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ include = [ "/scripts/verify_provider_contract_runs.py", "/README.md", "/LICENSE", - "/CHANGELOG.md", ] [project] @@ -160,7 +159,7 @@ Homepage = "https://www.ml4trading.io/" Documentation = "https://www.ml4trading.io/docs/data/" Repository = "https://github.com/ml4t/data" Issues = "https://github.com/ml4t/data/issues" -Changelog = "https://github.com/ml4t/data/blob/main/CHANGELOG.md" +Changelog = "https://github.com/ml4t/data/releases" [dependency-groups] test = [ @@ -209,7 +208,6 @@ markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", "paid_tier: marks tests requiring paid API tier (skipped by default)", "integration: marks tests as integration tests requiring external APIs", - "real_api: marks tests that use real API calls (deprecated, use integration)", "requires_api_key: marks tests requiring specific API keys", "expensive: marks tests with high API costs", "network_guard_probe: exercises the offline network guard itself", diff --git a/scripts/release_candidate.py b/scripts/release_candidate.py index 2ab2b13..e51378b 100644 --- a/scripts/release_candidate.py +++ b/scripts/release_candidate.py @@ -26,7 +26,7 @@ "Documentation": "https://www.ml4trading.io/docs/data/", "Repository": "https://github.com/ml4t/data", "Issues": "https://github.com/ml4t/data/issues", - "Changelog": "https://github.com/ml4t/data/blob/main/CHANGELOG.md", + "Changelog": "https://github.com/ml4t/data/releases", } EXPECTED_KEYWORDS = {"finance", "quantitative-finance", "algorithmic-trading", "market-data"} EXPECTED_CLASSIFIERS = { diff --git a/scripts/verify_distribution.py b/scripts/verify_distribution.py index 7c7a0f5..f81743c 100644 --- a/scripts/verify_distribution.py +++ b/scripts/verify_distribution.py @@ -28,7 +28,7 @@ "Documentation": "https://www.ml4trading.io/docs/data/", "Repository": "https://github.com/ml4t/data", "Issues": "https://github.com/ml4t/data/issues", - "Changelog": "https://github.com/ml4t/data/blob/main/CHANGELOG.md", + "Changelog": "https://github.com/ml4t/data/releases", } diff --git a/src/ml4t/data/config/models.py b/src/ml4t/data/config/models.py index 9ce1256..4b60cae 100644 --- a/src/ml4t/data/config/models.py +++ b/src/ml4t/data/config/models.py @@ -63,7 +63,6 @@ class ProviderType(StrEnum): DATABENTO = "databento" MASSIVE = "massive" OANDA = "oanda" - POLYGON = "polygon" TWELVE_DATA = "twelve_data" NASDAQ_ITCH = "nasdaq_itch" WIKI_PRICES = "wiki_prices" diff --git a/src/ml4t/data/config/validator.py b/src/ml4t/data/config/validator.py index d91fec1..81784e4 100644 --- a/src/ml4t/data/config/validator.py +++ b/src/ml4t/data/config/validator.py @@ -65,10 +65,7 @@ def _validate_providers(self) -> None: provider_names.add(provider.name) # Check API keys for providers that need them - if ( - provider.type in ["massive", "polygon", "cryptocompare", "alpaca"] - and not provider.api_key - ): + if provider.type in ["massive", "cryptocompare", "alpaca"] and not provider.api_key: self.warnings.append( f"Provider {provider.name} ({provider.type}) may require an API key" ) diff --git a/src/ml4t/data/provider_updater.py b/src/ml4t/data/provider_updater.py index 704fbfe..cbf11ac 100644 --- a/src/ml4t/data/provider_updater.py +++ b/src/ml4t/data/provider_updater.py @@ -55,7 +55,7 @@ def __init__( """Initialize provider updater. Args: - provider_name: Name of data provider (e.g., "yahoo", "polygon") + provider_name: Name of data provider (e.g., "yahoo", "massive") storage: Storage backend supporting incremental updates safety_margin_minutes: Minutes to subtract from latest timestamp to avoid missing data due to delayed updates diff --git a/src/ml4t/data/providers/__init__.py b/src/ml4t/data/providers/__init__.py index 2813aa0..c2ccaf1 100644 --- a/src/ml4t/data/providers/__init__.py +++ b/src/ml4t/data/providers/__init__.py @@ -101,7 +101,7 @@ OandaProvider = None # type: ignore # Multi-asset providers -from ml4t.data.providers.polygon import MassiveProvider, PolygonProvider +from ml4t.data.providers.polygon import MassiveProvider from ml4t.data.providers.twelve_data import TwelveDataProvider # Market data providers @@ -161,7 +161,6 @@ "OandaProvider", # Multi-asset providers "MassiveProvider", - "PolygonProvider", "TwelveDataProvider", # Market data providers "DataBentoProvider", diff --git a/src/ml4t/data/providers/polygon.py b/src/ml4t/data/providers/polygon.py index 5a05c6e..db2e841 100644 --- a/src/ml4t/data/providers/polygon.py +++ b/src/ml4t/data/providers/polygon.py @@ -4,7 +4,6 @@ """ import os -import warnings from typing import Any, ClassVar, Literal import polars as pl @@ -572,28 +571,3 @@ def _get_json(self, endpoint: str, params: dict[str, Any]) -> dict[str, Any]: raise except Exception as err: raise NetworkError(provider=self.name, message=f"Request failed: {endpoint}") from err - - -class PolygonProvider(MassiveProvider): - """Deprecated compatibility alias for Polygon.io integrations. - - Use ``MassiveProvider`` for new code. - """ - - def __init__( - self, - api_key: str | None = None, - rate_limit: tuple[int, float] | None = None, - base_url: str | None = None, - ) -> None: - warnings.warn( - "PolygonProvider is deprecated; use MassiveProvider instead.", - DeprecationWarning, - stacklevel=2, - ) - super().__init__(api_key=api_key, rate_limit=rate_limit, base_url=base_url) - - @property - def name(self) -> str: - """Return provider name.""" - return "polygon" diff --git a/src/ml4t/data/providers/registry.py b/src/ml4t/data/providers/registry.py index b370fe7..6589550 100644 --- a/src/ml4t/data/providers/registry.py +++ b/src/ml4t/data/providers/registry.py @@ -266,16 +266,6 @@ def _spec( "ohlcv", credentials=(_credential("api_key", "POLYGON_API_KEY", "MASSIVE_API_KEY"),), ), - _spec( - "polygon", - "polygon", - "PolygonProvider", - "Deprecated Polygon alias for Massive", - "ohlcv", - credentials=(_credential("api_key", "POLYGON_API_KEY"),), - advertised=False, - deprecated=True, - ), _spec( "twelve_data", "twelve_data", diff --git a/src/ml4t/data/providers/synthetic.py b/src/ml4t/data/providers/synthetic.py index 76d9a24..caa6dcb 100644 --- a/src/ml4t/data/providers/synthetic.py +++ b/src/ml4t/data/providers/synthetic.py @@ -24,7 +24,6 @@ from datetime import UTC, datetime from typing import ClassVar, Literal -from warnings import warn import numpy as np import polars as pl @@ -86,9 +85,6 @@ class SyntheticProvider(BaseProvider): GARCH: ARCH term weight (reaction to recent shocks) garch_beta : float, default=0.85 GARCH: GARCH term weight (persistence of variance) - garch_omega : float, optional - Deprecated compatibility parameter. Omega is derived from annual volatility, - frequency, alpha, and beta so the requested unconditional variance is preserved. Example ------- @@ -123,7 +119,6 @@ def __init__( heston_xi: float = 0.3, heston_rho: float = -0.7, # GARCH parameters - garch_omega: float | None = None, garch_alpha: float = 0.1, garch_beta: float = 0.85, calendar_mode: CalendarMode = "equity", @@ -148,13 +143,6 @@ def __init__( # GARCH model parameters self.garch_alpha = garch_alpha # ARCH term self.garch_beta = garch_beta # GARCH term - if garch_omega is not None: - warn( - "garch_omega is deprecated and ignored; omega is derived from " - "annual_volatility and the requested frequency", - DeprecationWarning, - stacklevel=2, - ) # Validate GARCH stationarity if model == "garch" and (garch_alpha + garch_beta) >= 1.0: diff --git a/src/ml4t/data/storage/protocols.py b/src/ml4t/data/storage/protocols.py index b35770e..1129974 100644 --- a/src/ml4t/data/storage/protocols.py +++ b/src/ml4t/data/storage/protocols.py @@ -22,7 +22,7 @@ def get_latest_timestamp(self, symbol: str, provider: str) -> datetime | None: Args: symbol: Symbol identifier (e.g., "AAPL", "BTC-USD") - provider: Data provider name (e.g., "yahoo", "polygon") + provider: Data provider name (e.g., "yahoo", "massive") Returns: Latest timestamp in the dataset, or None if no data exists diff --git a/src/ml4t/data/validation/ohlcv.py b/src/ml4t/data/validation/ohlcv.py index 46f28f4..7af048d 100644 --- a/src/ml4t/data/validation/ohlcv.py +++ b/src/ml4t/data/validation/ohlcv.py @@ -2,7 +2,6 @@ import time from typing import Any, Literal -from warnings import warn import polars as pl import structlog @@ -30,7 +29,7 @@ def __init__( self, check_nulls: bool = True, check_price_consistency: bool = True, - negative_price_policy: NegativePricePolicy | bool = "forbid", + negative_price_policy: NegativePricePolicy = "forbid", check_negative_volume: bool = True, check_duplicate_timestamps: bool = True, check_chronological_order: bool = True, @@ -38,8 +37,6 @@ def __init__( check_extreme_returns: bool = True, max_return_threshold: float = 0.5, # 50% return threshold staleness_threshold: int = 5, # Days of identical prices - *, - check_negative_prices: bool | None = None, ) -> None: """ Initialize OHLCV validator with configurable checks. @@ -55,24 +52,11 @@ def __init__( check_extreme_returns: Check for extreme price returns max_return_threshold: Threshold for extreme returns (as fraction) staleness_threshold: Days of identical prices to flag as stale - check_negative_prices: Deprecated boolean alias for negative_price_policy Numeric-column and finite-value checks are structural and always run. """ self.check_nulls = check_nulls self.check_price_consistency = check_price_consistency - legacy_policy = negative_price_policy if isinstance(negative_price_policy, bool) else None - if check_negative_prices is not None and legacy_policy is not None: - raise ValueError("Specify check_negative_prices only once") - if check_negative_prices is not None: - legacy_policy = check_negative_prices - if legacy_policy is not None: - warn( - "check_negative_prices is deprecated; use negative_price_policy", - DeprecationWarning, - stacklevel=2, - ) - negative_price_policy = "forbid" if legacy_policy else "allow" if negative_price_policy not in {"forbid", "warn", "allow"}: raise ValueError("negative_price_policy must be 'forbid', 'warn', or 'allow'") self.negative_price_policy = negative_price_policy diff --git a/src/ml4t/data/validation/rules.py b/src/ml4t/data/validation/rules.py index 03e5305..a5dccdb 100644 --- a/src/ml4t/data/validation/rules.py +++ b/src/ml4t/data/validation/rules.py @@ -4,11 +4,10 @@ from dataclasses import dataclass, field from pathlib import Path -from warnings import warn import structlog import yaml -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field from ml4t.data.assets.asset_class import AssetClass from ml4t.data.validation.ohlcv import NegativePricePolicy @@ -54,28 +53,6 @@ class ValidationRuleConfig(BaseModel): model_config = ConfigDict(use_enum_values=True, extra="forbid") - @model_validator(mode="before") - @classmethod - def migrate_legacy_negative_price_flag(cls, data: object) -> object: - """Map persisted boolean rules without silently reversing their behavior.""" - if not isinstance(data, dict) or "check_negative_prices" not in data: - return data - migrated = dict(data) - legacy_value = migrated.pop("check_negative_prices") - if not isinstance(legacy_value, bool): - raise ValueError("check_negative_prices must be a boolean") - if "negative_price_policy" in migrated: - raise ValueError( - "Use either deprecated check_negative_prices or negative_price_policy, not both" - ) - warn( - "check_negative_prices is deprecated; use negative_price_policy", - DeprecationWarning, - stacklevel=2, - ) - migrated["negative_price_policy"] = "forbid" if legacy_value else "allow" - return migrated - @dataclass class ValidationRuleSet: diff --git a/tests/conftest.py b/tests/conftest.py index c8a76fb..77243de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ # Set TESTING environment variable for all tests os.environ["TESTING"] = "true" -NETWORK_MARKERS = ("integration", "real_api", "requires_api_key", "paid_tier") +NETWORK_MARKERS = ("integration", "requires_api_key", "paid_tier") LOCAL_HOSTNAMES = {"localhost", "localhost.localdomain", "ip6-localhost"} # Configure structlog for tests without format_exc_info to avoid warnings diff --git a/tests/integration/test_polygon.py b/tests/integration/test_polygon.py index 9a1e214..beaf0cc 100644 --- a/tests/integration/test_polygon.py +++ b/tests/integration/test_polygon.py @@ -1,10 +1,10 @@ -"""Integration tests for Massive/Polygon provider (real API calls). +"""Integration tests for the Massive provider (real API calls). These tests verify the Massive provider works correctly with actual API calls. Requirements: - MASSIVE_API_KEY or POLYGON_API_KEY environment variable must be set - - Free tier: 5 requests/minute (tests respect rate limits) + - An account plan that permits the requested datasets - API key from: https://massive.com/ Test Coverage: diff --git a/tests/storage/test_metadata_tracker.py b/tests/storage/test_metadata_tracker.py index df3e457..ac90b16 100644 --- a/tests/storage/test_metadata_tracker.py +++ b/tests/storage/test_metadata_tracker.py @@ -281,7 +281,7 @@ def test_dataset_metadata_from_dict(self) -> None: "symbol": "GOOGL", "asset_class": "equities", "frequency": "daily", - "provider": "polygon", + "provider": "massive", "first_update": "2024-01-01T00:00:00", "last_update": "2024-01-15T00:00:00", "total_rows": 500, @@ -297,7 +297,7 @@ def test_dataset_metadata_from_dict(self) -> None: assert metadata.symbol == "GOOGL" assert metadata.asset_class == "equities" - assert metadata.provider == "polygon" + assert metadata.provider == "massive" assert metadata.total_rows == 500 assert metadata.update_count == 10 assert metadata.last_check == datetime(2024, 1, 15, 12, 0, 0) diff --git a/tests/test_data_manager_unit.py b/tests/test_data_manager_unit.py index b9a8988..bc276aa 100644 --- a/tests/test_data_manager_unit.py +++ b/tests/test_data_manager_unit.py @@ -181,13 +181,13 @@ def test_provider_manager_detects_massive_from_massive_key(self): assert manager.is_available("massive") assert not manager.is_available("polygon") - def test_provider_manager_detects_legacy_polygon_alias(self): - """Test POLYGON_API_KEY keeps both massive and polygon provider names available.""" + def test_legacy_polygon_key_does_not_restore_removed_provider_name(self): + """The former key still configures Massive without restoring its removed alias.""" with patch.dict("os.environ", {"POLYGON_API_KEY": "polygon_key"}, clear=True): manager = ProviderManager({"providers": {}}) assert manager.is_available("massive") - assert manager.is_available("polygon") + assert not manager.is_available("polygon") class TestDataManagerValidateDates: diff --git a/tests/test_distribution_verification.py b/tests/test_distribution_verification.py index a4f6a6a..16608de 100644 --- a/tests/test_distribution_verification.py +++ b/tests/test_distribution_verification.py @@ -37,7 +37,7 @@ def _write_distributions( "Project-URL: Documentation, https://www.ml4trading.io/docs/data/", "Project-URL: Repository, https://github.com/ml4t/data", "Project-URL: Issues, https://github.com/ml4t/data/issues", - "Project-URL: Changelog, https://github.com/ml4t/data/blob/main/CHANGELOG.md", + "Project-URL: Changelog, https://github.com/ml4t/data/releases", ] if license_expression is not None: metadata_fields.append(f"License-Expression: {license_expression}") diff --git a/tests/test_polygon_provider.py b/tests/test_polygon_provider.py index 3ac16f7..4c2f75e 100644 --- a/tests/test_polygon_provider.py +++ b/tests/test_polygon_provider.py @@ -8,6 +8,7 @@ import pytest from tenacity import wait_none +import ml4t.data.providers as providers from ml4t.data.core.exceptions import ( AuthenticationError, DataValidationError, @@ -16,12 +17,16 @@ RateLimitError, SymbolNotFoundError, ) -from ml4t.data.providers.polygon import MassiveProvider, PolygonProvider +from ml4t.data.providers.polygon import MassiveProvider class TestMassiveProviderInit: """Tests for canonical Massive provider initialization.""" + def test_removed_polygon_provider_is_not_exported(self): + """Only the canonical provider class is public.""" + assert not hasattr(providers, "PolygonProvider") + def test_init_with_api_key(self): """Test initialization with API key.""" provider = MassiveProvider(api_key="test_key") @@ -62,51 +67,6 @@ def test_custom_base_url(self): assert provider.base_url == "https://api.polygon.io" -class TestPolygonProviderInit: - """Tests for provider initialization.""" - - def test_init_with_api_key(self): - """Test initialization with API key.""" - with pytest.warns(DeprecationWarning, match="PolygonProvider is deprecated"): - provider = PolygonProvider(api_key="test_key") - - assert provider.name == "polygon" - assert provider.api_key == "test_key" - assert provider.base_url == "https://api.massive.com" - - def test_init_with_env_api_key(self): - """Test initialization with API key from environment.""" - with patch.dict("os.environ", {"POLYGON_API_KEY": "env_key"}, clear=True): - with pytest.warns(DeprecationWarning, match="PolygonProvider is deprecated"): - provider = PolygonProvider() - - assert provider.api_key == "env_key" - - def test_init_without_api_key_raises_error(self): - """Test initialization without API key raises AuthenticationError.""" - with patch.dict("os.environ", {}, clear=True): - with pytest.raises(AuthenticationError, match="API key required"): - PolygonProvider() - - def test_init_custom_rate_limit(self): - """Test initialization with custom rate limit.""" - with pytest.warns(DeprecationWarning, match="PolygonProvider is deprecated"): - provider = PolygonProvider(api_key="test_key", rate_limit=(100, 60.0)) - - # Provider should be initialized successfully - assert provider is not None - - -class TestNameProperty: - """Tests for name property.""" - - def test_name_returns_polygon(self): - """Test name property returns correct value.""" - with pytest.warns(DeprecationWarning, match="PolygonProvider is deprecated"): - provider = PolygonProvider(api_key="test_key") - assert provider.name == "polygon" - - class TestFetchRawData: """Tests for _fetch_raw_data method.""" diff --git a/tests/test_synthetic_provider.py b/tests/test_synthetic_provider.py index 11b5cd0..556488e 100644 --- a/tests/test_synthetic_provider.py +++ b/tests/test_synthetic_provider.py @@ -4,6 +4,7 @@ with proper relationships and statistical properties. """ +import inspect import json import os import subprocess @@ -18,6 +19,11 @@ from ml4t.data.synthetic import get_bars_per_day, get_periods_per_year +def test_removed_garch_omega_parameter_is_not_in_signature(): + """The 0.2 API no longer accepts the ignored compatibility parameter.""" + assert "garch_omega" not in inspect.signature(SyntheticProvider).parameters + + class TestSyntheticProviderBasics: """Basic functionality tests.""" @@ -30,12 +36,6 @@ def test_provider_name(self, provider): """Test provider name is correct.""" assert provider.name == "synthetic" - def test_legacy_garch_omega_is_accepted_with_deprecation(self): - with pytest.warns(DeprecationWarning, match="garch_omega"): - provider = SyntheticProvider(model="garch", garch_omega=0.000002, seed=42) - - assert provider.model == "garch" - def test_invalid_calendar_mode_is_rejected(self): with pytest.raises(ValueError, match="calendar_mode"): SyntheticProvider(calendar_mode="weekdays") diff --git a/tests/test_validation_rules.py b/tests/test_validation_rules.py index 9dc6be3..fb0e4d1 100644 --- a/tests/test_validation_rules.py +++ b/tests/test_validation_rules.py @@ -77,16 +77,15 @@ def test_custom_config(self): assert config.check_price_consistency is True assert config.staleness_threshold == 5 - def test_legacy_negative_price_flag_is_migrated(self): - with pytest.warns(DeprecationWarning, match="check_negative_prices"): - config = ValidationRuleConfig(check_negative_prices=False) - - assert config.negative_price_policy == "allow" - def test_unknown_persisted_key_is_rejected(self): with pytest.raises(ValidationError, match="unknown_rule"): ValidationRuleConfig(unknown_rule=True) + def test_removed_negative_price_boolean_is_rejected(self): + """Saved configuration must use the explicit policy field.""" + with pytest.raises(ValidationError, match="check_negative_prices"): + ValidationRuleConfig(check_negative_prices=True) + @pytest.mark.parametrize("asset_class", ["commodity", "commodities", "future", "futures"]) def test_derivative_aliases_use_negative_price_warning_policy(self, asset_class): config = ValidationRulePresets.for_asset_class(asset_class) diff --git a/tests/validation/test_ohlcv_validator.py b/tests/validation/test_ohlcv_validator.py index 9d499a7..16d51bf 100644 --- a/tests/validation/test_ohlcv_validator.py +++ b/tests/validation/test_ohlcv_validator.py @@ -7,6 +7,7 @@ - Configuration options """ +import inspect from datetime import UTC, datetime, timedelta import polars as pl @@ -42,6 +43,10 @@ def create_valid_ohlcv_df( class TestOHLCVValidatorInitialization: """Test validator initialization and configuration.""" + def test_removed_negative_price_boolean_is_not_in_signature(self): + """The 0.2 API exposes only the explicit policy.""" + assert "check_negative_prices" not in inspect.signature(OHLCVValidator).parameters + def test_default_initialization(self): """All checks enabled by default.""" validator = OHLCVValidator() @@ -1097,13 +1102,6 @@ def test_invalid_negative_price_policy_is_rejected(): OHLCVValidator(negative_price_policy="sometimes") -def test_legacy_negative_price_flag_is_mapped_with_deprecation(): - with pytest.warns(DeprecationWarning, match="check_negative_prices"): - validator = OHLCVValidator(check_negative_prices=False) - - assert validator.negative_price_policy == "allow" - - class TestEdgeCases: """Test edge cases and boundary conditions."""