From bdd4761b119bd4d0787deb0ebf5a512e6a300ab1 Mon Sep 17 00:00:00 2001 From: igerber Date: Fri, 7 Aug 2026 12:07:54 -0400 Subject: [PATCH] feat(v4): n_bootstrap validation sweep + fail-closed inference selector (2(d) PR-B, M-081/M-096) M-081: a shared utils.validate_n_bootstrap (promoted verbatim from ChangesInChanges' local validator - non-negative int, numpy integers accepted, bool/None/float/negative rejected) now runs at __init__ for CallawaySantAnna, SunAbraham, EfficientDiD, ImputationDiD, TwoStageDiD, WooldridgeDiD, ContinuousDiD, StaggeredTripleDifference and the DiD family (DiD's __init__; MPD/TWFE inherit); CiC/QDiD re-point to the shared helper with a byte-identical message. 0 stays legal and still means bootstrap off on every > 0-gated analytical lane; no numeric defaults change. Named exception (ledger + design doc): HAD keeps its >= 1 floor - its n_bootstrap powers only the optional sup-t band whose off-switch is fit(cband=False). M-096: the inference selector fails closed. The accepted set is exactly {"analytical", "wild_bootstrap"}, string-typed (an isinstance guard rejects the one-element-ndarray hole in bare tuple membership), at __init__ and transactional set_params. At fit - placed AFTER the survey and Conley front doors so their NotImplementedError rejections keep precedence - DiD with wild_bootstrap and no cluster= raises ValueError (previously a SILENT analytical fallback; the pinned test test_did_wild_bootstrap_requires_cluster flips BY DESIGN), and DiD/TWFE with n_bootstrap < 2 raise (n_bootstrap in {0, 1} ran WCR with too few draws and returned a wild-labeled all-NaN inference tuple; the < 2 floor amends the locked < 1, recorded in v4-design section 7 + the M-096 notes in this diff). TWFE's unit auto-cluster stays; MPD's warn-and-analytical-fallback stays (n_bootstrap-independent). Fixed: DiD/TWFE never cleared per-fit bootstrap state, so a wild fit followed by set_params(inference="analytical") + refit reported stale inference_method="wild_bootstrap" + n_bootstrap/n_clusters/p_val_type metadata. Both now reset _bootstrap_results at the top of fit(). Ledger: M-081 + M-096 planned -> done with test_ref tests/test_v4_inference_policy.py (126 tests: the validation sweep with rollback atomicity, the selector value-set incl. a non-string probe, cluster/floor coherence with full-message pins, boundary n=2 acceptance with a finite quintet, front-door precedence at sub-floor counts on both estimators, the MPD carve-out at n=0/1/999, refit transitions, and the dynamic roster guard pinning inference exposure to exactly {DiD, MPD, TWFE}). ContinuousDiD enrolled in the BAD_VALUES rollback lane (pre-existing catalog gap). Docs: REGISTRY WCR fail-closed Note + MPD wild rows corrected (SE-summary table + section bullet); troubleshooting/choosing_estimator examples gain the cluster= prerequisite (plus the adjacent weight_type= -> bootstrap_weights= kwarg fix); README/llms.txt/llms-practitioner normative wild recommendations qualified DiD-vs-TWFE and the two CS-adjacent advice strings reworded; llms-full DiD block annotated; a systematic wild-mention disposition sweep recorded. The wild-bootstrap cluster-count guidance is harmonized to the single 50-cluster convention (choosing_estimator.rst was the <30 outlier). doc-deps gains troubleshooting.rst + llms-practitioner.txt under estimators.py; TODO row for the type-blind n_bootstrap holes in already-validated estimators (HAD/dCDH/TROP/SyntheticDiD); DEFERRED MPD anchor corrected. --- CHANGELOG.md | 59 +++- DEFERRED.md | 2 +- README.md | 2 +- TODO.md | 1 + diff_diff/changes_in_changes.py | 10 +- diff_diff/continuous_did.py | 3 +- diff_diff/efficient_did.py | 3 +- diff_diff/estimators.py | 49 ++- diff_diff/guides/llms-full.txt | 4 +- diff_diff/guides/llms-practitioner.txt | 9 +- diff_diff/guides/llms.txt | 2 +- diff_diff/imputation.py | 7 +- diff_diff/staggered.py | 3 +- diff_diff/staggered_triple_diff.py | 3 +- diff_diff/sun_abraham.py | 2 + diff_diff/twfe.py | 16 + diff_diff/two_stage.py | 3 +- diff_diff/utils.py | 19 ++ diff_diff/wooldridge.py | 2 + docs/choosing_estimator.rst | 11 +- docs/doc-deps.yaml | 6 + docs/methodology/REGISTRY.md | 21 +- docs/troubleshooting.rst | 6 +- docs/v4-deprecations.yaml | 12 +- docs/v4-design.md | 102 +++--- tests/test_base_estimator.py | 1 + tests/test_estimators_vcov_type.py | 4 +- tests/test_v4_inference_policy.py | 437 +++++++++++++++++++++++++ tests/test_wild_bootstrap.py | 21 +- 29 files changed, 727 insertions(+), 93 deletions(-) create mode 100644 tests/test_v4_inference_policy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 401b92b2..a913433b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 "TwoStageDiD" (previously "TwoStageDiD (Gardner)"); the BusinessReport/DiagnosticReport Bacon caveats name classes likewise. +### Changed +- **`n_bootstrap` is validated and the `inference` selector fails closed** + (v4 program 2(d) PR-B; ledger rows [M-081] + [M-096] → `done`). + (a) A shared `validate_n_bootstrap` (promoted verbatim from + ChangesInChanges' local validator: non-negative integer, numpy integers + accepted, bool/None/float/negative rejected) now runs at `__init__` for + CallawaySantAnna, SunAbraham, EfficientDiD, ImputationDiD, TwoStageDiD, + WooldridgeDiD, ContinuousDiD, StaggeredTripleDifference and the DiD + family (DifferenceInDifferences; MultiPeriodDiD/TwoWayFixedEffects + inherit) - values that previously meant silent-bootstrap-off (negatives) + or latent breakage (floats/bools/None) now raise; `0` stays legal and + still means bootstrap off on every `> 0`-gated analytical lane. No + numeric defaults changed. (b) `inference=` accepts exactly + `{"analytical", "wild_bootstrap"}` (string-typed) at construction and + transactional `set_params` - unknown or non-string values raise instead + of silently running analytical inference. (c) At fit, + `DifferenceInDifferences` with `inference="wild_bootstrap"` and no + `cluster=` now raises `ValueError` where it previously fell back to + analytical SILENTLY - the pinned test + `test_did_wild_bootstrap_requires_cluster` flipped BY DESIGN to assert + the raise. (d) DiD/TWFE with `wild_bootstrap` and `n_bootstrap < 2` + raise at fit, closing the `n_bootstrap ∈ {0, 1}` states that ran WCR + with too few draws and returned a wild-labeled all-NaN inference tuple + with no warning. TWFE's unit auto-cluster still satisfies the cluster + prerequisite; the survey and Conley rejections keep precedence; + MultiPeriodDiD's warn-and-analytical-fallback is unchanged (and + n_bootstrap-independent). Emitted-guidance surfaces + (`docs/troubleshooting.rst`, `docs/choosing_estimator.rst`, `README.md`, + the bundled `llms*.txt` guides) now state the `cluster=` prerequisite + where they recommend wild bootstrap for DiD. + ### Added - **HeterogeneousAdoptionDiD post-fit `aggregate()` + panel-shape mode inference, and the per-level bootstrap-gate convergence** (v4 program 2(b) @@ -891,14 +922,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 lands while `__version__` is still 3.8.x) - the `M-091`/`M-092` pattern. `M-096` covers the `inference="wild_bootstrap"` selector contract, which governs wild CLUSTER bootstrap offered as an alternative to analytical SEs - - a roster of `DifferenceInDifferences` + `TwoWayFixedEffects`. Two things the - audit established. (i) The selector does **not** fail closed today: - `inference=` is stored with no valid-value check anywhere, and DiD routes - `wild_bootstrap` without `cluster=` silently to analytical (a currently - *pinned* behavior), so a typo or a missing prerequisite quietly changes which - SE/p-value/CI procedure runs. Phase 2 must validate the accepted value set on - `__init__` and transactional `set_params` and handle the missing-cluster case - explicitly. (ii) The apparent `inference=` vs `n_bootstrap>0` split is not + a SUPPORT roster of `DifferenceInDifferences` + `TwoWayFixedEffects` (the + selector *param* is additionally exposed by `MultiPeriodDiD`, which has no + wild path and falls back - support ≠ exposure). Two things the + audit established. (i) The selector did **not** fail closed at audit time: + `inference=` was stored with no valid-value check anywhere, and DiD routed + `wild_bootstrap` without `cluster=` silently to analytical (a then-*pinned* + behavior), so a typo or a missing prerequisite quietly changed which + SE/p-value/CI procedure ran. SHIPPED later in this release (the 2(d) PR-B + `### Changed` entry above): the accepted value set is validated on + `__init__` and transactional `set_params`, and the missing-cluster case + raises at fit. (ii) The apparent `inference=` vs `n_bootstrap>0` split is not drift - estimators whose bootstrap *is* their inference method run materially different procedures (CallawaySantAnna an influence-function multiplier bootstrap; SunAbraham a unit-pairs bootstrap, Rao-Wu rescaled on survey @@ -1339,6 +1373,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 default-flip and warning-retirement sweeps). Repo-internal documentation + test only - **no public API or numerical behavior change.** +### Fixed +- **Stale wild-bootstrap metadata on refits**: `DifferenceInDifferences` + and `TwoWayFixedEffects` never cleared their per-fit bootstrap state, so + a wild-bootstrap fit followed by `set_params(inference="analytical")` + and a refit reported `inference_method="wild_bootstrap"` plus stale + `n_bootstrap`/`n_clusters`/`p_val_type` on an analytically-inferred + result. Both estimators now reset the state at the top of `fit()`; + refit transitions label inference from the current fit only. + ## [3.8.0] - 2026-07-18 ### Added diff --git a/DEFERRED.md b/DEFERRED.md index 9144a69c..bccde399 100644 --- a/DEFERRED.md +++ b/DEFERRED.md @@ -104,7 +104,7 @@ For survey-specific limitations (`NotImplementedError` paths), see the | `SpilloverDiD` estimator-level end-to-end vcov reconstruction tests (`bread @ meat @ bread` against `res.vcov`): requires exposing the estimator's internal `X_2_kept` design arrays; the surface is currently pinned from different angles (uniform-weight bit-identity, drift goldens, manual lincom reconstruction at rtol=1e-6). | `spillover.py`, `tests/test_spillover.py` | follow-up | Low | | `SpilloverDiD` Wave E.3 `finite_mask + design-subset` hygiene not yet adopted for TwoStageDiD's analogous pattern (`two_stage.py:567-601`) — separate parity follow-up noted in the `docs/api/spillover.rst` Restrictions block. | `two_stage.py` | Wave E.3 | Low | | `HeterogeneousAdoptionDiD` `covariates=` (Theorem 6 multivariate-covariate extension) not implemented — `fit(covariates=...)` raises `NotImplementedError` via the shipped future-work trap (locked by the `test_had.py` / `test_methodology_had.py` L73 tests); the deferred work is the Theorem 6 extension itself. | `had.py` | Phase 2a | Low | -| MultiPeriodDiD wild bootstrap not supported (falls back to analytical) — user-facing edge-case limitation. | `estimators.py:1647` | — | Low | +| MultiPeriodDiD wild bootstrap not supported (falls back to analytical, n_bootstrap-independent) — user-facing edge-case limitation; the 4.0 removal replaces the fallback with a raise (v4-design §4.1). | `estimators.py:1574` | — | Low | | `predict()` raises `NotImplementedError` — rarely needed; user-facing limitation. | `estimators.py:890-911` | — | Low | ## Version-gated (v4) diff --git a/README.md b/README.md index a020d47d..13d54b33 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ For rigorous DiD analysis, follow these 8 steps. Skipping diagnostic steps produ 2. **State identification assumptions** - which parallel trends variant (unconditional, conditional, PT-GT-Nev, PT-GT-NYT), no-anticipation, overlap. 3. **Test parallel trends** - simple 2x2: `check_parallel_trends()`, `equivalence_test_trends()`; staggered: inspect CS event-study pre-period coefficients (generic PT tests are invalid for staggered designs). Insignificant pre-trends do NOT prove PT holds. 4. **Choose estimator** - staggered adoption -> CS/SA/BJS (NOT plain TWFE); few treated units -> SDiD; factor confounding -> TROP; simple 2x2 -> DiD. Run `BaconDecomposition` to diagnose TWFE bias. -5. **Estimate** - `estimator.fit(data, ...)`. Always print the cluster count first and choose inference method based on the result (cluster-robust if >= 50 clusters, wild bootstrap if fewer). +5. **Estimate** - `estimator.fit(data, ...)`. Always print the cluster count first and choose inference method based on the result (cluster-robust if >= 50 clusters, wild bootstrap if fewer - for DifferenceInDifferences pass `cluster=`; TwoWayFixedEffects auto-clusters at unit level). 6. **Sensitivity analysis** - `compute_honest_did(results)` for bounds under PT violations (MultiPeriodDiD, CS, or dCDH natively; a StackedDiD `results.aggregate('event_study')` container also admits - needs `kappa_pre >= 2`), `run_all_placebo_tests()` for 2x2 falsification, specification comparisons for staggered designs. 7. **Heterogeneity** - CS: `results.aggregate('group')`/`'event_study'` (post-fit, no refit); SA: `results.event_study_effects` / `to_dataframe(level='cohort')`; Stacked: `results.aggregate('event_study')`/`'simple'` post-fit views (surface always computed since 3.9); EDiD: `results.aggregate(...)` post-fit from retained EIFs (3.9); ImputationDiD/TwoStageDiD: `results.aggregate(...)` post-fit from panel-backed kits (3.9); ContinuousDiD: `results.aggregate('dose'/'simple'/'event_study')` post-fit (3.9; dose/simple are views, event_study recomputes); subgroup re-estimation. 8. **Robustness** - compare 2-3 estimators (CS vs SA vs BJS), report with and without covariates (shows whether conditioning drives identification), present pre-trends and sensitivity bounds. diff --git a/TODO.md b/TODO.md index a80b4a1d..4f86755c 100644 --- a/TODO.md +++ b/TODO.md @@ -65,6 +65,7 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | Issue | Location | Origin | Effort | Priority | |-------|----------|--------|--------|----------| +| Type-blind `n_bootstrap` acceptance in already-validated estimators - HAD bool (`isinstance(..., int)` passes `True`, runs as 1 replicate), dCDH bool+float (its bare `< 0` check passes both `True` and `2.5`), TROP float (`2.5` passes the `>= 2` floor), SyntheticDiD float under all three variance methods + bool/negative under jackknife (its floor check is skipped there) - align these local checks with the `utils.validate_n_bootstrap` type guard (M-081 kept them out of the sweep: it scoped to previously-UNvalidated estimators only) | `diff_diff/had.py`, `diff_diff/chaisemartin_dhaultfoeuille.py`, `diff_diff/trop.py`, `diff_diff/synthetic_did.py` | 2(d) PR-B | Quick | Low | | M-020-era CS fit-time `aggregate=` teachings persist in troubleshooting.rst (:215/:241/:244) and choosing_estimator.rst (:243) - CS examples still fit with the deprecated kwarg; migrate to post-fit `results.aggregate('event_study')` (the two HAD examples in the same file were migrated with M-027) | `docs/troubleshooting.rst` | 2(b) PR-4 | Quick | Low | | Evaluate adding the `BaseEstimator` param surface (get_params/set_params) to the exported classes that never had it - `PowerAnalysis`, `LinearRegression`, `BusinessReport`, `DiagnosticReport`, `TWFEWeightsResult` (a NEW public surface, deliberately out of the 2(c)-i pure-refactor scope; `LinearRegression` is the one `fit`-bearing class excluded from the contract suite's roster-completeness test). | `diff_diff/linalg.py`, `diff_diff/power.py` | mixin PR | Mid | Low | | Tighten the mypy suppressions that back the enforced-zero posture: burn down `prep_dgp`'s per-module `[index]` override (needs a None-vs-array restructure that preserves the seeded RNG stream), and evaluate re-enabling the globally disabled codes (`arg-type`, `return-value`, `var-annotated`, `assignment`) one at a time — `assignment` alone hid several real annotation drifts found during the 2026-07 triage. | `pyproject.toml` `[tool.mypy]`, `diff_diff/prep_dgp.py` | lint-CI | Mid | Low | diff --git a/diff_diff/changes_in_changes.py b/diff_diff/changes_in_changes.py index 6ba8e4ac..6345c2c8 100644 --- a/diff_diff/changes_in_changes.py +++ b/diff_diff/changes_in_changes.py @@ -61,6 +61,7 @@ safe_inference_batch, validate_binary, validate_covariate_names, + validate_n_bootstrap, ) # Default quantile grid: qte's ``probs = seq(0.05, 0.95, 0.05)`` (19 points), pinned to @@ -1132,13 +1133,6 @@ def _validate_quantiles(quantiles: Optional[Any]) -> None: raise ValueError(f"quantiles must be finite and strictly inside (0, 1), got '{quantiles}'") -def _validate_n_bootstrap(n_bootstrap: Any) -> None: - if isinstance(n_bootstrap, bool) or not isinstance(n_bootstrap, (int, np.integer)): - raise ValueError(f"n_bootstrap must be a non-negative integer, got '{n_bootstrap}'") - if n_bootstrap < 0: - raise ValueError(f"n_bootstrap must be a non-negative integer, got '{n_bootstrap}'") - - def _validate_alpha(alpha: Any) -> None: if not isinstance(alpha, (int, float, np.floating)) or isinstance(alpha, bool): raise ValueError(f"alpha must be a float strictly between 0 and 1, got '{alpha}'") @@ -1161,7 +1155,7 @@ def _validate_seed(seed: Any) -> None: def _validate_all_params(params: Dict[str, Any]) -> None: """Validate the full hyperparameter dict (used by __init__, set_params, and fit).""" _validate_quantiles(params["quantiles"]) - _validate_n_bootstrap(params["n_bootstrap"]) + validate_n_bootstrap(params["n_bootstrap"]) _validate_alpha(params["alpha"]) _validate_panel(params["panel"]) _validate_seed(params["seed"]) diff --git a/diff_diff/continuous_did.py b/diff_diff/continuous_did.py index 38284cc3..2521db8c 100644 --- a/diff_diff/continuous_did.py +++ b/diff_diff/continuous_did.py @@ -45,7 +45,7 @@ build_unit_first_row_index, compute_survey_vcov, ) -from diff_diff.utils import safe_inference +from diff_diff.utils import safe_inference, validate_n_bootstrap if TYPE_CHECKING: from diff_diff.survey import ResolvedSurveyDesign, SurveyDesign @@ -286,6 +286,7 @@ def __init__( self.anticipation = anticipation self.base_period = base_period self.alpha = alpha + validate_n_bootstrap(n_bootstrap) self.n_bootstrap = n_bootstrap self.bootstrap_weights = bootstrap_weights self.seed = seed diff --git a/diff_diff/efficient_did.py b/diff_diff/efficient_did.py index 2edd7b85..8aab3f72 100644 --- a/diff_diff/efficient_did.py +++ b/diff_diff/efficient_did.py @@ -64,7 +64,7 @@ compute_omega_star_nocov, enumerate_valid_triples, ) -from diff_diff.utils import safe_inference +from diff_diff.utils import safe_inference, validate_n_bootstrap # Re-export for convenience __all__ = ["EfficientDiD", "EfficientDiDResults", "EDiDBootstrapResults"] @@ -359,6 +359,7 @@ def __init__( self.cluster = cluster self.vcov_type = vcov_type self.control_group = control_group + validate_n_bootstrap(n_bootstrap) self.n_bootstrap = n_bootstrap self.bootstrap_weights = bootstrap_weights self.seed = seed diff --git a/diff_diff/estimators.py b/diff_diff/estimators.py index 6ed8f1ee..64d97ce0 100644 --- a/diff_diff/estimators.py +++ b/diff_diff/estimators.py @@ -48,9 +48,17 @@ validate_covariate_names, validate_design_term_names, validate_df_convention, + validate_n_bootstrap, wild_bootstrap_se, ) +# Accepted values for the `inference` selector (M-096). Exposed by exactly +# DifferenceInDifferences, MultiPeriodDiD and TwoWayFixedEffects (the two +# subclasses inherit this __init__); the fail-closed check lives in +# DifferenceInDifferences.__init__ and set_params inherits it via the +# BaseEstimator probe re-init. +_INFERENCE_METHODS = ("analytical", "wild_bootstrap") + class DifferenceInDifferences(BaseEstimator): """ @@ -108,9 +116,16 @@ class DifferenceInDifferences(BaseEstimator): inference : str, default="analytical" Inference method: "analytical" for standard asymptotic inference, or "wild_bootstrap" for wild cluster bootstrap (recommended when - number of clusters is small, <50). + number of clusters is small, <50). Exactly these two (string) + values are accepted; anything else raises ``ValueError`` at + construction. ``"wild_bootstrap"`` requires ``cluster=`` — a fit + without it raises ``ValueError`` (since 3.9; previously it fell + back to analytical inference silently). n_bootstrap : int, default=999 Number of bootstrap replications when inference="wild_bootstrap". + Must be a non-negative integer; ``>= 2`` is required when + ``inference="wild_bootstrap"`` (0 or 1 replications cannot produce + bootstrap inference — the fit raises ``ValueError``). bootstrap_weights : str, default="rademacher" Type of bootstrap weights: "rademacher" (standard), "webb" (recommended for <10 clusters), or "mammen" (skewness correction). @@ -233,6 +248,13 @@ def __init__( from diff_diff.linalg import resolve_vcov_type validate_df_convention(df_convention) + validate_n_bootstrap(n_bootstrap) + # Fail-closed inference selector (M-096): an unrecognized or + # non-string value must never silently route to analytical. The + # isinstance guard matters — bare tuple membership admits a + # one-element ndarray via elementwise __eq__. + if not isinstance(inference, str) or inference not in _INFERENCE_METHODS: + raise ValueError(f"inference must be one of {_INFERENCE_METHODS}, got {inference!r}") # `robust` is deprecated (rows M-045..M-047; removed in 4.0). None is # the not-supplied sentinel: default constructions and get_params @@ -380,6 +402,11 @@ def fit( ) # Body-local name; the public parameter is post (M-030). time = post + # Per-fit bootstrap state: cleared up front so the result builder + # labels inference from THIS fit only. Without the reset, a wild fit + # followed by set_params(inference="analytical") + refit reported + # stale inference_method="wild_bootstrap" + bootstrap metadata. + self._bootstrap_results = None # Parse formula if provided if formula is not None: outcome, treatment, time, covariates = self._parse_formula(formula, data) @@ -513,6 +540,26 @@ def fit( cluster=self.cluster, ) + # Fail-closed wild-bootstrap coherence (M-096). Placed AFTER the + # survey and Conley front doors so their NotImplementedError + # rejections keep precedence (raising "pass cluster=" on a + # wild+Conley fit would be contradictory guidance — Conley rejects + # the combination regardless of cluster). + if self.inference == "wild_bootstrap": + if self.cluster is None: + raise ValueError( + "inference='wild_bootstrap' requires cluster=. The wild cluster " + "bootstrap resamples at the cluster level; pass cluster= or use " + "inference='analytical'." + ) + if self.n_bootstrap < 2: + raise ValueError( + f"inference='wild_bootstrap' requires n_bootstrap >= 2 " + f"(got {self.n_bootstrap}). At least 2 replications are needed " + f"for bootstrap inference; use inference='analytical' for " + f"analytical SEs." + ) + if absorb: # FWL theorem: demean ALL regressors alongside outcome. # Regressors collinear with absorbed FE (e.g., treatment after diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 5a907c5b..e92e5a9b 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -55,8 +55,8 @@ DifferenceInDifferences( vcov_type: str | None = None, # Variance family: "hc1" (default), "classical", "hc2", "hc2_bm", "conley" cluster: str | None = None, # Column for cluster-robust SEs alpha: float = 0.05, # Significance level - inference: str = "analytical", # "analytical" or "wild_bootstrap" - n_bootstrap: int = 999, # Bootstrap replications (if inference="wild_bootstrap") + inference: str = "analytical", # "analytical" or "wild_bootstrap" (wild_bootstrap requires cluster=) + n_bootstrap: int = 999, # Bootstrap replications; >= 2 under inference="wild_bootstrap" bootstrap_weights: str = "rademacher", # "rademacher", "webb", or "mammen" seed: int | None = None, # Random seed rank_deficient_action: str = "warn", # "warn", "error", or "silent" diff --git a/diff_diff/guides/llms-practitioner.txt b/diff_diff/guides/llms-practitioner.txt index d432d6ee..3ee8cea9 100644 --- a/diff_diff/guides/llms-practitioner.txt +++ b/diff_diff/guides/llms-practitioner.txt @@ -13,7 +13,9 @@ > Step 2), to ensure AI agents execute it as a distinct action. > - **Sources of uncertainty** (paper's Step 4) are folded into Step 5 > (Estimate) with an explicit cluster-count check directive: >= 50 clusters -> for asymptotic SEs, otherwise wild bootstrap. The 50-cluster threshold is +> for asymptotic SEs, otherwise wild bootstrap where the estimator supports +> it (DiD with `cluster=`; TWFE auto-clusters) or a bootstrapped +> `n_bootstrap` otherwise. The 50-cluster threshold is > a diff-diff convention. > - **Step 8** is "Robustness & Reporting" (compare estimators, report with > and without covariates). The paper's Step 8 is "Keep learning." The @@ -268,7 +270,8 @@ print(f"Number of clusters: {n_clusters}") if n_clusters >= 50: print("-> Use cluster-robust SEs (asymptotic approximation is reliable)") else: - print(f"-> Only {n_clusters} clusters — use wild cluster bootstrap") + print(f"-> Only {n_clusters} clusters — use wild cluster bootstrap " + f"(DiD/TWFE: inference='wild_bootstrap'; CS/SA: a bootstrapped n_bootstrap)") ``` Now run the estimator chosen in Step 4. Examples for common designs: @@ -634,7 +637,7 @@ print(bacon_result.summary()) # Step 5: Estimate (cluster at county level — treatment assignment unit) n_clusters = data['countyreal'].nunique() -print(f"Clusters: {n_clusters} -> {'cluster-robust SEs' if n_clusters >= 50 else 'wild bootstrap'}") +print(f"Clusters: {n_clusters} -> {'cluster-robust SEs' if n_clusters >= 50 else 'few clusters — CS below uses cluster-robust SEs; a bootstrapped n_bootstrap is the CS small-G option'}") cs = CallawaySantAnna( control_group='never_treated', estimation_method='dr', cluster='countyreal', diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index 0dbe5ab9..502a20e9 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -19,7 +19,7 @@ diagnostic steps produces unreliable results. 2. **State identification assumptions** — which parallel trends variant (unconditional, conditional, PT-GT-Nev, PT-GT-NYT), no-anticipation, overlap. 3. **Test parallel trends** — simple 2x2: `check_parallel_trends()`, `equivalence_test_trends()`; staggered: inspect CS event-study pre-period coefficients (generic PT tests are invalid for staggered designs). Insignificant pre-trends do NOT prove PT holds. 4. **Choose estimator** — staggered adoption → CS/SA/BJS (NOT plain TWFE); few treated units → SDiD; factor confounding → TROP; simple 2x2 → DiD. Run `BaconDecomposition` to diagnose TWFE bias. -5. **Estimate** — `estimator.fit(data, ...)`. Always print the cluster count first and choose inference method based on the result (cluster-robust if >= 50 clusters, wild bootstrap if fewer). +5. **Estimate** — `estimator.fit(data, ...)`. Always print the cluster count first and choose inference method based on the result (cluster-robust if >= 50 clusters, wild bootstrap if fewer — for DifferenceInDifferences pass `cluster=`; TwoWayFixedEffects auto-clusters at unit level). 6. **Sensitivity analysis** — `compute_honest_did(results)` for bounds under PT violations (MultiPeriodDiD, CS, or dCDH natively; a StackedDiD `results.aggregate('event_study')` container also admits - needs `kappa_pre >= 2` so estimated pre-periods exist), `run_all_placebo_tests()` for 2x2 falsification, specification comparisons for staggered designs. 7. **Heterogeneity** — CS: `results.aggregate('group')`/`.aggregate('event_study')` post-fit, no refit (fit-time `aggregate=`/`balance_e=` are deprecated since 3.9, removed in 4.0; `compute_honest_did` / `compute_pretrends_power` / `plot_event_study` all accept the post-fit `results.aggregate('event_study')` container directly; EXCEPTION: on a BOOTSTRAPPED CS fit the recompute levels `'event_study'`/`'group'` raise while `.aggregate('simple')` relays the stored bootstrap inference (NaN df column); use the fit-time aggregation for a bootstrapped event-study surface); dCDH: `results.aggregate('event_study')`/`.aggregate('simple')` post-fit views (bootstrap fits included — pure views); SA: `results.event_study_effects`/`to_dataframe(level='cohort')`; StackedDiD: `results.aggregate('event_study')`/`.aggregate('simple')` post-fit views (the surface is ALWAYS computed at fit since 3.9 - row M-024 - and the container admits into `compute_honest_did`/`compute_pretrends_power` with `kappa_pre >= 2`); EDiD: `results.aggregate('event_study')`/`.aggregate('group')`/`.aggregate('simple')` post-fit, RECOMPUTED from retained EIFs (3.9, row M-023; fit-time `aggregate=`/`balance_e=` deprecated; on bootstrapped EDiD fits the recompute levels raise while `.aggregate('simple')` relays the stored bootstrap inference - use the fit-time aggregation for a bootstrapped ES/group surface; EDiD containers are NOT admitted into honest/pretrends - no joint ES covariance); BJS/TwoStageDiD: `results.aggregate('event_study')`/`.aggregate('group')`/`.aggregate('simple')` post-fit on ImputationDiD and TwoStageDiD too (3.9, rows M-021/M-022; recomputed from panel-backed kits, `balance_e=` on `aggregate('event_study')`; on bootstrapped fits the recompute levels raise while `.aggregate('simple')` relays the stored bootstrap inference - use the deprecated fit-time aggregation for a bootstrapped ES/group surface; their containers are not admitted into honest/pretrends - Imputation by design, TwoStage deferred pending a normalization derivation); CGBS continuous: ContinuousDiD is a MIXED adopter (3.9, row M-025) - `results.aggregate('dose')` (ATT(d)+ACRT(d) rows) and `.aggregate('simple')` (att+acrt rows) are views over the always-computed curves and work on ANY fit incl. bootstrapped, while `.aggregate('event_study')` recomputes the binarized event study from a pruned per-cell IF kit and raises on bootstrapped fits (use the deprecated fit-time `aggregate='eventstudy'` there until 4.0; its container is not admitted into honest/pretrends - no joint ES covariance and no reference normalization); HAD: `results.aggregate('simple')` (overall two-period fits; the target column carries the WAS estimand label) / `.aggregate('event_study')` (multi-period fits) - pure views, work on any fit (3.9, rows M-027/M-139; fit() selects the mode from the panel shape; HAD containers are not admitted into honest/pretrends - no joint cross-horizon covariance, deferred); subgroup re-estimation. 8. **Robustness** — compare 2-3 estimators (CS vs SA vs BJS), MUST report with and without covariates (shows whether conditioning drives identification), present pre-trends and sensitivity bounds. diff --git a/diff_diff/imputation.py b/diff_diff/imputation.py index 2b44f7dd..a1db4999 100644 --- a/diff_diff/imputation.py +++ b/diff_diff/imputation.py @@ -49,6 +49,7 @@ from diff_diff.utils import ( safe_inference, validate_df_convention, + validate_n_bootstrap, ) if TYPE_CHECKING: @@ -237,6 +238,7 @@ def __init__( self.alpha = alpha self.cluster = cluster self.vcov_type = vcov_type + validate_n_bootstrap(n_bootstrap) self.n_bootstrap = n_bootstrap self.bootstrap_weights = bootstrap_weights self.seed = seed @@ -384,8 +386,9 @@ def fit( # rejects fit-time ES and the post-fit path fails closed too) AND # (the deprecated fit-time ES/all was supplied OR n_bootstrap <= 0 — # a bootstrapped fit builds no ES surface and post-fit aggregate() - # fails closed on it; <= 0 because n_bootstrap is unvalidated and - # every bootstrap gate is `> 0`). Reachability-BASED, not exact: a + # fails closed on it; validate_n_bootstrap rejects negatives at + # __init__, so 0 is the only reachable off value and `<= 0` is + # equivalent to `== 0` — kept as-is, no behavior change). Reachability-BASED, not exact: a # fit whose bootstrap later FAILS (bootstrap_results=None) can still # aggregate post-fit, so that corner warns spuriously — the warning # fires before the bootstrap runs and cannot know. (The post-fit diff --git a/diff_diff/staggered.py b/diff_diff/staggered.py index 20e32f8a..82d37747 100644 --- a/diff_diff/staggered.py +++ b/diff_diff/staggered.py @@ -38,7 +38,7 @@ CallawaySantAnnaResults, GroupTimeEffect, ) -from diff_diff.utils import safe_inference, safe_inference_batch +from diff_diff.utils import safe_inference, safe_inference_batch, validate_n_bootstrap if TYPE_CHECKING: from diff_diff.survey import SurveyDesign @@ -613,6 +613,7 @@ def __init__( # narrow contract makes the flag a no-op today but consistency # avoids surprises if the contract ever broadens). self._vcov_type_explicit = vcov_type != "hc1" + validate_n_bootstrap(n_bootstrap) self.n_bootstrap = n_bootstrap self.bootstrap_weights = bootstrap_weights self.seed = seed diff --git a/diff_diff/staggered_triple_diff.py b/diff_diff/staggered_triple_diff.py index 107ffac0..d60641bc 100644 --- a/diff_diff/staggered_triple_diff.py +++ b/diff_diff/staggered_triple_diff.py @@ -28,7 +28,7 @@ CallawaySantAnnaBootstrapMixin, ) from diff_diff.staggered_triple_diff_results import StaggeredTripleDiffResults -from diff_diff.utils import safe_inference +from diff_diff.utils import safe_inference, validate_n_bootstrap if TYPE_CHECKING: from diff_diff.survey import SurveyDesign @@ -150,6 +150,7 @@ def __init__( self.alpha = alpha self.anticipation = anticipation self.base_period = base_period + validate_n_bootstrap(n_bootstrap) self.n_bootstrap = n_bootstrap self.bootstrap_weights = bootstrap_weights self.seed = seed diff --git a/diff_diff/sun_abraham.py b/diff_diff/sun_abraham.py index cf1acf4c..dd473153 100644 --- a/diff_diff/sun_abraham.py +++ b/diff_diff/sun_abraham.py @@ -43,6 +43,7 @@ safe_inference, snap_absorbed_regressors, validate_df_convention, + validate_n_bootstrap, ) from diff_diff.utils import ( within_transform as _within_transform_util, @@ -729,6 +730,7 @@ def __init__( self.anticipation = anticipation self.alpha = alpha self.cluster = cluster + validate_n_bootstrap(n_bootstrap) self.n_bootstrap = n_bootstrap self.seed = seed self.rank_deficient_action = rank_deficient_action diff --git a/diff_diff/twfe.py b/diff_diff/twfe.py index 1963bbd0..96be91c9 100644 --- a/diff_diff/twfe.py +++ b/diff_diff/twfe.py @@ -180,6 +180,10 @@ def fit( # type: ignore[override] If a covariate name collides with a reserved structural term name or duplicates another covariate. """ + # Per-fit bootstrap state: cleared up front so the result builder + # labels inference from THIS fit only (see the matching reset in + # DifferenceInDifferences.fit). + self._bootstrap_results = None # Validate unit column exists if unit not in data.columns: raise ValueError(f"Unit column '{unit}' not found in data") @@ -273,6 +277,18 @@ def fit( # type: ignore[override] if _replicate_vcov_remap_twfe: use_full_dummy = False + # Fail-closed wild-bootstrap floor (M-096), after BOTH front doors + # (Conley + survey resolution) so their rejections keep precedence. + # No cluster-required check for TWFE: cluster=None auto-clusters at + # unit under wild bootstrap (see the class docstring). + if self.inference == "wild_bootstrap" and self.n_bootstrap < 2: + raise ValueError( + f"inference='wild_bootstrap' requires n_bootstrap >= 2 " + f"(got {self.n_bootstrap}). At least 2 replications are needed " + f"for bootstrap inference; use inference='analytical' for " + f"analytical SEs." + ) + # Unit-level clustering is the TWFE default when `cluster` is not # explicitly provided. But the one-way ``classical`` and ``hc2`` # families are by construction not cluster-robust and the validator diff --git a/diff_diff/two_stage.py b/diff_diff/two_stage.py index fb92557f..b60e179c 100644 --- a/diff_diff/two_stage.py +++ b/diff_diff/two_stage.py @@ -51,7 +51,7 @@ TwoStageBootstrapResults, # noqa: F401 TwoStageDiDResults, ) # noqa: F401 (re-export) -from diff_diff.utils import safe_inference +from diff_diff.utils import safe_inference, validate_n_bootstrap if TYPE_CHECKING: # Forward reference for the Wave E.1 survey-design path. Imported under @@ -1343,6 +1343,7 @@ def __init__( self.alpha = alpha self.cluster = cluster self.vcov_type = vcov_type + validate_n_bootstrap(n_bootstrap) self.n_bootstrap = n_bootstrap self.bootstrap_weights = bootstrap_weights self.seed = seed diff --git a/diff_diff/utils.py b/diff_diff/utils.py index cf729f89..60652494 100644 --- a/diff_diff/utils.py +++ b/diff_diff/utils.py @@ -482,6 +482,25 @@ def validate_df_convention(value: str) -> None: raise ValueError(f"df_convention must be one of {_DF_CONVENTIONS}, got {value!r}") +def validate_n_bootstrap(n_bootstrap: Any) -> None: + """Raise ValueError unless ``n_bootstrap`` is a non-negative integer. + + Shared by every estimator constructor whose ``n_bootstrap`` gates an + optional bootstrap (promoted from ChangesInChanges' local validator; + accepts numpy integers, rejects bool/None/float/negative). ``0`` means + bootstrap off wherever a ``> 0`` gate exists — the zero-default + estimators (CallawaySantAnna, SunAbraham, EfficientDiD, ImputationDiD, + TwoStageDiD, WooldridgeDiD, ContinuousDiD, StaggeredTripleDifference) + and every analytical lane. On the DiD/TWFE wild-bootstrap lane 0 never + meant off (the routing consults only ``inference=``); their fit-level + floor rejects ``n_bootstrap < 2`` under ``inference="wild_bootstrap"``. + """ + if isinstance(n_bootstrap, bool) or not isinstance(n_bootstrap, (int, np.integer)): + raise ValueError(f"n_bootstrap must be a non-negative integer, got '{n_bootstrap}'") + if n_bootstrap < 0: + raise ValueError(f"n_bootstrap must be a non-negative integer, got '{n_bootstrap}'") + + def resolve_tail_df( df_convention: str, *, diff --git a/diff_diff/wooldridge.py b/diff_diff/wooldridge.py index fce9415d..cf65f414 100644 --- a/diff_diff/wooldridge.py +++ b/diff_diff/wooldridge.py @@ -37,6 +37,7 @@ safe_inference, snap_absorbed_regressors, validate_df_convention, + validate_n_bootstrap, within_transform, ) from diff_diff.wooldridge_results import WooldridgeDiDResults @@ -1007,6 +1008,7 @@ def __init__( self.demean_covariates = demean_covariates self.alpha = alpha self.cluster = cluster + validate_n_bootstrap(n_bootstrap) self.n_bootstrap = n_bootstrap self.bootstrap_weights = bootstrap_weights self.seed = seed diff --git a/docs/choosing_estimator.rst b/docs/choosing_estimator.rst index d7416879..51ffd18e 100644 --- a/docs/choosing_estimator.rst +++ b/docs/choosing_estimator.rst @@ -684,7 +684,7 @@ differences helps interpret results and choose appropriate inference. - Details * - ``DifferenceInDifferences`` - HC1 (heteroskedasticity-robust) - - Uses White's robust SEs by default. Specify ``cluster`` for cluster-robust SEs. Use ``inference='wild_bootstrap'`` for few clusters (<30). + - Uses White's robust SEs by default. Specify ``cluster`` for cluster-robust SEs. Use ``inference='wild_bootstrap'`` (with ``cluster=`` — required) for few clusters (<50). * - ``TwoWayFixedEffects`` - Cluster-robust (unit level) - Always clusters at unit level after within-transformation. Specify ``cluster`` to override. Use ``inference='wild_bootstrap'`` for few clusters. @@ -733,10 +733,11 @@ differences helps interpret results and choose appropriate inference. **Recommendations by sample size:** -- **Large samples (N > 1000, clusters > 50)**: Default analytical SEs are reliable -- **Medium samples (clusters 30-50)**: Cluster-robust SEs recommended -- **Small samples (clusters < 30)**: Use wild cluster bootstrap (``inference='wild_bootstrap'``) -- **Very few clusters (< 10)**: Use Webb 6-point distribution (``weight_type='webb'``) +- **Large samples (>= 50 clusters)**: Cluster-robust SEs are reliable (the asymptotic + approximation holds; the 50-cluster threshold is the diff-diff convention used + throughout the guides) +- **Small samples (clusters < 50)**: Use wild cluster bootstrap (``inference='wild_bootstrap'`` with ``cluster=`` — ``TwoWayFixedEffects`` auto-clusters at unit level) +- **Very few clusters (< 10)**: Use Webb 6-point distribution (``bootstrap_weights='webb'``) **Common pitfall:** Forgetting to cluster when units are observed multiple times. For panel data, always cluster at the unit level unless you have a strong reason not to. diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index 5a860fde..5cefa35e 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -125,6 +125,12 @@ sources: type: user_guide - path: docs/practitioner_decision_tree.rst type: user_guide + - path: docs/troubleshooting.rst + type: user_guide + note: "Wild-bootstrap examples and inference guidance (cluster= prerequisite, n_bootstrap floor) track the DiD selector contract (M-096)" + - path: diff_diff/guides/llms-practitioner.txt + type: user_guide + note: "Step-5 inference decision rule and small-G advice strings track the DiD/TWFE wild-bootstrap contract (M-096)" diff_diff/twfe.py: drift_risk: low diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 8025f06d..c516f54e 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -73,6 +73,7 @@ where τ is the ATT. - Default: HC1 heteroskedasticity-robust - Optional: Cluster-robust (specify `cluster` parameter) - Optional: Wild cluster bootstrap for small number of clusters + (`inference="wild_bootstrap"` with `cluster=` — required since 3.9, see the WCR Note) - With `absorb=`, the absorbed-FE degrees-of-freedom adjustment uses the component-aware rank (`diff_diff.utils.absorbed_fe_rank`) — see the TwoWayFixedEffects section's absorbed-FE degrees-of-freedom note for the @@ -111,6 +112,17 @@ bootstrap of Cameron, Gelbach & Miller (2008), matching the defaults of R's `TwoWayFixedEffects`. (`MultiPeriodDiD` does **not** support it — it falls back to analytical inference and the inherited `p_val_type` is inert there.) +- **Note:** Since 3.9 the selector fails closed (rows M-081/M-096): `inference=` + accepts exactly `{"analytical", "wild_bootstrap"}` (string-typed) at construction + and `set_params`; `DifferenceInDifferences` with `wild_bootstrap` and no + `cluster=` raises `ValueError` at fit — the code now matches this section's own + "(with `cluster=`)" definition, where it previously fell back to analytical + silently — and DiD/TWFE with `n_bootstrap < 2` under `wild_bootstrap` raise + (0 or 1 replications deterministically produce the all-NaN degenerate tuple). + `TwoWayFixedEffects`' unit auto-cluster satisfies the cluster prerequisite; + the survey and Conley `NotImplementedError` rejections keep precedence. + `MultiPeriodDiD` keeps its warn-and-analytical-fallback until its 4.0 removal. + *Algorithm (test of H₀: τ = r, default r = 0):* 1. **Impose the null** by dropping the interaction column and re-fitting the reduced model; the restricted residuals are `ũ(r) = M₋ⱼ y − r·M₋ⱼ xⱼ` (linear in `r`, where `M₋ⱼ` is the @@ -282,8 +294,9 @@ where V is the VCV sub-matrix for post-treatment δ_e coefficients. BLAS-dependent rounding; the clamp keeps the SE finite (0 for a genuinely-zero variance) and deterministic across BLAS implementations, never `NaN`. No effect on any positive variance. Regression: `tests/test_methodology_wls_cr2.py::TestLinearRegressionFENanGuardEndToEnd`. -- Optional: Wild cluster bootstrap (complex for multi-coefficient testing; - requires joint bootstrap distribution) +- Wild cluster bootstrap: **not supported** — `inference="wild_bootstrap"` warns and + falls back to analytical inference (multi-coefficient testing would require the + joint bootstrap distribution; the 4.0 removal replaces the fallback with a raise) - Degrees of freedom adjusted for absorbed fixed effects: component-aware rank via `diff_diff.utils.absorbed_fe_rank` (own `_absorbed_fe_vcov_scale` gate at its fit site — MultiPeriodDiD is a second implementation, not an alias of @@ -4934,8 +4947,8 @@ should be a deliberate user choice. | Estimator | Default SE | Alternatives | |-----------|-----------|--------------| -| DifferenceInDifferences | HC1 robust | Cluster-robust, wild bootstrap | -| MultiPeriodDiD | HC1 robust | Cluster-robust (via `cluster` param), wild bootstrap | +| DifferenceInDifferences | HC1 robust | Cluster-robust, wild bootstrap (with `cluster=`) | +| MultiPeriodDiD | HC1 robust | Cluster-robust (via `cluster` param); no wild path — `inference="wild_bootstrap"` warns and falls back to analytical | | TwoWayFixedEffects | Cluster at unit | Wild bootstrap | | CallawaySantAnna | Analytical (influence fn) | Multiplier bootstrap | | SunAbraham | Cluster-robust + delta method | Pairs bootstrap | diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index cc617e65..28dd5b03 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -158,7 +158,8 @@ Standard Error Issues .. code-block:: python # Reduce number of bootstrap iterations (default is 999) - did = DifferenceInDifferences(inference='wild_bootstrap', n_bootstrap=499) + did = DifferenceInDifferences(inference='wild_bootstrap', cluster='unit_id', + n_bootstrap=499) # Note: Fewer iterations = less precise p-values # 499 is minimum recommended for publication @@ -290,7 +291,8 @@ Performance Issues unit='unit_id', time='period') # Reduce bootstrap iterations for initial exploration - did = DifferenceInDifferences(inference='wild_bootstrap', n_bootstrap=99) + did = DifferenceInDifferences(inference='wild_bootstrap', cluster='unit_id', + n_bootstrap=99) # For CallawaySantAnna, start without bootstrap cs = CallawaySantAnna() diff --git a/docs/v4-deprecations.yaml b/docs/v4-deprecations.yaml index 3a678c6f..e2b18966 100644 --- a/docs/v4-deprecations.yaml +++ b/docs/v4-deprecations.yaml @@ -1032,10 +1032,11 @@ rows: introduced_in: "3.9" deprecated_in: null removed_in: null - status: planned + status: done phase: 2 - code_refs: [diff_diff/estimators.py, diff_diff/synthetic_did.py, diff_diff/changes_in_changes.py] - notes: "Semantic unification only: n_bootstrap=0 = bootstrap off wherever an analytical path exists; per-estimator counts stay tuned (999 light / 200 compute-heavy). No numeric default changes. introduced_in gates the 3.9 cut (pattern of [M-091]/[M-092]): the unification is a Phase 2 obligation and must not slip past the bump unflipped. deprecated_in stays null deliberately - a flip version would arm the early-flip guard against the Phase 2 PR, which lands while __version__ is still 3.8.x." + code_refs: [diff_diff/estimators.py, diff_diff/synthetic_did.py, diff_diff/changes_in_changes.py, diff_diff/utils.py, diff_diff/staggered.py, diff_diff/sun_abraham.py, diff_diff/efficient_did.py, diff_diff/imputation.py, diff_diff/two_stage.py, diff_diff/wooldridge.py, diff_diff/continuous_did.py, diff_diff/staggered_triple_diff.py] + test_ref: tests/test_v4_inference_policy.py + notes: "Semantic unification only: n_bootstrap=0 = bootstrap off wherever an analytical path exists; per-estimator counts stay tuned (999 light / 200 compute-heavy). No numeric default changes. introduced_in gates the 3.9 cut (pattern of [M-091]/[M-092]): the unification is a Phase 2 obligation and must not slip past the bump unflipped. deprecated_in stays null deliberately - a flip version would arm the early-flip guard against the Phase 2 PR, which lands while __version__ is still 3.8.x. SHIPPED in 2(d) PR-B: a shared utils.validate_n_bootstrap (promoted verbatim from ChangesInChanges' local validator; non-negative int, accepts numpy integers, rejects bool/None/float/negative) is applied at __init__ across the previously-unvalidated roster - DifferenceInDifferences (MultiPeriodDiD/TwoWayFixedEffects inherit), CallawaySantAnna, SunAbraham, EfficientDiD, ImputationDiD, TwoStageDiD, WooldridgeDiD, ContinuousDiD, StaggeredTripleDifference - with CiC/QDiD re-pointed to the shared helper (byte-identical message). SCOPE of 0=off: 0 stays legal at construction and still means bootstrap off on every > 0-gated analytical lane (the eight zero-default sweep classes); under DiD/TWFE inference='wild_bootstrap', where 0 NEVER meant off (the routing consults only the selector - it ran WCR with zero draws and returned a wild-labeled all-NaN inference tuple), 0 and 1 now raise at fit per [M-096]. EXCEPTION: HeterogeneousAdoptionDiD has an analytical pointwise path but deliberately floors n_bootstrap >= 1 (its n_bootstrap powers ONLY the optional sup-t band, whose off-switch is fit(cband=False); n_bootstrap=0 would be a second, ambiguous off-switch) - the 0=off clause does not apply to HAD and no HAD behavior changed. Already-validated estimators (SyntheticDiD >= 2 unless jackknife, TROP >= 2, HAD >= 1, dCDH non-negative) keep their own checks; their type-blind holes are a TODO.md row, not this PR." - id: M-096 kind: behavior group: policy-wild-bootstrap @@ -1044,10 +1045,11 @@ rows: introduced_in: "3.9" deprecated_in: null removed_in: null - status: planned + status: done phase: 2 code_refs: [diff_diff/estimators.py, diff_diff/twfe.py] - notes: "Spec section 7 selector contract + roster guard. inference='wild_bootstrap' selects wild CLUSTER bootstrap offered as an ALTERNATIVE to analytical SEs; the SUPPORTING roster is DifferenceInDifferences + TwoWayFixedEffects. NOT a uniformity mandate over every bootstrap - each estimator whose bootstrap IS its inference method runs a materially different procedure and keeps n_bootstrap as documented domain vocabulary (section 6 precedent): CallawaySantAnna = influence-function multiplier bootstrap; SunAbraham = unit-pairs bootstrap (Rao-Wu rescaled under stratified/PSU survey designs); ChaisemartinDHaultfoeuille = group-level influence-function multiplier bootstrap, upgrading to PSU-level Hall-Mammen wild clustering under survey designs with strictly-coarser PSUs. Spelling any of those 'wild_bootstrap' would make a uniform name carry altered meaning (section 8 rule 8). MultiPeriodDiD EXPOSES the param but has no wild-bootstrap path (warns, falls back to analytical); it is removed at 4.0 [M-010] and section 4.1 replaces the fallback with a raise - so the roster pins SUPPORT, not param presence. SCOPE: the selector does not fail closed today - inference= is stored unvalidated (no valid-value check anywhere), and DiD silently routes wild_bootstrap WITHOUT cluster= to analytical (estimators.py, pinned by tests/test_wild_bootstrap.py::test_did_wild_bootstrap_requires_cluster), so a typo or a missing prerequisite silently changes the SE/p/CI procedure - a no-silent-failures violation. Phase 2 makes it fail closed, and 'fail closed' means REFUSE, not warn-and-degrade: (a) the accepted value set is exactly {'analytical', 'wild_bootstrap'}, validated on __init__ AND transactional set_params (an unrecognized spelling raises, never falls through to analytical); (b) wild_bootstrap WITHOUT cluster= raises ValueError - REGISTRY defines WCR as 'inference=\"wild_bootstrap\" (with cluster=)', so the combination is incoherent rather than merely unsupported, and a warned analytical fallback would still run a procedure the caller did not ask for. Flipping the pinned test is expected and its CHANGELOG entry must say so. done requires a test_ref covering: the exact accepted value set and an invalid spelling, transactional set_params, DiD with and without cluster, TWFE, MultiPeriodDiD's fallback, and the roster itself (a future estimator gaining WCR must adopt the selector or land its own row). Companion to [M-081] (n_bootstrap semantics). introduced_in gates the 3.9 cut for the same reason as [M-081]; behavior-at-done requires test_ref." + test_ref: tests/test_v4_inference_policy.py + notes: "Spec section 7 selector contract + roster guard. inference='wild_bootstrap' selects wild CLUSTER bootstrap offered as an ALTERNATIVE to analytical SEs; the SUPPORTING roster is DifferenceInDifferences + TwoWayFixedEffects. NOT a uniformity mandate over every bootstrap - each estimator whose bootstrap IS its inference method runs a materially different procedure and keeps n_bootstrap as documented domain vocabulary (section 6 precedent): CallawaySantAnna = influence-function multiplier bootstrap; SunAbraham = unit-pairs bootstrap (Rao-Wu rescaled under stratified/PSU survey designs); ChaisemartinDHaultfoeuille = group-level influence-function multiplier bootstrap, upgrading to PSU-level Hall-Mammen wild clustering under survey designs with strictly-coarser PSUs. Spelling any of those 'wild_bootstrap' would make a uniform name carry altered meaning (section 8 rule 8). MultiPeriodDiD EXPOSES the param but has no wild-bootstrap path (warns, falls back to analytical); it is removed at 4.0 [M-010] and section 4.1 replaces the fallback with a raise - so the roster pins SUPPORT, not param presence. SCOPE (pre-3.9 state, closed by this row): the selector did not fail closed - inference= was stored unvalidated, and DiD silently routed wild_bootstrap WITHOUT cluster= to analytical (formerly pinned by tests/test_wild_bootstrap.py::test_did_wild_bootstrap_requires_cluster, flipped BY DESIGN in the shipping PR with a CHANGELOG disclosure), so a typo or a missing prerequisite silently changed the SE/p/CI procedure - a no-silent-failures violation. SHIPPED in 2(d) PR-B, fail closed = REFUSE, not warn-and-degrade: (a) the accepted value set is exactly {'analytical', 'wild_bootstrap'}, string-typed (an isinstance guard rejects non-string values that pass bare tuple membership, e.g. a one-element ndarray), validated on __init__ AND transactional set_params; (b) wild_bootstrap WITHOUT cluster= raises ValueError at fit - REGISTRY defines WCR as 'inference=\"wild_bootstrap\" (with cluster=)', so the combination is incoherent rather than merely unsupported; (c) AMENDED FLOOR (user-approved 2026-08-07, deviation recorded in v4-design section 7 in the same diff): wild with n_bootstrap < 2 raises at fit - not the originally-locked < 1 - because n_bootstrap in {0, 1} deterministically degenerates (the wild routing never consulted n_bootstrap > 0; both values ran WCR with too few draws and returned a wild-labeled all-NaN inference tuple with no warning). The survey and Conley front doors keep precedence (their NotImplementedError rejections fire first); TWFE's unit auto-cluster satisfies the cluster prerequisite; MPD's warn-and-fallback is n_bootstrap-independent and unchanged. Also fixed in the same PR: _bootstrap_results is now reset per-fit, closing stale inference_method/bootstrap metadata on refits after set_params(inference='analytical'). test_ref covers: the exact accepted value set, an invalid spelling and a non-string value, transactional set_params, DiD with and without cluster, the {0,1} floor with message pins, boundary n=2 acceptance with a finite quintet, front-door precedence at sub-floor counts on DiD and TWFE, MPD's fallback at n=0/1/999, refit transitions, and the roster itself (a future estimator gaining WCR must adopt the selector or land its own row). Companion to [M-081] (n_bootstrap semantics). introduced_in gates the 3.9 cut for the same reason as [M-081]; behavior-at-done requires test_ref." - id: M-082 kind: param group: renames-post diff --git a/docs/v4-design.md b/docs/v4-design.md index d7b98f16..784b99d7 100644 --- a/docs/v4-design.md +++ b/docs/v4-design.md @@ -524,35 +524,46 @@ domain vocabulary, not drift. fallback with an explicit raise in the merged event-study mode, so the roster pins estimators that SUPPORT WCR, not those that merely expose the param.) - **The selector must fail closed** - it does not today. `inference=` is stored - without any valid-value check, and DiD routes `wild_bootstrap` WITHOUT - `cluster=` silently to analytical (pinned by - `tests/test_wild_bootstrap.py::test_did_wild_bootstrap_requires_cluster`), so - a typo or a missing prerequisite quietly changes which SE/p-value/CI - procedure runs - the no-silent-failures principle applied to inference - selection. Fail closed means REFUSE, not warn-and-degrade: Phase 2 pins the - accepted set to exactly `{"analytical", "wild_bootstrap"}` on `__init__` and - transactional `set_params`, and makes `wild_bootstrap` without `cluster=` - raise `ValueError`. REGISTRY specifies WCR as `inference="wild_bootstrap"` - *(with `cluster=`)*, so that combination is incoherent rather than merely - unsupported - and a warning still leaves a procedure running that the caller - did not ask for. - - **Locked implementation decisions (2026-08-06, for the 2(d) PR-B).** - The accepted value set `{"analytical", "wild_bootstrap"}` is validated - at `__init__` (transactional `set_params` inherits it via the + **The selector must fail closed** - and since 3.9 (2(d) PR-B, [M-096] + `done`) it does. Before the fix, `inference=` was stored without any + valid-value check, and DiD routed `wild_bootstrap` WITHOUT `cluster=` + silently to analytical (formerly pinned by + `tests/test_wild_bootstrap.py::test_did_wild_bootstrap_requires_cluster`, + flipped BY DESIGN), so a typo or a missing prerequisite quietly changed + which SE/p-value/CI procedure ran - the no-silent-failures principle + applied to inference selection. Fail closed means REFUSE, not + warn-and-degrade: the accepted set is pinned to exactly + `{"analytical", "wild_bootstrap"}` (string-typed) on `__init__` and + transactional `set_params`, and `wild_bootstrap` without `cluster=` + raises `ValueError` at fit. REGISTRY specifies WCR as + `inference="wild_bootstrap"` *(with `cluster=`)*, so that combination is + incoherent rather than merely unsupported - and a warning would still + leave a procedure running that the caller did not ask for. + + **Implementation decisions (locked 2026-08-06; shipped in the 2(d) + PR-B).** The accepted value set `{"analytical", "wild_bootstrap"}` is + validated at `__init__` (transactional `set_params` inherits it via the BaseEstimator probe re-init); the COHERENCE checks run at fit - DiD with `inference="wild_bootstrap"` and no `cluster=` raises `ValueError`, and DiD/TWFE with `wild_bootstrap` and - `n_bootstrap < 1` raise `ValueError`; TWFE's unit auto-cluster stays; - MultiPeriodDiD's warn-and-fallback stays until its 4.0 removal - (DEFERRED.md documents the limitation). The pinned fallback test - `test_did_wild_bootstrap_requires_cluster` flips BY DESIGN (its - CHANGELOG entry must say so). The roster guard is a dynamic sweep: - the set of estimators exposing `inference` in `get_params()` is - exactly {DifferenceInDifferences, MultiPeriodDiD, - TwoWayFixedEffects} - a future estimator gaining WCR must adopt the - selector or land its own row. + `n_bootstrap < 2` raise `ValueError`. **Amendment (2026-08-07, + user-approved, same-diff with the [M-096] ledger notes): the floor is + `< 2`, not the originally-locked `< 1`** - review verified by execution + that `n_bootstrap ∈ {0, 1}` deterministically degenerates (the wild + routing never consulted `n_bootstrap > 0`; both values ran WCR with too + few draws and returned a wild-labeled all-NaN inference tuple with no + warning), and `>= 2` is already the house floor in SyntheticDiD/TROP. + TWFE's unit auto-cluster stays; MultiPeriodDiD's warn-and-fallback + stays until its 4.0 removal (DEFERRED.md documents the limitation; the + fallback is n_bootstrap-independent). The flipped fallback test carries + its CHANGELOG disclosure. The roster guard is a dynamic sweep: the set + of estimators exposing `inference` in `get_params()` is exactly + {DifferenceInDifferences, MultiPeriodDiD, TwoWayFixedEffects} - a + future estimator gaining WCR must adopt the selector or land its own + row. The shipping PR also fixed a latent fit-idempotency bug in the + same class: `_bootstrap_results` is now reset per-fit, so a refit after + `set_params(inference="analytical")` no longer reports stale + `inference_method="wild_bootstrap"` + bootstrap metadata. The apparent `inference=` vs `n_bootstrap>0` split is NOT drift: an estimator whose bootstrap IS its inference method runs a different procedure - @@ -567,17 +578,27 @@ domain vocabulary, not drift. documented domain vocabulary on the section 6 precedent. Should one of them later want an `inference=` selector, the value names its own method - that is additive, post-4.0, minor-version work, not part of this program. -- `n_bootstrap` semantic unification [M-081]: `0` = bootstrap off wherever an - analytical path exists; bootstrap-only estimators document their positive - defaults. Counts stay tuned per estimator (999 light / 200 compute-heavy) - - NO numeric default changes. **Locked implementation decision - (2026-08-06, for the 2(d) PR-B)**: a shared `validate_n_bootstrap` - helper (non-negative int; rejects bool/None/negative; 0 stays legal - wherever it means off) is promoted to utils and applied to EVERY - estimator with a currently-unvalidated `n_bootstrap` - the roster is - GREP-DERIVED at PR-B implementation (known so far: CallawaySantAnna, - SunAbraham, EfficientDiD, ImputationDiD, TwoStageDiD, WooldridgeDiD, - ContinuousDiD, the DiD family, and StaggeredTripleDifference). +- `n_bootstrap` semantic unification [M-081, `done` since 3.9]: `0` = + bootstrap off wherever an analytical path exists; bootstrap-only + estimators document their positive defaults. Counts stay tuned per + estimator (999 light / 200 compute-heavy) - NO numeric default changes. + **Implementation decision (locked 2026-08-06; shipped in the 2(d) + PR-B)**: a shared `validate_n_bootstrap` helper (non-negative int; + accepts numpy integers; rejects bool/None/float/negative; 0 stays legal + wherever it means off) is promoted to utils - verbatim from + ChangesInChanges' local validator - and applied to EVERY estimator with + a previously-unvalidated `n_bootstrap`: CallawaySantAnna, SunAbraham, + EfficientDiD, ImputationDiD, TwoStageDiD, WooldridgeDiD, ContinuousDiD, + the DiD family (DiD's `__init__`, inherited by MPD/TWFE), and + StaggeredTripleDifference; CiC/QDiD re-point to the shared helper. + **Named exception**: HeterogeneousAdoptionDiD has an analytical + pointwise path but deliberately floors `n_bootstrap >= 1` - its + `n_bootstrap` powers ONLY the optional sup-t band, whose off-switch is + `fit(cband=False)`, so `n_bootstrap=0` would be a second, ambiguous + off-switch; the 0=off clause does not apply to HAD and no HAD behavior + changed. On the DiD/TWFE wild lane, 0 never meant off (the routing + consults only the selector) - the [M-096] floor now rejects + `n_bootstrap < 2` there at fit. - **Auto-cluster policy** [M-080], flips at 4.0: every panel estimator (required `unit` column) defaults to clustering at unit (Bertrand-Duflo-Mullainathan practice), setting `cluster_name` / @@ -777,8 +798,11 @@ re-enumerate the cells' M-id lists: `tests/test_v4_wrapper_shims.py`, the module `__getattr__` + `tests/test_aliases.py`, SCM); then PR-B - the two inference-surface policies, `n_bootstrap` semantic unification [M-081] and the - wild-cluster-bootstrap roster guard [M-096] (locked implementation - decisions live in section 7, per this section's boundary rule). + wild-cluster-bootstrap roster guard [M-096] (implementation + decisions live in section 7, per this section's boundary rule) + (shipped: the shared `validate_n_bootstrap` sweep, the fail-closed + selector + `< 2` wild floor + per-fit bootstrap-state reset, and + `tests/test_v4_inference_policy.py`). 7. Phase 3 merges (a)/(b)/(c) per the phase-3 cell. 8. Phase 4: migration guide, the 3.9-cut checklist below, cut. diff --git a/tests/test_base_estimator.py b/tests/test_base_estimator.py index 6cab1d0a..ed9657d5 100644 --- a/tests/test_base_estimator.py +++ b/tests/test_base_estimator.py @@ -85,6 +85,7 @@ def _discover(): "TwoStageDiD": {"vcov_type": "hc4"}, "TripleDifference": {"vcov_type": "hc4"}, "EfficientDiD": {"vcov_type": "hc4"}, + "ContinuousDiD": {"n_bootstrap": -3}, "StackedDiD": {"control_group": "not_a_mode"}, "LPDiD": {"alpha": 5.0}, "ChangesInChanges": {"alpha": 5.0}, diff --git a/tests/test_estimators_vcov_type.py b/tests/test_estimators_vcov_type.py index 67e6bbfe..f719a589 100644 --- a/tests/test_estimators_vcov_type.py +++ b/tests/test_estimators_vcov_type.py @@ -1397,11 +1397,13 @@ def test_wild_bootstrap_preserves_vcov_type_no_error(self): The wild-bootstrap SE comes from resampling, not from the analytical sandwich. `vcov_type` has no effect on the bootstrap SE output, but - the fit should still succeed without errors. + the fit should still succeed without errors. (cluster= added in 3.9: + the fail-closed selector rejects wild bootstrap without it — M-096.) """ data = _make_did_panel(n_units=20) est = DifferenceInDifferences( vcov_type="hc2_bm", + cluster="unit", inference="wild_bootstrap", n_bootstrap=50, seed=42, diff --git a/tests/test_v4_inference_policy.py b/tests/test_v4_inference_policy.py new file mode 100644 index 00000000..e13d7915 --- /dev/null +++ b/tests/test_v4_inference_policy.py @@ -0,0 +1,437 @@ +"""Inference-surface policies (rows M-081 + M-096, 2(d) PR-B). + +This suite is both rows' shared ``test_ref``. + +M-081 (``n_bootstrap`` semantic unification): the shared +``diff_diff.utils.validate_n_bootstrap`` (promoted from ChangesInChanges' +local validator) rejects bool/None/float/negative at ``__init__`` across +every estimator whose ``n_bootstrap`` was previously unvalidated; ``0`` +stays legal at construction and still means bootstrap off on every +``> 0``-gated analytical lane. Transactional ``set_params`` inherits the +validation via the BaseEstimator probe re-init. + +M-096 (fail-closed ``inference`` selector): the accepted value set is +exactly ``("analytical", "wild_bootstrap")`` (string-typed) at +``__init__`` and set_params; at fit, DiD wild-bootstrap without +``cluster=`` raises, and DiD/TWFE wild with ``n_bootstrap < 2`` raise +(``n_bootstrap ∈ {0, 1}`` previously ran WCR with too few draws and +returned a wild-labeled all-NaN inference tuple). The survey and Conley +front doors keep precedence (their ``NotImplementedError`` rejections fire +before the coherence checks). TWFE's unit auto-cluster stays; +MultiPeriodDiD's warn-and-analytical-fallback stays (n_bootstrap- +independent). The roster guard pins ``inference`` exposure in +``get_params()`` to exactly {DifferenceInDifferences, MultiPeriodDiD, +TwoWayFixedEffects}. + +Message pins match the FULL text via ``re.escape`` (repo convention). +""" + +import re + +import numpy as np +import pandas as pd +import pytest + +import diff_diff +from diff_diff import ( + CallawaySantAnna, + ChangesInChanges, + ContinuousDiD, + DifferenceInDifferences, + EfficientDiD, + ImputationDiD, + MultiPeriodDiD, + QDiD, + StaggeredTripleDifference, + SunAbraham, + TwoStageDiD, + TwoWayFixedEffects, + WooldridgeDiD, +) +from diff_diff._base import BaseEstimator +from diff_diff.survey import SurveyDesign +from tests.test_base_estimator import _make + +# --------------------------------------------------------------------------- +# Pinned messages +# --------------------------------------------------------------------------- + +N_BOOTSTRAP_MSG_PREFIX = "n_bootstrap must be a non-negative integer" + +INFERENCE_MSG = "inference must be one of ('analytical', 'wild_bootstrap'), got {value!r}" + +CLUSTER_REQUIRED_MSG = ( + "inference='wild_bootstrap' requires cluster=. The wild cluster " + "bootstrap resamples at the cluster level; pass cluster= or use " + "inference='analytical'." +) + + +def _floor_msg(n: int) -> str: + return ( + f"inference='wild_bootstrap' requires n_bootstrap >= 2 " + f"(got {n}). At least 2 replications are needed " + f"for bootstrap inference; use inference='analytical' for " + f"analytical SEs." + ) + + +MPD_FALLBACK_MSG = ( + "Wild bootstrap inference is not yet supported for MultiPeriodDiD. " + "Using analytical inference instead." +) + +# The M-081 sweep roster (9 previously-unvalidated classes) plus CiC/QDiD, +# whose local validator was the promotion source and now routes through the +# shared helper. +VALIDATED_CLASSES = [ + DifferenceInDifferences, + MultiPeriodDiD, + TwoWayFixedEffects, + CallawaySantAnna, + SunAbraham, + EfficientDiD, + ImputationDiD, + TwoStageDiD, + WooldridgeDiD, + ContinuousDiD, + StaggeredTripleDifference, + ChangesInChanges, + QDiD, +] + +SELECTOR_CLASSES = [DifferenceInDifferences, MultiPeriodDiD, TwoWayFixedEffects] + + +# --------------------------------------------------------------------------- +# DGPs +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def clustered_panel(): + """Two-period clustered DiD panel (8 clusters, healthy effect).""" + rng = np.random.default_rng(0) + n_units, periods = 40, 2 + df = pd.DataFrame( + { + "unit": np.repeat(np.arange(n_units), periods), + "post": np.tile([0, 1], n_units), + "cluster": np.repeat(np.arange(8), periods * 5), + } + ) + df["treated"] = (df["unit"] < 20).astype(int) + df["w"] = 1.0 + df["y"] = 1.0 + 0.5 * df["treated"] * df["post"] + rng.normal(0, 1, len(df)) + return df + + +@pytest.fixture(scope="module") +def conley_panel(clustered_panel): + df = clustered_panel.copy() + rng = np.random.default_rng(1) + df["lat"] = rng.uniform(-30, 30, len(df)) + df["lon"] = rng.uniform(-100, 100, len(df)) + return df + + +@pytest.fixture(scope="module") +def multi_period_panel(): + rng = np.random.default_rng(2) + n_units, periods = 30, 4 + df = pd.DataFrame( + { + "unit": np.repeat(np.arange(n_units), periods), + "time": np.tile(np.arange(periods), n_units), + } + ) + df["treated"] = (df["unit"] < 15).astype(int) + df["y"] = 1.0 + 0.3 * df["treated"] * (df["time"] >= 2) + rng.normal(0, 1, len(df)) + return df + + +def _fit_did(est, df, **kw): + return est.fit(df, outcome="y", treatment="treated", post="post", **kw) + + +def _fit_twfe(est, df, **kw): + return est.fit(df, outcome="y", treatment="treated", time="post", unit="unit", **kw) + + +def _assert_finite_quintet(results): + """The full inference tuple is finite together (never field-by-field).""" + lo, hi = results.conf_int + assert np.all(np.isfinite([results.att, results.se, results.t_stat, results.p_value, lo, hi])) + + +# =========================================================================== +# M-081: shared n_bootstrap validation +# =========================================================================== + + +class TestNBootstrapValidation: + @pytest.mark.parametrize("cls", VALIDATED_CLASSES, ids=lambda c: c.__name__) + @pytest.mark.parametrize("bad", [-3, 1.5, True, None], ids=repr) + def test_bad_value_raises_at_init(self, cls, bad): + with pytest.raises(ValueError, match=re.escape(N_BOOTSTRAP_MSG_PREFIX)): + cls(n_bootstrap=bad) + + def test_bad_value_message_echoes_value(self): + with pytest.raises(ValueError, match=re.escape("got '-3'")): + DifferenceInDifferences(n_bootstrap=-3) + + @pytest.mark.parametrize("cls", VALIDATED_CLASSES, ids=lambda c: c.__name__) + def test_zero_stays_legal_at_construction(self, cls): + est = cls(n_bootstrap=0) + assert est.get_params()["n_bootstrap"] == 0 + + @pytest.mark.parametrize("cls", VALIDATED_CLASSES, ids=lambda c: c.__name__) + def test_numpy_integer_accepted(self, cls): + est = cls(n_bootstrap=np.int64(5)) + assert est.get_params()["n_bootstrap"] == 5 + + @pytest.mark.parametrize("cls", VALIDATED_CLASSES, ids=lambda c: c.__name__) + def test_set_params_rejects_and_rolls_back(self, cls): + est = cls() + before = est.get_params() + with pytest.raises(ValueError, match=re.escape(N_BOOTSTRAP_MSG_PREFIX)): + est.set_params(n_bootstrap=-1) + assert est.get_params() == before + + +# =========================================================================== +# M-096: fail-closed inference selector +# =========================================================================== + + +class TestInferenceSelector: + @pytest.mark.parametrize("cls", SELECTOR_CLASSES, ids=lambda c: c.__name__) + @pytest.mark.parametrize("value", ["analytical", "wild_bootstrap"]) + def test_accepted_values_construct(self, cls, value): + assert cls(inference=value).get_params()["inference"] == value + + @pytest.mark.parametrize("cls", SELECTOR_CLASSES, ids=lambda c: c.__name__) + def test_invalid_spelling_raises_at_init(self, cls): + with pytest.raises(ValueError, match=re.escape(INFERENCE_MSG.format(value="banana"))): + cls(inference="banana") + + @pytest.mark.parametrize("cls", SELECTOR_CLASSES, ids=lambda c: c.__name__) + def test_non_string_value_raises_at_init(self, cls): + # Bare tuple membership admits a one-element ndarray via elementwise + # __eq__; the isinstance guard must reject it. + with pytest.raises(ValueError, match="inference must be one of"): + cls(inference=np.array(["wild_bootstrap"])) + + def test_set_params_rejects_and_rolls_back(self): + did = DifferenceInDifferences() + before = did.get_params() + with pytest.raises(ValueError, match=re.escape(INFERENCE_MSG.format(value="banana"))): + did.set_params(inference="banana") + assert did.get_params() == before + + # -- fit-level coherence: cluster requirement (DiD only) ---------------- + + def test_did_wild_without_cluster_raises(self, clustered_panel): + did = DifferenceInDifferences(inference="wild_bootstrap", n_bootstrap=99, seed=42) + with pytest.raises(ValueError, match=re.escape(CLUSTER_REQUIRED_MSG)): + _fit_did(did, clustered_panel) + + def test_did_wild_with_cluster_fits(self, clustered_panel, ci_params): + n_boot = ci_params.bootstrap(99) + did = DifferenceInDifferences( + inference="wild_bootstrap", cluster="cluster", n_bootstrap=n_boot, seed=42 + ) + results = _fit_did(did, clustered_panel) + assert results.inference_method == "wild_bootstrap" + assert results.n_bootstrap == n_boot + + def test_twfe_auto_cluster_under_wild_fits(self, clustered_panel, ci_params): + # TWFE has NO cluster-required check: cluster=None auto-clusters at + # unit under wild bootstrap. + twfe = TwoWayFixedEffects( + cluster=None, + inference="wild_bootstrap", + n_bootstrap=ci_params.bootstrap(99), + seed=42, + ) + results = _fit_twfe(twfe, clustered_panel) + assert results.inference_method == "wild_bootstrap" + assert np.isfinite(results.se) + + # -- fit-level coherence: the n_bootstrap >= 2 floor -------------------- + + @pytest.mark.parametrize("n", [0, 1]) + def test_did_wild_below_floor_raises(self, clustered_panel, n): + # cluster= supplied so the raise comes from the FLOOR guard, not the + # cluster guard (B2 checks cluster first). + did = DifferenceInDifferences( + inference="wild_bootstrap", cluster="cluster", n_bootstrap=n, seed=42 + ) + with pytest.raises(ValueError, match=re.escape(_floor_msg(n))): + _fit_did(did, clustered_panel) + + @pytest.mark.parametrize("n", [0, 1]) + def test_twfe_wild_below_floor_raises(self, clustered_panel, n): + twfe = TwoWayFixedEffects(inference="wild_bootstrap", n_bootstrap=n, seed=42) + with pytest.raises(ValueError, match=re.escape(_floor_msg(n))): + _fit_twfe(twfe, clustered_panel) + + def test_boundary_n_bootstrap_2_accepted_did(self, clustered_panel): + # Boundary acceptance catches an erroneous `<= 2`. The label alone + # derives from `_bootstrap_results is not None` (which a degenerate + # run also satisfies), so the full quintet must be finite too. + did = DifferenceInDifferences( + inference="wild_bootstrap", cluster="cluster", n_bootstrap=2, seed=42 + ) + results = _fit_did(did, clustered_panel) + assert results.inference_method == "wild_bootstrap" + _assert_finite_quintet(results) + + def test_boundary_n_bootstrap_2_accepted_twfe(self, clustered_panel): + twfe = TwoWayFixedEffects(inference="wild_bootstrap", n_bootstrap=2, seed=42) + results = _fit_twfe(twfe, clustered_panel) + assert results.inference_method == "wild_bootstrap" + _assert_finite_quintet(results) + + # -- negative control: the floor must live inside the wild branch ------- + + def test_did_analytical_with_zero_n_bootstrap_fits(self, clustered_panel): + # M-081 keeps n_bootstrap=0 legal; a guard accidentally hoisted out + # of the `inference == "wild_bootstrap"` block would break this. + results = _fit_did( + DifferenceInDifferences(inference="analytical", n_bootstrap=0), clustered_panel + ) + assert results.inference_method == "analytical" + _assert_finite_quintet(results) + + def test_twfe_analytical_with_zero_n_bootstrap_fits(self, clustered_panel): + results = _fit_twfe( + TwoWayFixedEffects(inference="analytical", n_bootstrap=0), clustered_panel + ) + assert results.inference_method == "analytical" + _assert_finite_quintet(results) + + # -- precedence: survey / Conley front doors fire before the floor ------ + + def test_did_wild_survey_precedence_at_sub_floor_count(self, clustered_panel): + did = DifferenceInDifferences(inference="wild_bootstrap", n_bootstrap=0) + sd = SurveyDesign(weights="w", weight_type="pweight") + with pytest.raises(NotImplementedError, match="Wild bootstrap"): + _fit_did(did, clustered_panel, survey_design=sd) + + def test_twfe_wild_survey_precedence_at_sub_floor_count(self, clustered_panel): + # The survey resolver rejects wild x survey for ANY design BEFORE the + # estimator-level replicate/floor checks (survey.py), so the + # exception TYPE is the pin. + twfe = TwoWayFixedEffects(inference="wild_bootstrap", n_bootstrap=0) + sd = SurveyDesign(weights="w", weight_type="pweight") + with pytest.raises(NotImplementedError, match="Wild bootstrap"): + _fit_twfe(twfe, clustered_panel, survey_design=sd) + + def test_did_wild_conley_precedence_at_sub_floor_count(self, conley_panel): + did = DifferenceInDifferences( + vcov_type="conley", + conley_coords=("lat", "lon"), + conley_cutoff_km=1000.0, + conley_lag_cutoff=0, + inference="wild_bootstrap", + n_bootstrap=1, + ) + with pytest.raises(NotImplementedError, match=r"(?i)wild.bootstrap|conley"): + did.fit(conley_panel, outcome="y", treatment="treated", post="post", unit="unit") + + def test_twfe_wild_conley_precedence_at_sub_floor_count(self, conley_panel): + twfe = TwoWayFixedEffects( + vcov_type="conley", + conley_coords=("lat", "lon"), + conley_cutoff_km=1000.0, + conley_lag_cutoff=0, + inference="wild_bootstrap", + n_bootstrap=1, + ) + with pytest.raises(NotImplementedError, match=r"(?i)wild.bootstrap|conley"): + _fit_twfe(twfe, conley_panel) + + # -- MPD carve-out: warn + analytical fallback, n_bootstrap-independent - + + @pytest.mark.parametrize("n", [0, 1, 999]) + def test_mpd_wild_falls_back_never_raises(self, multi_period_panel, n): + mpd = MultiPeriodDiD(inference="wild_bootstrap", n_bootstrap=n) + with pytest.warns(UserWarning, match=re.escape(MPD_FALLBACK_MSG)): + results = mpd.fit( + multi_period_panel, + outcome="y", + treatment="treated", + time="time", + post_periods=[2, 3], + ) + assert np.isfinite(results.avg_att) + + # -- per-fit bootstrap-state reset (refit transitions) ------------------ + + def test_did_refit_transition_clears_bootstrap_metadata(self, clustered_panel, ci_params): + did = DifferenceInDifferences( + inference="wild_bootstrap", + cluster="cluster", + n_bootstrap=ci_params.bootstrap(99), + seed=42, + ) + wild = _fit_did(did, clustered_panel) + assert wild.inference_method == "wild_bootstrap" + + did.set_params(inference="analytical") + analytical = _fit_did(did, clustered_panel) + # All four metadata fields flow from the one _bootstrap_results + # conditional - pin the full set. + assert analytical.inference_method == "analytical" + assert analytical.n_bootstrap is None + assert analytical.n_clusters is None + assert analytical.p_val_type is None + _assert_finite_quintet(analytical) + + did.set_params(inference="wild_bootstrap") + rewild = _fit_did(did, clustered_panel) + assert rewild.inference_method == "wild_bootstrap" + + def test_twfe_refit_transition_clears_bootstrap_metadata(self, clustered_panel, ci_params): + twfe = TwoWayFixedEffects( + inference="wild_bootstrap", n_bootstrap=ci_params.bootstrap(99), seed=42 + ) + wild = _fit_twfe(twfe, clustered_panel) + assert wild.inference_method == "wild_bootstrap" + + twfe.set_params(inference="analytical") + analytical = _fit_twfe(twfe, clustered_panel) + assert analytical.inference_method == "analytical" + assert analytical.n_bootstrap is None + assert analytical.n_clusters is None + assert analytical.p_val_type is None + + twfe.set_params(inference="wild_bootstrap") + rewild = _fit_twfe(twfe, clustered_panel) + assert rewild.inference_method == "wild_bootstrap" + + +# =========================================================================== +# Roster guard +# =========================================================================== + + +class TestInferenceRoster: + def test_inference_exposed_by_exactly_the_wcr_roster(self): + discovered, seen = [], set() + for name in diff_diff.__all__: + obj = getattr(diff_diff, name) + if not isinstance(obj, type) or id(obj) in seen: + continue + seen.add(id(obj)) + if issubclass(obj, BaseEstimator): + discovered.append(obj) + exposing = {cls for cls in discovered if "inference" in _make(cls).get_params()} + assert exposing == {DifferenceInDifferences, MultiPeriodDiD, TwoWayFixedEffects}, ( + "The `inference` selector roster changed. A future estimator " + "gaining wild cluster bootstrap must adopt the fail-closed " + "selector contract (M-096) or land its own ledger row; one whose " + "bootstrap IS its inference method must keep n_bootstrap as " + "documented domain vocabulary instead (v4-design section 7)." + ) diff --git a/tests/test_wild_bootstrap.py b/tests/test_wild_bootstrap.py index d5c5fa5a..cf81c03d 100644 --- a/tests/test_wild_bootstrap.py +++ b/tests/test_wild_bootstrap.py @@ -5,6 +5,7 @@ """ import json +import re from pathlib import Path import numpy as np @@ -451,16 +452,26 @@ def test_did_wild_bootstrap_with_webb_weights(self, clustered_did_data, ci_param assert results.se > 0 def test_did_wild_bootstrap_requires_cluster(self, clustered_did_data, ci_params): - """Test that wild bootstrap is only used when cluster is specified.""" + """Wild bootstrap without cluster= raises (fail-closed, M-096). + + Flipped BY DESIGN in 3.9: this previously pinned a SILENT fallback + to analytical inference — a no-silent-failures violation the + selector contract closes. The name is now literally true. + """ n_boot = ci_params.bootstrap(99) did = DifferenceInDifferences( inference="wild_bootstrap", n_bootstrap=n_boot, seed=42 # No cluster specified ) - results = did.fit(clustered_did_data, outcome="outcome", treatment="treated", post="post") - - # Should fall back to analytical since no cluster specified - assert results.inference_method == "analytical" + with pytest.raises( + ValueError, + match=re.escape( + "inference='wild_bootstrap' requires cluster=. The wild cluster " + "bootstrap resamples at the cluster level; pass cluster= or use " + "inference='analytical'." + ), + ): + did.fit(clustered_did_data, outcome="outcome", treatment="treated", post="post") def test_twfe_with_wild_bootstrap(self, clustered_did_data, ci_params): """Test TwoWayFixedEffects with wild bootstrap."""