diff --git a/.github/workflows/build-doc.yml b/.github/workflows/build-doc.yml index 952e941..4c8083b 100644 --- a/.github/workflows/build-doc.yml +++ b/.github/workflows/build-doc.yml @@ -29,7 +29,7 @@ jobs: - name: Build documentation run: | cd docs - uv run make html + uv run mkdocs build --strict - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 diff --git a/.gitignore b/.gitignore index c43730a..34edf3e 100644 --- a/.gitignore +++ b/.gitignore @@ -68,7 +68,8 @@ instance/ # Scrapy stuff: .scrapy -# Sphinx documentation +# MkDocs documentation +docs/build/ docs/_build/ # PyBuilder diff --git a/_typos.toml b/_typos.toml index b47387e..df51aa3 100644 --- a/_typos.toml +++ b/_typos.toml @@ -2,7 +2,12 @@ # Allow dataset-specific column names from Hillstrom dataset womens = "womens" mens = "mens" +# Domain term: subjects who comply with treatment assignment +compliers = "compliers" +complier = "complier" [default.extend-identifiers] # Author name in SciPy citation VanderPlas = "VanderPlas" +# numpy function +arange = "arange" diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d0c3cbf..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/docs/api/index.md b/docs/docs/api/index.md new file mode 100644 index 0000000..e5e2711 --- /dev/null +++ b/docs/docs/api/index.md @@ -0,0 +1,29 @@ +# API Reference + +This section provides comprehensive documentation for all classes and functions in the dte_adj package. The API is organized into logical groups based on functionality and use cases. + +## Overview + +The dte_adj package provides several types of estimators for computing distribution treatment effects: + +- **Simple Randomization Estimators**: For estimating distributional effects in simple randomized experiments where treatment assignment is independent of all covariates +- **Covariate Adaptive Randomization Estimators**: For estimating distributional effects under covariate-adaptive randomization (CAR) designs, including stratified block randomization and other adaptive schemes +- **Local Distribution Estimators**: For estimating local distribution treatment effects weighted by treatment propensity within strata +- **Utility Functions**: Helper functions for confidence intervals and statistical computations +- **Plotting Utilities**: Visualization tools for treatment effects and distributions + +For theoretical foundations, see Byambadalai et al. (2024)[^simple2024] for simple randomization, Byambadalai et al. (2025)[^car2025] for covariate-adaptive randomization, and Byambadalai et al. (2025)[^compliance2025] for imperfect compliance scenarios. + +For multi-task learning approaches that train models for all locations simultaneously (using `is_multi_task=True`), see the neural network framework in [^multitask2025]. + +## Detailed Documentation + +- [Simple Randomization Estimators](simple.md) +- [Covariate Adaptive Randomization Estimators](stratified.md) +- [Local Distribution Estimators](local.md) +- [Plotting Utilities](plot.md) + +[^simple2024]: Byambadalai, U., Oka, T., & Yasui, S. (2024). Estimating Distributional Treatment Effects in Randomized Experiments: Machine Learning for Variance Reduction. In Proceedings of the 41st International Conference on Machine Learning (ICML'24). [arXiv:2407.16037](https://arxiv.org/abs/2407.16037). +[^car2025]: Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). On Efficient Estimation of Distributional Treatment Effects under Covariate-Adaptive Randomization. In Proceedings of the 42nd International Conference on Machine Learning (ICML'25). [arXiv:2506.05945](https://arxiv.org/abs/2506.05945). +[^multitask2025]: Hirata, T., Byambadalai, U., Oka, T., Yasui, S., & Uto, S. (2025). Efficient and Scalable Estimation of Distributional Treatment Effects with Multi-Task Neural Networks. arXiv preprint [arXiv:2507.07738](https://arxiv.org/abs/2507.07738). +[^compliance2025]: Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). Beyond the Average: Distributional Causal Inference under Imperfect Compliance. arXiv preprint [arXiv:2509.15594](https://arxiv.org/abs/2509.15594). diff --git a/docs/source/api/local.rst b/docs/docs/api/local.md similarity index 52% rename from docs/source/api/local.rst rename to docs/docs/api/local.md index b6a4d08..a9131d7 100644 --- a/docs/source/api/local.rst +++ b/docs/docs/api/local.md @@ -1,26 +1,15 @@ -Local Distribution Estimators -============================== +# Local Distribution Estimators This page documents local distribution treatment effect estimators that compute treatment effects weighted by treatment propensity within each stratum. These estimators are particularly useful for handling treatment assignment heterogeneity across strata and scenarios with imperfect compliance. Local distribution treatment effects (LDTE) and local probability treatment effects (LPTE) provide methods for causal inference that account for treatment assignment vs. treatment receipt differences. For theoretical foundations on imperfect compliance scenarios, see: -* Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). *Beyond the Average: Distributional Causal Inference under Imperfect Compliance*. `arXiv:2509.15594 `_. +- Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). *Beyond the Average: Distributional Causal Inference under Imperfect Compliance*. [arXiv:2509.15594](https://arxiv.org/abs/2509.15594). -SimpleLocalDistributionEstimator --------------------------------- +## SimpleLocalDistributionEstimator -.. autoclass:: dte_adj.SimpleLocalDistributionEstimator - :members: - :undoc-members: - :show-inheritance: - :no-index: +::: dte_adj.SimpleLocalDistributionEstimator -AdjustedLocalDistributionEstimator ----------------------------------- +## AdjustedLocalDistributionEstimator -.. autoclass:: dte_adj.AdjustedLocalDistributionEstimator - :members: - :undoc-members: - :show-inheritance: - :no-index: +::: dte_adj.AdjustedLocalDistributionEstimator diff --git a/docs/docs/api/plot.md b/docs/docs/api/plot.md new file mode 100644 index 0000000..4267c77 --- /dev/null +++ b/docs/docs/api/plot.md @@ -0,0 +1,5 @@ +# Plotting Utilities + +This page documents plotting utilities for visualizing distribution treatment effects, confidence intervals, and other results. + +::: dte_adj.plot diff --git a/docs/docs/api/simple.md b/docs/docs/api/simple.md new file mode 100644 index 0000000..28deda2 --- /dev/null +++ b/docs/docs/api/simple.md @@ -0,0 +1,23 @@ +# Simple Randomization Estimators + +This page documents estimators that work with simple randomized experimental designs where treatment assignment is completely randomized. + +These estimators leverage pre-treatment covariates through distributional regression frameworks to improve the precision of distributional treatment effect estimates. The key methodological contribution is using machine learning techniques for variance reduction while maintaining validity as long as nuisance components are reasonably well estimated. + +Byambadalai et al. (2024)[^simple2024] propose a regression adjustment method that incorporates covariates into distributional regression, enabling deeper insights beyond average treatment effects by estimating full distributional treatment effects in randomized experiments. + +## SimpleDistributionEstimator + +::: dte_adj.SimpleDistributionEstimator + options: + inherited_members: true + show_inheritance_diagram: true + +## AdjustedDistributionEstimator + +::: dte_adj.AdjustedDistributionEstimator + options: + inherited_members: true + show_inheritance_diagram: true + +[^simple2024]: Byambadalai, U., Oka, T., & Yasui, S. (2024). Estimating Distributional Treatment Effects in Randomized Experiments: Machine Learning for Variance Reduction. arXiv preprint [arXiv:2407.16037](https://arxiv.org/abs/2407.16037). diff --git a/docs/docs/api/stratified.md b/docs/docs/api/stratified.md new file mode 100644 index 0000000..8cca322 --- /dev/null +++ b/docs/docs/api/stratified.md @@ -0,0 +1,21 @@ +# Covariate Adaptive Randomization Estimators + +This page documents estimators that work with stratified experimental designs, particularly for covariate-adaptive randomization (CAR) within strata. + +These estimators are designed to handle stratified block randomization where participants are grouped into strata based on baseline covariates before treatment assignment. The key methodological contribution is leveraging additional covariates beyond strata indicators using machine learning methods to enhance the precision of distributional treatment effect estimates. + +Byambadalai et al. (2025)[^car2025] propose a flexible distribution regression framework that achieves the semiparametric efficiency bound for distributional treatment effects under CAR, demonstrating that regression-adjusted estimators can optimally utilize covariate information in stratified designs. + +## SimpleStratifiedDistributionEstimator + +::: dte_adj.SimpleStratifiedDistributionEstimator + options: + inherited_members: true + +## AdjustedStratifiedDistributionEstimator + +::: dte_adj.AdjustedStratifiedDistributionEstimator + options: + inherited_members: true + +[^car2025]: Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). On Efficient Estimation of Distributional Treatment Effects under Covariate-Adaptive Randomization. arXiv preprint [arXiv:2506.05945](https://arxiv.org/abs/2506.05945). diff --git a/docs/source/_static/distributional_effect.png b/docs/docs/assets/distributional_effect.png similarity index 100% rename from docs/source/_static/distributional_effect.png rename to docs/docs/assets/distributional_effect.png diff --git a/docs/source/_static/dte_empirical.png b/docs/docs/assets/dte_empirical.png similarity index 100% rename from docs/source/_static/dte_empirical.png rename to docs/docs/assets/dte_empirical.png diff --git a/docs/source/_static/dte_moment.png b/docs/docs/assets/dte_moment.png similarity index 100% rename from docs/source/_static/dte_moment.png rename to docs/docs/assets/dte_moment.png diff --git a/docs/source/_static/dte_simple.png b/docs/docs/assets/dte_simple.png similarity index 100% rename from docs/source/_static/dte_simple.png rename to docs/docs/assets/dte_simple.png diff --git a/docs/source/_static/dte_uniform.png b/docs/docs/assets/dte_uniform.png similarity index 100% rename from docs/source/_static/dte_uniform.png rename to docs/docs/assets/dte_uniform.png diff --git a/docs/source/_static/hillstorm_dte.png b/docs/docs/assets/hillstorm_dte.png similarity index 100% rename from docs/source/_static/hillstorm_dte.png rename to docs/docs/assets/hillstorm_dte.png diff --git a/docs/source/_static/hillstorm_dte_control.png b/docs/docs/assets/hillstorm_dte_control.png similarity index 100% rename from docs/source/_static/hillstorm_dte_control.png rename to docs/docs/assets/hillstorm_dte_control.png diff --git a/docs/source/_static/hillstorm_pte.png b/docs/docs/assets/hillstorm_pte.png similarity index 100% rename from docs/source/_static/hillstorm_pte.png rename to docs/docs/assets/hillstorm_pte.png diff --git a/docs/source/_static/hillstorm_pte_control.png b/docs/docs/assets/hillstorm_pte_control.png similarity index 100% rename from docs/source/_static/hillstorm_pte_control.png rename to docs/docs/assets/hillstorm_pte_control.png diff --git a/docs/source/_static/hillstorm_subgroup_men_dte.png b/docs/docs/assets/hillstorm_subgroup_men_dte.png similarity index 100% rename from docs/source/_static/hillstorm_subgroup_men_dte.png rename to docs/docs/assets/hillstorm_subgroup_men_dte.png diff --git a/docs/source/_static/hillstorm_subgroup_men_pte.png b/docs/docs/assets/hillstorm_subgroup_men_pte.png similarity index 100% rename from docs/source/_static/hillstorm_subgroup_men_pte.png rename to docs/docs/assets/hillstorm_subgroup_men_pte.png diff --git a/docs/source/_static/hillstorm_subgroup_women_dte.png b/docs/docs/assets/hillstorm_subgroup_women_dte.png similarity index 100% rename from docs/source/_static/hillstorm_subgroup_women_dte.png rename to docs/docs/assets/hillstorm_subgroup_women_dte.png diff --git a/docs/source/_static/hillstorm_subgroup_women_pte.png b/docs/docs/assets/hillstorm_subgroup_women_pte.png similarity index 100% rename from docs/source/_static/hillstorm_subgroup_women_pte.png rename to docs/docs/assets/hillstorm_subgroup_women_pte.png diff --git a/docs/source/_static/oregon_ldte_costs_comparison.png b/docs/docs/assets/oregon_ldte_costs_comparison.png similarity index 100% rename from docs/source/_static/oregon_ldte_costs_comparison.png rename to docs/docs/assets/oregon_ldte_costs_comparison.png diff --git a/docs/source/_static/oregon_ldte_costs_strata.png b/docs/docs/assets/oregon_ldte_costs_strata.png similarity index 100% rename from docs/source/_static/oregon_ldte_costs_strata.png rename to docs/docs/assets/oregon_ldte_costs_strata.png diff --git a/docs/source/_static/oregon_ldte_visits_comparison.png b/docs/docs/assets/oregon_ldte_visits_comparison.png similarity index 100% rename from docs/source/_static/oregon_ldte_visits_comparison.png rename to docs/docs/assets/oregon_ldte_visits_comparison.png diff --git a/docs/source/_static/oregon_lpte_costs_comparison.png b/docs/docs/assets/oregon_lpte_costs_comparison.png similarity index 100% rename from docs/source/_static/oregon_lpte_costs_comparison.png rename to docs/docs/assets/oregon_lpte_costs_comparison.png diff --git a/docs/source/_static/oregon_lpte_visits_comparison.png b/docs/docs/assets/oregon_lpte_visits_comparison.png similarity index 100% rename from docs/source/_static/oregon_lpte_visits_comparison.png rename to docs/docs/assets/oregon_lpte_visits_comparison.png diff --git a/docs/source/_static/pte_empirical.png b/docs/docs/assets/pte_empirical.png similarity index 100% rename from docs/source/_static/pte_empirical.png rename to docs/docs/assets/pte_empirical.png diff --git a/docs/source/_static/qte.png b/docs/docs/assets/qte.png similarity index 100% rename from docs/source/_static/qte.png rename to docs/docs/assets/qte.png diff --git a/docs/docs/contributing.md b/docs/docs/contributing.md new file mode 100644 index 0000000..f980371 --- /dev/null +++ b/docs/docs/contributing.md @@ -0,0 +1,3 @@ +# Contribution Guide + +Regarding how to contribute to this package, please refer to [CONTRIBUTING.md](https://github.com/CyberAgentAILab/python-dte-adjustment/blob/main/CONTRIBUTING.md) for more details. diff --git a/docs/docs/get_started.md b/docs/docs/get_started.md new file mode 100644 index 0000000..0ece1e1 --- /dev/null +++ b/docs/docs/get_started.md @@ -0,0 +1,156 @@ +# Get Started + +This page contains basic usage of dte_adj library. + +Generate data for training cumulative distribution function: + +```python +import numpy as np + +def generate_data(n, d_x=100, rho=0.5): + """ + Generate data according to the described data generating process (DGP). + + Args: + n (int): Number of samples. + d_x (int): Number of covariates. Default is 100. + rho (float): Success probability for the Bernoulli distribution. Default is 0.5. + + Returns: + X (np.ndarray): Covariates matrix of shape (n, d_x). + D (np.ndarray): Treatment variable array of shape (n,). + Y (np.ndarray): Outcome variable array of shape (n,). + """ + # Generate covariates X from a uniform distribution on (0, 1) + X = np.random.uniform(0, 1, (n, d_x)) + + # Generate treatment variable D from a Bernoulli distribution with success probability rho + D = np.random.binomial(1, rho, n) + + # Define beta_j and gamma_j according to the problem statement + beta = np.zeros(d_x) + gamma = np.zeros(d_x) + + # Set the first 50 values of beta and gamma to 1 + beta[:50] = 1 + gamma[:50] = 1 + + # Compute the outcome Y + U = np.random.normal(0, 1, n) # Error term + linear_term = np.dot(X, beta) + quadratic_term = np.dot(X**2, gamma) + + # Outcome equation + Y = 5 * D + linear_term + quadratic_term + U + + return X, D, Y + +n = 1000 # Sample size +X, D, Y = generate_data(n) +``` + +Then, let's build an empirical cumulative distribution function (CDF). + +```python +import dte_adj +from dte_adj.plot import plot + +estimator = dte_adj.SimpleDistributionEstimator() +estimator.fit(X, D, Y) +locations = np.linspace(Y.min(), Y.max(), 20) +cdf = estimator.predict(1, locations) +``` + +Distributional treatment effect (DTE) can be computed easily in the following code. + +```python +dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="simple") +``` + +A convenience function is available to visualize distribution effects. This method can be used for other distribution parameters including Probability Treatment Effect (PTE) and Quantile Treatment Effect (QTE). + +```python +plot(locations, dte, lower_bound, upper_bound, title="DTE of simple estimator") +``` + +![DTE of empirical estimator](assets/dte_empirical.png) + +To initialize the adjusted distribution function, the base model for conditional distribution function needs to be passed. +In the following example, Logistic Regression is used. Please make sure that your base model implements `fit` and `predict_proba` methods. + +```python +from sklearn.linear_model import LogisticRegression +logit = LogisticRegression() +estimator = dte_adj.AdjustedDistributionEstimator(logit, folds=3) +estimator.fit(X, D, Y) +cdf = estimator.predict(1, locations) +``` + +DTE can be computed and visualized in the following code. + +```python +dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="simple") +plot(locations, dte, lower_bound, upper_bound, title="DTE of adjusted estimator with simple confidence band") +``` + +![DTE of adjusted estimator with simple confidence band](assets/dte_simple.png) + +Confidence bands can be computed in different ways. In the following code, moment condition is used to calculate the confidence band. + +```python +dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="moment") +plot(locations, dte, lower_bound, upper_bound, title="DTE of adjusted estimator with moment confidence band") +``` + +![DTE of adjusted estimator with moment confidence band](assets/dte_moment.png) + +Also, an uniform confidence band is used when `uniform` is specified for the `variance_type` argument. + +```python +dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="uniform") +plot(locations, dte, lower_bound, upper_bound, title="DTE of adjusted estimator with uniform confidence band") +``` + +![DTE of adjusted estimator with uniform confidence band](assets/dte_uniform.png) + +To compute PTE, you can use `predict_pte` method. The `locations` parameter defines interval boundaries, and the method returns probability treatment effects for each interval. +For each interval, the starting point is not included but the ending point is included. For example, if the `locations` is [0, 1, 2], PTE is computed for `(0, 1]` and `(1, 2]`. + +```python +pte, lower_bound, upper_bound = estimator.predict_pte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="simple") +# Note: pte will have shape (len(locations)-1,) since it computes intervals between locations +plot(locations[:-1], pte, lower_bound, upper_bound, chart_type="bar", title="PTE of adjusted estimator with simple confidence band") +``` + +![PTE of adjusted estimator with simple confidence band](assets/pte_empirical.png) + +To compute QTE, you can use `predict_qte` method. The confidence band is computed by bootstrap method. + +```python +quantiles = np.array([0.1 * i for i in range(1, 10)], dtype=np.float32) +qte, lower_bound, upper_bound = estimator.predict_qte(target_treatment_arm=1, control_treatment_arm=0, quantiles=quantiles, n_bootstrap=30) +plot(quantiles, qte, lower_bound, upper_bound, title="QTE of adjusted estimator") +``` + +![QTE of adjusted estimator](assets/qte.png) + +You can use any model with `predict_proba` or `predict` method to adjust the distribution function estimation. +For example, the following code use XGBoost classifier to estimate the conditional distribution. + +```python +import xgboost as xgb +estimator = dte_adj.AdjustedDistributionEstimator(xgb.XGBClassifier(), folds=3) +estimator.fit(X, D, Y) +cdf = estimator.predict(1, locations) +``` + +`predict_dte` and `predict_pte` methods provide an option to train a model for multiple locations simultaneously. +To enable the feature, pass `is_multi_task=True`. + +```python +from sklearn.linear_model import LinearRegression +model = LinearRegression() +estimator = dte_adj.AdjustedDistributionEstimator(model, folds=3) +estimator.fit(X, D, Y) +dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, is_multi_task=True, locations=locations, variance_type="moment") +``` diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 0000000..43c2001 --- /dev/null +++ b/docs/docs/index.md @@ -0,0 +1,36 @@ +# dte_adj + +## A Python Package for Estimating Distribution Treatment Effects + +`dte_adj` is a Python package for estimating distribution treatment effects in randomized experiments. +It provides APIs for conducting regression adjustment to estimate precise distribution functions, enabling deeper insights beyond average treatment effects through machine learning-enhanced estimation methods. + +

+ distributional effect +

+ +## Estimator Types + +The package provides several types of estimators for computing distribution treatment effects: + +- **Simple Randomization Estimators**: For estimating distributional effects in simple randomized experiments where treatment assignment is independent of all covariates +- **Covariate Adaptive Randomization Estimators**: For estimating distributional effects under covariate-adaptive randomization (CAR) designs, including stratified block randomization and other adaptive schemes +- **Local Distribution Estimators**: For estimating local distribution treatment effects weighted by treatment propensity within strata + +## Theoretical Foundations + +For theoretical foundations, see: + +- **Simple randomization**: Byambadalai et al. (2024)[^simple2024] +- **Covariate-adaptive randomization**: Byambadalai et al. (2025)[^car2025] +- **Multi-task learning**: Hirata et al. (2025)[^multitask2025] +- **Imperfect compliance**: Byambadalai et al. (2025)[^compliance2025] + +## License + +MIT License + +[^simple2024]: Byambadalai, U., Oka, T., & Yasui, S. (2024). Estimating Distributional Treatment Effects in Randomized Experiments: Machine Learning for Variance Reduction. In Proceedings of the 41st International Conference on Machine Learning (ICML'24). [arXiv:2407.16037](https://arxiv.org/abs/2407.16037). +[^car2025]: Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). On Efficient Estimation of Distributional Treatment Effects under Covariate-Adaptive Randomization. In Proceedings of the 42nd International Conference on Machine Learning (ICML'25). [arXiv:2506.05945](https://arxiv.org/abs/2506.05945). +[^multitask2025]: Hirata, T., Byambadalai, U., Oka, T., Yasui, S., & Uto, S. (2025). Efficient and Scalable Estimation of Distributional Treatment Effects with Multi-Task Neural Networks. arXiv preprint [arXiv:2507.07738](https://arxiv.org/abs/2507.07738). +[^compliance2025]: Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). Beyond the Average: Distributional Causal Inference under Imperfect Compliance. arXiv preprint [arXiv:2509.15594](https://arxiv.org/abs/2509.15594). diff --git a/docs/docs/installation.md b/docs/docs/installation.md new file mode 100644 index 0000000..5792a70 --- /dev/null +++ b/docs/docs/installation.md @@ -0,0 +1,25 @@ +# Installation Guide + +This package can be installed either through PyPI or source code. + +## Requirement + +You need to use Python version 3.10 or higher to use this package. + +## Install from PyPI + +For installing the package from PyPI, please use the following command. + +```bash +pip install dte_adj +``` + +## Install from source code + +For installing the package from the source code, please use the following commands. + +```bash +git clone https://github.com/CyberAgentAILab/python-dte-adjustment +cd python-dte-adjustment +pip install -e . +``` diff --git a/docs/docs/tutorials/hillstrom.md b/docs/docs/tutorials/hillstrom.md new file mode 100644 index 0000000..d5c2c72 --- /dev/null +++ b/docs/docs/tutorials/hillstrom.md @@ -0,0 +1,623 @@ +# Hillstrom Email Marketing Experiment + +The Hillstrom email marketing dataset is a classic example from digital marketing, involving 64,000 customers randomly assigned to receive either a men's merchandise email, women's merchandise email, or no email (control). This experiment allows us to examine which email campaign strategy is most effective using revenue as the outcome. + +**Background**: Kevin Hillstrom provided this dataset to demonstrate email marketing analytics. Customers who purchased within the last 12 months were randomly divided into three groups to test targeted email campaigns against a control group. + +**Research Question**: Which email campaign performed best: the men's version or the women's version, and how do the effects vary across the revenue distribution? + +### Data Setup and Loading + +```python +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression +from sklearn.preprocessing import LabelEncoder +import dte_adj +from dte_adj.plot import plot + +# Load the real Hillstrom dataset +url = "http://www.minethatdata.com/Kevin_Hillstrom_MineThatData_E-MailAnalytics_DataMiningChallenge_2008.03.20.csv" +df = pd.read_csv(url) + +print(f"Dataset shape: {df.shape}") +print(f"Average spend by segment:\n{df.groupby('segment')['spend'].mean()}") + +# Prepare the data for dte_adj analysis +# Create treatment indicator: 0=No E-Mail, 1=Mens E-Mail, 2=Women E-Mail +treatment_mapping = {'No E-Mail': 0, 'Mens E-Mail': 1, 'Women E-Mail': 2} +D = df['segment'].map(treatment_mapping).values + +# Use spend as the outcome variable (revenue) +revenue = df['spend'].values + +zip_code_mapping = {'Surburban': 0, 'Rural': 1, 'Urban': 2} # Note: typo in original data +channel_mapping = {'Phone': 0, 'Web': 1, 'Multichannel': 2} + +# Create feature matrix +features = pd.DataFrame({ + 'recency': df['recency'], + 'history': df['history'], + 'history_segment': df['history_segment'].map(lambda s: int(s[0])), + 'mens': df['mens'], + 'womens': df['womens'], + 'zip_code': df['zip_code'].map(zip_code_mapping), + 'newbie': df['newbie'], + 'channel': df['channel'].map(channel_mapping) +}) + +X = features.values + +print(f"\nDataset size: {len(D):,} customers") +print(f"Control group (No Email): {(D==0).sum():,} ({(D==0).mean():.1%})") +print(f"Men's Email group: {(D==1).sum():,} ({(D==1).mean():.1%})") +print(f"Women's Email group: {(D==2).sum():,} ({(D==2).mean():.1%})") +print("Average Spend by Treatment:") +print(f"No Email: ${revenue[D==0].mean():.2f}") +print(f"Men's Email: ${revenue[D==1].mean():.2f}") +print(f"Women's Email: ${revenue[D==2].mean():.2f}") + +# Also show conversion rates +print("\nConversion Rates:") +print(f"No Email: {df[df['segment']=='No E-Mail']['conversion'].mean():.3f}") +print(f"Men's Email: {df[df['segment']=='Mens E-Mail']['conversion'].mean():.3f}") +print(f"Women's Email: {df[df['segment']=='Women E-Mail']['conversion'].mean():.3f}") +``` + +### Email Campaign Effectiveness Analysis + +```python +# Initialize estimators +simple_estimator = dte_adj.SimpleDistributionEstimator() +ml_estimator = dte_adj.AdjustedDistributionEstimator( + LinearRegression(), + folds=5 +) + +# Fit estimators on the full dataset +simple_estimator.fit(X, D, revenue) +ml_estimator.fit(X, D, revenue) + +# Define revenue evaluation points +revenue_locations = np.linspace(0, 500, 51) +``` + +### Women's Email vs Control Analysis + +First, let's examine how the Women's email campaign performs compared to no email (control): + +```python +# Compute DTE: Women's email vs Control +dte_women_ctrl, lower_women_ctrl, upper_women_ctrl = simple_estimator.predict_dte( + target_treatment_arm=2, # Women's email + control_treatment_arm=0, # No email control + locations=revenue_locations, + variance_type="moment" +) + +# Visualize Women's vs Control using dte_adj's plot function +plot(revenue_locations, dte_women_ctrl, lower_women_ctrl, upper_women_ctrl, + title="Women's Email Campaign vs Control", + xlabel="Spending ($)", ylabel="Distribution Treatment Effect") +``` + +### Men's Email vs Control Analysis + +Next, let's examine how the Men's email campaign performs compared to no email (control): + +```python +# Compute DTE: Men's email vs Control +dte_men_ctrl, lower_men_ctrl, upper_men_ctrl = simple_estimator.predict_dte( + target_treatment_arm=1, # Men's email + control_treatment_arm=0, # No email control + locations=revenue_locations, + variance_type="moment" +) + +# Visualize Men's vs Control using dte_adj's plot function +plot(revenue_locations, dte_men_ctrl, lower_men_ctrl, upper_men_ctrl, + title="Men's Email Campaign vs Control", + xlabel="Spending ($)", ylabel="Distribution Treatment Effect", color="purple") +``` + +![Hillstrom Email Campaigns vs Control Analysis](../assets/hillstorm_dte_control.png) + +### Spending Category Effects: Each Campaign vs Control + +Let's also examine how each campaign affects spending in specific intervals using Probability Treatment Effects (PTE): + +```python +# Compute PTE: Women's email vs Control +pte_women_ctrl, pte_lower_women_ctrl, pte_upper_women_ctrl = simple_estimator.predict_pte( + target_treatment_arm=2, # Women's email + control_treatment_arm=0, # No email control + locations=np.insert(revenue_locations, 0, -1), + variance_type="moment" +) + +# Compute PTE: Men's email vs Control +pte_men_ctrl, pte_lower_men_ctrl, pte_upper_men_ctrl = simple_estimator.predict_pte( + target_treatment_arm=1, # Men's email + control_treatment_arm=0, # No email control + locations=np.insert(revenue_locations, 0, -1), + variance_type="moment" +) + +# Visualize PTE results using dte_adj's plot function with bar charts side by side +import matplotlib.pyplot as plt + +# Create subplots for side-by-side comparison +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +# Women's vs Control PTE +plot(revenue_locations[1:], pte_women_ctrl, pte_lower_women_ctrl, pte_upper_women_ctrl, + chart_type="bar", + title="Women's Email vs Control", + xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", + ax=ax1) + +# Men's vs Control PTE +plot(revenue_locations[1:], pte_men_ctrl, pte_lower_men_ctrl, pte_upper_men_ctrl, + chart_type="bar", + title="Men's Email vs Control", + xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", + color="purple", ax=ax2) + +plt.tight_layout() +plt.show() +``` + +The side-by-side PTE analysis produces the following visualization: + +![Hillstrom Email Campaigns vs Control PTE Analysis](../assets/hillstorm_pte_control.png) + +These bar charts show how each email campaign affects the probability of customers spending in specific intervals compared to no email: + +**Women's Email vs Control (Left Panel)**: The bar chart reveals specific spending intervals where women's email campaigns increase or decrease customer probability. Positive bars indicate intervals where the campaign increases the likelihood of spending in that range, while negative bars show intervals where it decreases probability. + +**Men's Email vs Control (Right Panel)**: Similarly shows the interval-specific effects of men's email campaigns. The side-by-side comparison allows direct assessment of which campaign is more effective in driving specific spending behaviors. + +**Key Insights from the Hillstrom Email Campaign Analysis**: + +The distributional treatment effects and probability treatment effects reveal several important patterns in how email campaigns affect customer spending behavior: + +1. **Email Campaigns Reduce Zero Spending**: Both men's and women's email campaigns show strong negative effects at the $0 spending level, indicating that email campaigns successfully convert non-purchasers into purchasers. This confirms that email marketing has a clear activation effect. + +2. **Spending Category Redistribution**: The PTE analysis reveals how campaigns redistribute customers across spending intervals. Both campaigns show clear redistribution patterns, with negative effects in the $0 spending category (reducing non-purchase probability) and varying effects across other spending ranges. This redistribution pattern confirms that campaigns shift spending behavior rather than uniformly increasing it across all categories. + +3. **Campaign-Specific Interval Effects**: The side-by-side PTE comparison reveals distinct patterns between campaigns. Women's email campaigns show stronger effects in certain spending intervals (particularly in moderate spending ranges), while men's campaigns display different interval-specific patterns. The visual comparison makes it clear that each campaign has optimal spending ranges where it excels, providing actionable insights for customer segmentation and targeting strategies. + +4. **Statistical Significance Across Intervals**: The confidence intervals in both DTE and PTE analyses reveal that the most reliable effects occur at the zero spending level and in low-to-moderate spending ranges. PTE analysis provides additional granularity by showing which specific spending intervals have statistically significant changes in probability. + +5. **Business Implications**: Email campaigns are most effective at converting non-buyers to buyers and redistributing customers toward moderate purchase amounts. The campaigns have minimal effect on driving high-value purchases ($200+). The PTE analysis provides actionable insights for campaign optimization by identifying which spending intervals are most responsive to each campaign type, enabling more precise targeting and resource allocation strategies. + +This distributional analysis reveals that email marketing's primary value lies in customer activation (reducing zero spending) and encouraging moderate purchase amounts, rather than dramatically increasing high-value purchases. The heterogeneous effects across the spending distribution provide actionable insights for optimizing email campaign strategies and customer segmentation. + +### Direct Campaign Comparison: Men's vs Women's Email + +Finally, let's directly compare the two email campaigns to answer the key research question. +This time we estimate the distribution treatment effects with regression adjustment using linear regression for higher precision. + +```python +# Compute DTE: Women's vs Men's email campaigns +dte_simple, lower_simple, upper_simple = simple_estimator.predict_dte( + target_treatment_arm=2, # Women's email + control_treatment_arm=1, # Men's email (as "control") + locations=revenue_locations, + variance_type="moment" +) + +dte_ml, lower_ml, upper_ml = ml_estimator.predict_dte( + target_treatment_arm=2, # Women's email + control_treatment_arm=1, # Men's email + locations=revenue_locations, + variance_type="moment" +) + +# Visualize the distribution treatment effects using dte_adj's built-in plot function +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +# Simple estimator +plot(revenue_locations, dte_simple, lower_simple, upper_simple, + title="Email Campaign Comparison: Women's vs Men's (Simple Estimator)", + xlabel="Spending ($)", ylabel="Distribution Treatment Effect", + color="purple", + ax=ax1) + +# ML-adjusted estimator +plot(revenue_locations, dte_ml, lower_ml, upper_ml, + title="Email Campaign Comparison: Women's vs Men's (ML-Adjusted Estimator)", + xlabel="Spending ($)", ylabel="Distribution Treatment Effect", + ax=ax2) + +plt.tight_layout() +plt.show() +``` + +The analysis produces the following distribution treatment effects visualization: + +![Hillstrom Email Marketing DTE Analysis](../assets/hillstorm_dte.png) + +The side-by-side plots show the distribution treatment effects (DTE) comparing Women's vs Men's email campaigns across different spending levels. Key observations: + +**DTE Interpretation**: The predominantly positive DTE values indicate that women's campaigns increase the cumulative probability of customers spending at or below each threshold compared to men's campaigns. This means women's campaigns result in more customers having lower revenue levels, which is unfavorable for business outcomes. + +**Men's Campaign Superiority**: The statistical significance of positive DTE values across most spending levels provides strong evidence that men's campaigns outperform women's campaigns by reducing the probability of customers spending small amounts and encouraging higher revenue per customer. + +**Business Implication**: The DTE analysis clearly demonstrates that men's campaigns are superior for revenue maximization, as they consistently reduce the cumulative probability of low spending levels, effectively shifting customers toward higher revenue categories. + +### Revenue Category Analysis with PTE + +```python +# Compute Probability Treatment Effects +pte_simple, pte_lower_simple, pte_upper_simple = simple_estimator.predict_pte( + target_treatment_arm=1, # Women's email + control_treatment_arm=0, # Men's email + locations=np.insert(revenue_locations, 0, -1), + variance_type="moment" +) + +pte_ml, pte_lower_ml, pte_upper_ml = ml_estimator.predict_pte( + target_treatment_arm=1, # Women's email + control_treatment_arm=0, # Men's email + locations=np.insert(revenue_locations, 0, -1), + variance_type="moment" +) + +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +# Simple estimator +plot(revenue_locations[1:], pte_simple, pte_lower_simple, pte_upper_simple, + chart_type="bar", + title="Spending Category Effects: Women's vs Men's (Simple Estimator)", + xlabel="Spending Category", ylabel="Probability Treatment Effect", color="purple", + ax=ax1) + +# ML-adjusted estimator +plot(revenue_locations[1:], pte_ml, pte_lower_ml, pte_upper_ml, + chart_type="bar", + title="Spending Category Effects: Women's vs Men's (ML-Adjusted Estimator)", + xlabel="Spending Category", ylabel="Probability Treatment Effect", + ax=ax2) +plt.tight_layout() +plt.show() +``` + +The Probability Treatment Effects analysis produces the following visualization: + +![Hillstrom Email Marketing PTE Analysis](../assets/hillstorm_pte.png) + +The side-by-side bar charts show probability treatment effects across different spending intervals, revealing the true story of campaign effectiveness: + +**Critical Finding - Zero Revenue Effect**: Women's campaigns show a positive effect in the $0 revenue category, meaning they increase the probability of customers making no purchase compared to men's campaigns. This is a negative outcome indicating that women's campaigns are less effective at driving any purchase behavior. + +**Spending Category Analysis**: Men's campaigns demonstrate superior performance in driving actual revenue. The negative PTE values in revenue-generating categories for women's campaigns indicate that men's campaigns are more effective at encouraging customers to make purchases and spend meaningful amounts. + +**Revenue Generation Patterns**: Men's campaigns show stronger performance in categories that generate actual revenue, while women's campaigns appear to be associated with higher non-purchase rates. This pattern suggests that men's campaigns are more effective at converting prospects into paying customers. + +**Methodological Confirmation**: Both simple and ML-adjusted estimators confirm this pattern, with the ML-adjusted analysis providing more precise estimates that strengthen the evidence for men's campaign superiority in driving revenue-generating behavior. + +**Strategic Implications**: Men's campaigns should be prioritized for revenue generation and customer conversion goals, as they demonstrate superior ability to drive actual purchases rather than just engagement. + +**Key PTE Findings**: + +1. **Men's Campaigns Drive More Purchases**: The critical finding is that women's campaigns increase the probability of zero revenue (non-purchase) compared to men's campaigns. This means men's campaigns are more effective at converting prospects into paying customers. + +2. **Revenue Generation Superiority**: Men's campaigns show consistently better performance in revenue-generating categories. The negative PTE values for women's campaigns in spending intervals indicate that men's campaigns drive more customers to make actual purchases across most revenue ranges. + +3. **Quantified Business Impact**: The analysis reveals that men's campaigns reduce non-purchase rates and increase the probability of revenue generation. Switching from women's to men's campaigns could improve overall conversion rates and revenue per customer. + +4. **Statistical Significance**: The statistical significance of the zero-revenue effect for women's campaigns provides strong evidence that men's campaigns are superior for business outcomes focused on revenue generation rather than just engagement. + +**Conclusion**: Using the real Hillstrom dataset with 64,000 customers, the distributional analysis reveals nuanced patterns in how email campaigns affect customer spending. The analysis goes beyond simple average comparisons to show how treatment effects vary across the entire spending distribution, providing insights into which customer segments respond best to different campaign types. This demonstrates the power of distribution treatment effect analysis for understanding heterogeneous responses in digital marketing experiments. + +### Subgroup Analysis by Purchase History + +Beyond comparing email campaigns overall, we can examine how campaign effectiveness varies by customer purchase history. This analysis segments customers based on their past purchasing behavior: + +- **Men's merchandise purchasers** (`mens=1`): Customers who previously purchased men's merchandise (35,266 customers, 55.1%) +- **Women's merchandise purchasers** (`womens=1`): Customers who previously purchased women's merchandise (35,182 customers, 55.0%) + +Note that these segments overlap (6,448 customers purchased both categories), so a customer can appear in both analyses. + +**Research Question**: Does the effectiveness of men's vs women's email campaigns vary by the type of merchandise customers have historically purchased? + +#### Defining Subgroups + +```python +# Define subgroup masks based on purchase history +mens_purchasers = (df['mens'] == 1) +womens_purchasers = (df['womens'] == 1) + +print(f"Men's merchandise purchaser segment: {mens_purchasers.sum():,} customers") +print(f"Women's merchandise purchaser segment: {womens_purchasers.sum():,} customers") +print(f"Overlap: {(mens_purchasers & womens_purchasers).sum():,} customers") +``` + +#### Average Treatment Effects by Subgroup + +Let's first compute the average treatment effects (ATEs) to quantify the overall impact: + +```python +# Compute ATEs for each campaign-subgroup combination +# Women's Email Campaign +ate_women_male = (revenue[(D==2) & mens_purchasers].mean() - + revenue[(D==0) & mens_purchasers].mean()) +ate_women_female = (revenue[(D==2) & womens_purchasers].mean() - + revenue[(D==0) & womens_purchasers].mean()) + +# Men's Email Campaign +ate_men_male = (revenue[(D==1) & mens_purchasers].mean() - + revenue[(D==0) & mens_purchasers].mean()) +ate_men_female = (revenue[(D==1) & womens_purchasers].mean() - + revenue[(D==0) & womens_purchasers].mean()) + +print("Average Treatment Effects by Subgroup:") +print("\nWomen's Email Campaign:") +print(f" Men's Merch. Purchasers: ATE = ${ate_women_male:.4f}") +print(f" Women's Merch. Purchasers: ATE = ${ate_women_female:.4f}") +print("\nMen's Email Campaign:") +print(f" Men's Merch. Purchasers: ATE = ${ate_men_male:.4f}") +print(f" Women's Merch. Purchasers: ATE = ${ate_men_female:.4f}") +``` + +Expected output: + +``` +Average Treatment Effects by Subgroup: + +Women's Email Campaign: + Men's Merch. Purchasers: ATE = $0.2564 + Women's Merch. Purchasers: ATE = $0.5442 + +Men's Email Campaign: + Men's Merch. Purchasers: ATE = $0.8966 + Women's Merch. Purchasers: ATE = $0.8412 +``` + +These results reveal important patterns: + +- **Women's Email Campaign**: Shows 2× stronger effect for women's merchandise purchasers ($0.54) vs men's merchandise purchasers ($0.26) +- **Men's Email Campaign**: Demonstrates consistent strong effects across both segments ($0.84-$0.89) + +While these averages provide a useful summary, they don't tell us *how* customer spending distributions change. The distributional and probability treatment effect analyses that follow reveal the complete picture of campaign effectiveness. + +#### Distribution Treatment Effects: Women's Email Campaign + +Beyond the average effects, let's examine how the Women's Email campaign shifts the entire spending distribution for each subgroup: + +```python +# Analyze men's merchandise purchaser segment +estimator_male = dte_adj.SimpleDistributionEstimator() +estimator_male.fit(X[mens_purchasers], D[mens_purchasers], revenue[mens_purchasers]) + +# Analyze women's merchandise purchaser segment +estimator_female = dte_adj.SimpleDistributionEstimator() +estimator_female.fit(X[womens_purchasers], D[womens_purchasers], revenue[womens_purchasers]) + +# Define evaluation points +locations = np.linspace(0, 500, 51) + +# Compute DTE for Women's Email vs Control in each subgroup +dte_women_male, lower_women_male, upper_women_male = estimator_male.predict_dte( + target_treatment_arm=2, # Women's Email + control_treatment_arm=0, # No Email + locations=locations, + variance_type="moment" +) + +dte_women_female, lower_women_female, upper_women_female = estimator_female.predict_dte( + target_treatment_arm=2, # Women's Email + control_treatment_arm=0, # No Email + locations=locations, + variance_type="moment" +) + +# Visualize side-by-side +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +plot(locations, dte_women_male, lower_women_male, upper_women_male, + title="Women's Email vs Control\nMen's Merch. Purchasers", + xlabel="Spending ($)", ylabel="Distribution Treatment Effect", + color="purple", ax=ax1) +ax1.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) + +plot(locations, dte_women_female, lower_women_female, upper_women_female, + title="Women's Email vs Control\nWomen's Merch. Purchasers", + xlabel="Spending ($)", ylabel="Distribution Treatment Effect", + color="green", ax=ax2) +ax2.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) + +plt.tight_layout() +plt.show() +``` + +![Women's Email Campaign Subgroup Analysis](../assets/hillstorm_subgroup_women_dte.png) + +**Key Finding for Women's Email Campaign**: The distributional treatment effects reveal that women's email campaigns are significantly more effective for the women's merchandise purchaser segment (right panel) compared to the men's merchandise purchaser segment (left panel). The DTE curves show that women's emails reduce the probability of low spending levels (negative DTE at lower thresholds) for women's merchandise purchasers, indicating a shift toward higher spending. In contrast, the men's merchandise purchaser segment shows minimal or non-significant effects across most of the spending distribution, with confidence intervals overlapping zero. + +#### Distribution Treatment Effects: Men's Email Campaign + +Now let's examine how the Men's Email campaign affects spending distributions: + +```python +# Compute DTE for Men's Email vs Control in each subgroup +dte_men_male, lower_men_male, upper_men_male = estimator_male.predict_dte( + target_treatment_arm=1, # Men's Email + control_treatment_arm=0, # No Email + locations=locations, + variance_type="moment" +) + +dte_men_female, lower_men_female, upper_men_female = estimator_female.predict_dte( + target_treatment_arm=1, # Men's Email + control_treatment_arm=0, # No Email + locations=locations, + variance_type="moment" +) + +# Visualize side-by-side +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +plot(locations, dte_men_male, lower_men_male, upper_men_male, + title="Men's Email vs Control\nMen's Merch. Purchasers", + xlabel="Spending ($)", ylabel="Distribution Treatment Effect", + color="purple", ax=ax1) +ax1.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) + +plot(locations, dte_men_female, lower_men_female, upper_men_female, + title="Men's Email vs Control\nWomen's Merch. Purchasers", + xlabel="Spending ($)", ylabel="Distribution Treatment Effect", + color="green", ax=ax2) +ax2.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) + +plt.tight_layout() +plt.show() +``` + +![Men's Email Campaign Subgroup Analysis](../assets/hillstorm_subgroup_men_dte.png) + +**Key Finding for Men's Email Campaign**: In contrast to women's email campaigns, men's email campaigns show consistent effectiveness across both purchase history segments. The DTE curves in both panels show similar patterns, with negative values at lower spending levels indicating reduced probability of low spending for both male and women's merchandise purchasers. This suggests that men's emails have broad appeal regardless of whether customers historically purchased men's or women's merchandise. + +#### Probability Treatment Effects: Women's Email Campaign + +While DTE shows how cumulative distributions shift, Probability Treatment Effects (PTE) reveal which specific spending intervals are most affected by the campaign. PTE measures the change in probability mass within each spending category: + +```python +# Compute PTE for Women's Email vs Control in each subgroup +pte_locations = np.insert(locations, 0, -1) # Add -1 at beginning for intervals + +pte_women_male, pte_lower_women_male, pte_upper_women_male = estimator_male.predict_pte( + target_treatment_arm=2, # Women's Email + control_treatment_arm=0, # No Email + locations=pte_locations, + variance_type="moment" +) + +pte_women_female, pte_lower_women_female, pte_upper_women_female = estimator_female.predict_pte( + target_treatment_arm=2, # Women's Email + control_treatment_arm=0, # No Email + locations=pte_locations, + variance_type="moment" +) + +# Visualize side-by-side with bar charts +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +plot(locations, pte_women_male, pte_lower_women_male, pte_upper_women_male, + chart_type="bar", + title="Women's Email vs Control\nMen's Merch. Purchasers", + xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", + color="purple", ax=ax1) +ax1.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) + +plot(locations, pte_women_female, pte_lower_women_female, pte_upper_women_female, + chart_type="bar", + title="Women's Email vs Control\nWomen's Merch. Purchasers", + xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", + color="green", ax=ax2) +ax2.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) + +plt.tight_layout() +plt.show() +``` + +![Women's Email Campaign PTE Subgroup Analysis](../assets/hillstorm_subgroup_women_pte.png) + +**Interval-Specific Insights**: The PTE bar charts reveal the mechanism behind the average treatment effect. For women's merchandise purchasers (right panel), women's emails significantly reduce the probability of zero spending (non-purchasers converting to purchasers), which is the primary driver of the positive ATE. However, no significant increase in high spending categories is observed. For men's merchandise purchasers (left panel), the effects are much smaller and less consistent, confirming the limited impact suggested by the ATE and DTE analyses. + +#### Probability Treatment Effects: Men's Email Campaign + +Let's examine which spending categories are most affected by men's email campaigns: + +```python +# Compute PTE for Men's Email vs Control in each subgroup +pte_men_male, pte_lower_men_male, pte_upper_men_male = estimator_male.predict_pte( + target_treatment_arm=1, # Men's Email + control_treatment_arm=0, # No Email + locations=pte_locations, + variance_type="moment" +) + +pte_men_female, pte_lower_men_female, pte_upper_men_female = estimator_female.predict_pte( + target_treatment_arm=1, # Men's Email + control_treatment_arm=0, # No Email + locations=pte_locations, + variance_type="moment" +) + +# Visualize side-by-side with bar charts +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +plot(locations, pte_men_male, pte_lower_men_male, pte_upper_men_male, + chart_type="bar", + title="Men's Email vs Control\nMen's Merch. Purchasers", + xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", + color="purple", ax=ax1) +ax1.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) + +plot(locations, pte_men_female, pte_lower_men_female, pte_upper_men_female, + chart_type="bar", + title="Men's Email vs Control\nWomen's Merch. Purchasers", + xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", + color="green", ax=ax2) +ax2.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) + +plt.tight_layout() +plt.show() +``` + +![Men's Email Campaign PTE Subgroup Analysis](../assets/hillstorm_subgroup_men_pte.png) + +**Interval-Specific Insights**: Men's email campaigns show similar PTE patterns across both segments (left and right panels). The key mechanism is twofold: (1) significant reduction in zero spending probability (converting non-purchasers to purchasers), and (2) increased probability in the $40-100 spending range. This dual effect—both purchase conversion and mid-range spending increases—occurs consistently across both male and women's merchandise purchaser segments, confirming the broad effectiveness of men's campaigns. + +#### Key Insights from Subgroup Analysis + +Combining Average Treatment Effects (ATE), Distribution Treatment Effects (DTE), and Probability Treatment Effects (PTE) provides a comprehensive understanding of campaign effectiveness: + +**1. Campaign Targeting Effectiveness (from ATE)** + +- Women's email campaigns show 2× stronger average effects for women's merchandise purchasers ($0.54) vs men's merchandise purchasers ($0.26) +- Men's email campaigns demonstrate consistent strong effects across both segments ($0.89-$0.84) +- This suggests women's campaigns benefit from precise targeting, while men's campaigns have broader appeal + +**2. Distributional Shifts Beyond Averages (from DTE)** + +- For women's emails, the women's merchandise purchaser segment shows negative DTE at lower spending thresholds, indicating a systematic shift away from low-spending behavior +- Men's merchandise purchasers show minimal distributional changes from women's emails, with confidence intervals overlapping zero at most thresholds +- Men's emails produce similar distributional patterns across both segments, confirming broad effectiveness + +**3. Spending Category Changes (from PTE)** + +- PTE analysis reveals *which specific spending intervals* change in response to campaigns, particularly identifying the mechanisms behind average effects +- **Women's emails**: For women's merchandise purchasers, the primary effect is converting non-purchasers to purchasers (significant reduction in zero spending probability). No significant increase in high spending categories was observed. +- **Men's emails**: Show a dual mechanism across both segments: (1) converting non-purchasers to purchasers (zero spending reduction), and (2) increasing purchases in the $40-100 range +- PTE enables identification of behavioral change mechanisms that are invisible in average treatment effects alone—specifically revealing that lift comes primarily from purchase conversion (0→1 effect) rather than spending increases among existing purchasers + +**4. Strategic Implications** + +Based on these findings, several practical implications emerge: + +- **For Women's Campaigns**: Target customers with history of purchasing women's merchandise to maximize ROI. The PTE analysis reveals that effectiveness comes primarily from converting non-purchasers to purchasers among women's merchandise purchaser segments, rather than increasing spending among existing buyers. +- **For Men's Campaigns**: Deploy broadly as they produce consistent positive effects across diverse customer segments. Both male and women's merchandise purchasers show both purchase conversion and mid-range spending increases, suggesting broader appeal. +- **Resource Allocation**: One practical implication is to prioritize precise targeting for gender-specific content (women's emails) but invest confidently in broad deployment for broadly appealing content (men's emails). + +**5. Methodological Value** + +This three-tier analysis demonstrates why distributional methods matter: + +- **ATE alone** would show that both campaigns have positive effects, but with varying magnitudes across subgroups +- **Adding DTE** reveals *how* spending distributions shift, not just average changes +- **Adding PTE** pinpoints *which spending categories* are most affected, enabling precise business decisions + +By examining effects at average, distributional, and interval-specific levels, we gain actionable insights that would be invisible to traditional mean-comparison approaches. This demonstrates the power of distribution treatment effect methods for understanding heterogeneous responses in digital marketing experiments. + +For the complete reproducible code including helper functions and visualizations, see [example/hillstrom.ipynb](https://github.com/CyberAgentAILab/python-dte-adjustment/blob/main/example/hillstrom.ipynb). + +### Next Steps + +- Try with your own randomized experiment data +- Experiment with different ML models (XGBoost, Neural Networks) for adjustment +- Explore stratified estimators for covariate-adaptive randomization designs +- Use multi-task learning (`is_multi_task=True`) for computational efficiency with many locations diff --git a/docs/docs/tutorials/index.md b/docs/docs/tutorials/index.md new file mode 100644 index 0000000..fc1cd10 --- /dev/null +++ b/docs/docs/tutorials/index.md @@ -0,0 +1,10 @@ +# Tutorials + +This section provides comprehensive tutorials showing how to use the `dte_adj` library to analyze distributional treatment effects in famous randomized control trials. These examples demonstrate the power of looking beyond average treatment effects to understand how interventions affect entire outcome distributions. + +## Available Tutorials + +- [Hillstrom Email Marketing](hillstrom.md) +- [Oregon Health Insurance Experiment](oregon.md) + +The tutorials demonstrate practical applications of the `dte_adj` library using real-world datasets from famous randomized experiments. Each tutorial provides complete code examples, visualizations, and interpretations of distributional treatment effects. diff --git a/docs/docs/tutorials/oregon.md b/docs/docs/tutorials/oregon.md new file mode 100644 index 0000000..a6f8d2b --- /dev/null +++ b/docs/docs/tutorials/oregon.md @@ -0,0 +1,594 @@ +# Oregon Health Insurance Experiment + +The Oregon Health Insurance Experiment is a landmark randomized controlled trial conducted in 2008, where approximately 24,000 low-income adults were randomly assigned to either receive the opportunity to enroll in Medicaid (treatment group) or remain uninsured (control group). This unique natural experiment allows us to examine how public health insurance affects healthcare utilization and costs across the entire distribution. + +**Background**: Due to budget constraints, Oregon decided to expand its Medicaid program through a lottery system, randomly selecting eligible individuals for enrollment opportunities. This created a rare natural experiment with non-compliance (not all selected individuals enrolled) that enables rigorous causal evaluation using Local Distribution Treatment Effects (LDTE) methodology. + +**Research Question**: How does Medicaid assignment (and enrollment) affect healthcare utilization (emergency department visits and costs), accounting for non-compliance, and how do these effects vary across the entire distribution of healthcare outcomes? + +### Data Setup and Loading + +**Data Source**: The Oregon Health Insurance Experiment data used in this tutorial is publicly available through the National Bureau of Economic Research (NBER). You can download the dataset from the official NBER Public Use Data Archive at: https://www.nber.org/research/data/oregon-health-insurance-experiment-data + +The dataset includes multiple files containing information about participants in the experiment: + +- `oregonhie_descriptive_vars.dta`: Demographic and baseline characteristics +- `oregonhie_ed_vars.dta`: Emergency department utilization data +- `oregonhie_inperson_vars.dta`: In-person survey responses +- `oregonhie_stateprograms_vars.dta`: State program participation data + +This data supports research on how health insurance affects healthcare utilization and is maintained by researchers Amy Finkelstein and Katherine Baicker. Please ensure you comply with the data use agreements when downloading and using this dataset. + +#### Import Libraries + +First, we import the necessary libraries for data processing, analysis, and visualization: + +```python +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import os +from sklearn.linear_model import LinearRegression +from sklearn.preprocessing import LabelEncoder +import dte_adj +from dte_adj.plot import plot +``` + +#### Load and Merge Datasets + +Next, we load the four separate data files and merge them into a single dataset: + +```python +# Load the Oregon Health Insurance Experiment dataset +base_path = "OHIE_Public_Use_Files/OHIE_Data" +df_descriptive = pd.read_stata(os.path.join(base_path, "oregonhie_descriptive_vars.dta")) +df_ed = pd.read_stata(os.path.join(base_path, "oregonhie_ed_vars.dta")) +df_inp = pd.read_stata(os.path.join(base_path, "oregonhie_inperson_vars.dta")) +df_state = pd.read_stata(os.path.join(base_path, "oregonhie_stateprograms_vars.dta")) + +# Merge all datasets +df = ( + df_descriptive + .merge(df_ed, on='person_id', how='inner') + .merge(df_inp, on='person_id', how='left') + .merge(df_state, on='person_id', how='inner') +) + +print(f"Dataset shape: {df.shape}") +print(f"Average num_visit_cens_ed by enrollment:\n{df.groupby('ohp_all_ever_inperson')['num_visit_cens_ed'].mean()}") +print(f"Average ed_charg_tot_ed by enrollment:\n{df.groupby('ohp_all_ever_inperson')['ed_charg_tot_ed'].mean()}") +``` + +#### Data Preprocessing + +Next, we prepare the data for the DTE analysis. This involves creating treatment variables, encoding categorical features, and selecting control variables: + +```python +# Create treatment assignment (instrumental variable): 0=Not selected, 1=Selected +treatment_assignment_mapping = {'Not selected': 0, 'Selected': 1} +df['Z'] = df['treatment'].map(treatment_assignment_mapping) + +# Create actual treatment indicator: 0=Not enrolled, 1=Enrolled +treatment_mapping = {'NOT enrolled': 0, 'Enrolled': 1} +df['D'] = df['ohp_all_ever_inperson'].map(treatment_mapping) + +# Create strata based on household size +df.rename(columns={'numhh_list': 'strata'}, inplace=True) +df['strata'] = df['strata'].astype(str).replace({ + 'signed self up + 1 additional person': 'signed self up + others', + 'signed self up + 2 additional people': 'signed self up + others' +}) + +# Create feature mappings for categorical variables +gender_mapping = {'Male': 0, 'Female': 1, 'Transgender F to M': 2, 'Transgender M to F': 3} +health_last12_mapping = {'1: Very poor': 1, '2: Poor': 2, '3: Fair': 3, '4: Good': 4, '5: Very good': 5, '6: Excellent': 6} +edu_mapping = {'HS diploma or GED': 0, 'Post HS, not 4-year': 1, 'Less than HS': 2, '4 year degree or more': 3} + +df['age'] = 2008 - df['birthyear_list'] +df['gender_inp'] = df['gender_inp'].map(gender_mapping).astype(float).fillna(-1).astype(int) +df['health_last12_inp'] = df['health_last12_inp'].map(health_last12_mapping).astype(float).fillna(-1).astype(int) +df['edu_inp'] = df['edu_inp'].map(edu_mapping).astype(float).fillna(-1).astype(int) + +# Select control variables: pre-randomization ED utilization variables +ctrl_cols = [col for col in df_ed.columns if 'pre' in col and 'num' in col] + ['gender_inp', 'age', 'health_last12_inp', 'edu_inp', 'charg_tot_pre_ed'] +selected_cols = ['person_id', 'strata', 'ed_charg_tot_ed', 'num_visit_cens_ed', 'Z', 'D'] + ctrl_cols +df = df[selected_cols] +df = df.dropna().reset_index(drop=True) +``` + +#### Prepare Variables for Analysis + +Finally, we create the feature matrices and outcome variables needed for the Local Distribution Treatment Effect analysis: + +```python +# Create feature matrix (excluding treatment variables) +X = df[ctrl_cols].values + +Z = df['Z'].astype(int).values # Treatment assignment (instrumental variable) +D = df['D'].astype(int).values # Actual treatment (endogenous variable) +strata = df['strata'].values # Stratification variable + +# Use num_visit_cens_ed and ed_charg_tot_ed as outcome variables +Y_ED_CHARG_TOT_ED = df['ed_charg_tot_ed'].values +Y_NUM_VISIT_CENS_ED = df['num_visit_cens_ed'].values + +print(f"\nDataset size: {len(D):,} people") +print(f"Treatment assignment (Z) - Not selected: {(Z==0).sum():,} ({(Z==0).mean():.1%})") +print(f"Treatment assignment (Z) - Selected: {(Z==1).sum():,} ({(Z==1).mean():.1%})") +print(f"Actual treatment (D) - Not enrolled: {(D==0).sum():,} ({(D==0).mean():.1%})") +print(f"Actual treatment (D) - Enrolled: {(D==1).sum():,} ({(D==1).mean():.1%})") +print("\nCompliance rate (among those assigned to treatment):") +print(f"Compliance rate: {(D[Z==1]==1).mean():.1%}") +print("\nAverage Outcome by Actual Treatment (D):") +print(f"Not enrolled (D=0): {Y_NUM_VISIT_CENS_ED[D==0].mean():.2f} visits, ${Y_ED_CHARG_TOT_ED[D==0].mean():.2f} in ED costs") +print(f"Enrolled (D=1): {Y_NUM_VISIT_CENS_ED[D==1].mean():.2f} visits, ${Y_ED_CHARG_TOT_ED[D==1].mean():.2f} in ED costs") +``` + +### Emergency Department Cost Analysis + +```python +# Initialize LOCAL estimators for non-compliance scenario +simple_local_estimator = dte_adj.SimpleLocalDistributionEstimator() +ml_local_estimator = dte_adj.AdjustedLocalDistributionEstimator( + LinearRegression(), + folds=5 +) + +# Fit estimators: fit(covariates, treatment_arms, treatment_indicator, outcomes, strata) +# treatment_arms = Z (Treatment assignment), treatment_indicator = D (Actual treatment) +simple_local_estimator.fit(X, Z, D, Y_ED_CHARG_TOT_ED, strata) +ml_local_estimator.fit(X, Z, D, Y_ED_CHARG_TOT_ED, strata) + +# Define evaluation points for emergency department costs +outcome_ed_costs_locations = np.arange(Y_ED_CHARG_TOT_ED.min(), Y_ED_CHARG_TOT_ED.max(), 3000) +``` + +### Local Estimator Comparison: Simple vs ML-Adjusted (Costs) + +Let's compare the results from both simple and machine learning-adjusted local estimators to examine the robustness of our findings: + +```python +# Compute LDTE: Treatment vs Control +ldte_simple, lower_simple, upper_simple = simple_local_estimator.predict_ldte( + target_treatment_arm=1, # Z=1 Selected in lottery for Medicaid (treatment assignment) + control_treatment_arm=0, # Z=0 Not selected in lottery for Medicaid (control assignment) + locations=outcome_ed_costs_locations +) + +ldte_ml, lower_ml, upper_ml = ml_local_estimator.predict_ldte( + target_treatment_arm=1, # Z=1 Selected in lottery for Medicaid (treatment assignment) + control_treatment_arm=0, # Z=0 Not selected in lottery for Medicaid (control assignment) + locations=outcome_ed_costs_locations +) + +# Visualize the distribution treatment effects using dte_adj's built-in plot function +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +# Visualize Treatment vs Control using dte_adj's plot function +plot(outcome_ed_costs_locations, ldte_simple, lower_simple, upper_simple, + title="ED Costs: Treatment vs Control (Simple Local Estimator)", + xlabel="Emergency Department Costs", + ylabel="Local Distribution Treatment Effect", + color="purple", + ax=ax1) + +plot(outcome_ed_costs_locations, ldte_ml, lower_ml, upper_ml, + title="ED Costs: Treatment vs Control (ML-Adjusted Local Estimator)", + xlabel="Emergency Department Costs", + ylabel="Local Distribution Treatment Effect", + ax=ax2) + +plt.tight_layout() +plt.show() +``` + +The analysis produces the following local distribution treatment effects visualization: + +![Oregon Health Insurance Experiment LDTE Analysis](../assets/oregon_ldte_costs_comparison.png) + +**1. LDTE Interpretation and Distribution-Level Insights** + +- **Simple Local Estimator**: Shows LDTE ≈ -0.12 at zero costs, meaning 12 percentage points fewer insured individuals have zero ED costs. The effect converges to zero around $10,000 and remains flat thereafter. +- **ML-Adjusted Local Estimator**: Shows an LDTE ≈ -0.15 at zero costs—a slightly larger (more negative) effect than -0.12—with similar convergence patterns. +- **Key Finding**: Both estimators reveal insurance primarily affects the lower tail (zero to ~$10,000), shifting the distribution rightward. This indicates insurance increases ED access among those who would otherwise not seek care, while having minimal impact on high-cost users. + +The confidence intervals are not substantially narrower with ML adjustment. Both methods show comparably wide confidence bands, indicating limited efficiency gains. This result reflects the **limited predictive power of available covariates** (R² ≈ 0.21 when predicting ED costs from pre-treatment ED history and demographics). + +ML adjustment provides efficiency gains proportional to covariate predictive power. When covariates weakly predict outcomes (R² < 0.3), as in this case, ML adjustment yields minimal improvements over simple estimation. This is a characteristic of the data—pre-treatment healthcare utilization and basic demographics cannot strongly predict future emergency department costs—not a failure of the ML methodology. + +### Cost Analysis with Local PTE + +```python +# Compute Local Probability Treatment Effects +lpte_simple, lpte_lower_simple, lpte_upper_simple = simple_local_estimator.predict_lpte( + target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) + control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) + locations=np.insert(outcome_ed_costs_locations, 0, -1) +) + +lpte_ml, lpte_lower_ml, lpte_upper_ml = ml_local_estimator.predict_lpte( + target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) + control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) + locations=np.insert(outcome_ed_costs_locations, 0, -1) +) + +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +# Simple local estimator +plot(outcome_ed_costs_locations[1:], lpte_simple, lpte_lower_simple, lpte_upper_simple, + chart_type="bar", + title="Effects of Emergency Department Costs (Simple Local Estimator)", + xlabel="Emergency Department Costs", + ylabel="Local Probability Treatment Effect", + color="purple", + ax=ax1) + +# ML-adjusted local estimator +plot(outcome_ed_costs_locations[1:], lpte_ml, lpte_lower_ml, lpte_upper_ml, + chart_type="bar", + title="Effects of Emergency Department Costs (ML-Adjusted Local Estimator)", + xlabel="Emergency Department Costs", + ylabel="Local Probability Treatment Effect", + ax=ax2) +plt.tight_layout() +plt.show() +``` + +The Local Probability Treatment Effects analysis produces the following visualization: + +![Oregon Health Insurance Experiment LPTE Analysis](../assets/oregon_lpte_costs_comparison.png) + +**1. LPTE Interpretation and Distribution-Level Insights** + +- **Simple Local Estimator**: Shows mixed effects across cost bins. At zero costs, LPTE ≈ -0.02 (not statistically significant given wide confidence intervals). Small positive effects appear in the $5,000-$10,000 range (LPTE ≈ 0.01-0.02), converging to zero beyond $30,000. +- **ML-Adjusted Local Estimator**: Shows a larger negative effect at zero costs (LPTE ≈ -0.04) and positive effects in the $5,000-$15,000 range (LPTE ≈ 0.01-0.02). Effects converge to zero at higher costs. +- **Key Finding**: Insurance reduces the probability mass at zero costs while increasing it in the moderate cost range ($5,000-$15,000). This represents a redistribution of probability mass from non-users to moderate ED cost users, with minimal effect on high-cost outliers. + +**2. Covariate Adjustment Effects and Confidence Intervals** + +The confidence intervals remain wide for both estimators, though ML adjustment shows slightly more consistent patterns in the moderate cost range. The limited precision suggests: (1) substantial heterogeneity in treatment effects within cost bins, (2) limited predictive power of covariates for specific cost levels, or (3) relatively small sample sizes within individual bins. + +### Emergency Department Visits Analysis + +Now let's examine how Medicaid enrollment affects the distribution of emergency department visits (rather than costs): + +```python +# Initialize local estimators for visits analysis +simple_local_estimator = dte_adj.SimpleLocalDistributionEstimator() +ml_local_estimator = dte_adj.AdjustedLocalDistributionEstimator( + LinearRegression(), + folds=5 +) + +# Fit local estimators on the full dataset +# Parameters: X, treatment_arms, treatment_indicator, outcomes, strata +simple_local_estimator.fit(X, Z, D, Y_NUM_VISIT_CENS_ED, strata) +ml_local_estimator.fit(X, Z, D, Y_NUM_VISIT_CENS_ED, strata) + +# Define evaluation points for emergency department visits +outcome_ed_visits_locations = np.arange(Y_NUM_VISIT_CENS_ED.min(), Y_NUM_VISIT_CENS_ED.max(), 1) +``` + +### Local Estimator Comparison: Simple vs ML-Adjusted (Visits) + +Let's compare the results from both simple and machine learning-adjusted local estimators for visits analysis: + +```python +# Compute LDTE: Treatment vs Control +ldte_simple, lower_simple, upper_simple = simple_local_estimator.predict_ldte( + target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) + control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) + locations=outcome_ed_visits_locations +) + +ldte_ml, lower_ml, upper_ml = ml_local_estimator.predict_ldte( + target_treatment_arm=1, # Selected for treatment (Enrolled) + control_treatment_arm=0, # Not selected for treatment (Not enrolled) + locations=outcome_ed_visits_locations +) + +# Visualize the local distribution treatment effects using dte_adj's built-in plot function +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +# Visualize Treatment vs Control using dte_adj's plot function +plot(outcome_ed_visits_locations, ldte_simple, lower_simple, upper_simple, + title="ED Visits: Treatment vs Control (Simple Local Estimator)", + xlabel="Emergency Department Visits", + ylabel="Local Distribution Treatment Effect", + color="purple", + ax=ax1) + +plot(outcome_ed_visits_locations, ldte_ml, lower_ml, upper_ml, + title="ED Visits: Treatment vs Control (ML-Adjusted Local Estimator)", + xlabel="Emergency Department Visits", + ylabel="Local Distribution Treatment Effect", + ax=ax2) + +plt.tight_layout() +plt.show() +``` + +![Oregon Health Insurance Experiment LDTE Visits Analysis](../assets/oregon_ldte_visits_comparison.png) + +**1. LDTE Interpretation and Distribution-Level Insights** + +- **Simple Local Estimator**: Shows LDTE ≈ -0.12 at zero visits, meaning 12 percentage points fewer insured individuals have zero ED visits. The effect gradually converges toward zero around 16 visits and remains near zero thereafter. +- **ML-Adjusted Local Estimator**: Shows a more negative effect of LDTE ≈ -0.15 at zero visits (a larger-magnitude reduction), with similar convergence patterns. +- **Key Finding**: Both estimators reveal insurance primarily affects the lower tail (zero to ~7 visits), shifting the distribution rightward. This indicates insurance increases ED utilization among those who would otherwise not visit, while having minimal impact on frequent ED users. + +**2. Covariate Adjustment Effects and Confidence Intervals** + +The confidence intervals are not substantially narrower with ML adjustment. Both methods show comparably wide confidence bands, indicating limited efficiency gains. This suggests: (1) covariates have limited predictive power for ED visit frequency, (2) the linear regression model may be too simple, or (3) the simple estimator is already reasonably efficient. + +### Visits Analysis with Local PTE + +```python +# Compute Local Probability Treatment Effects +lpte_simple, lpte_lower_simple, lpte_upper_simple = simple_local_estimator.predict_lpte( + target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) + control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) + locations=np.insert(outcome_ed_visits_locations, 0, -1) +) + +lpte_ml, lpte_lower_ml, lpte_upper_ml = ml_local_estimator.predict_lpte( + target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) + control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) + locations=np.insert(outcome_ed_visits_locations, 0, -1) +) + +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) + +# Simple local estimator +plot(outcome_ed_visits_locations[1:], lpte_simple, lpte_lower_simple, lpte_upper_simple, + chart_type="bar", + title="Effects of Emergency Department Visits (Simple Local Estimator)", + xlabel="Emergency Department Visits", + ylabel="Local Probability Treatment Effect", + color="purple", + ax=ax1) + +# ML-adjusted local estimator +plot(outcome_ed_visits_locations[1:], lpte_ml, lpte_lower_ml, lpte_upper_ml, + chart_type="bar", + title="Effects of Emergency Department Visits (ML-Adjusted Local Estimator)", + xlabel="Emergency Department Visits", + ylabel="Local Probability Treatment Effect", + ax=ax2) + +plt.tight_layout() +plt.show() +``` + +![Oregon Health Insurance Experiment LPTE Visits Analysis](../assets/oregon_lpte_visits_comparison.png) + +**1. LPTE Interpretation and Distribution-Level Insights** + +- **Simple Local Estimator**: Shows a large negative effect at zero visits (LPTE ≈ -0.12), indicating 12 percentage points fewer insured individuals have zero ED visits. Small positive effects appear in the 1-5 visit range (LPTE ≈ 0.02-0.03), converging to zero beyond 7 visits. +- **ML-Adjusted Local Estimator**: Shows a larger negative effect at zero visits (LPTE ≈ -0.14) and positive effects in the 1-5 visit range (LPTE ≈ 0.03-0.04). Effects converge to zero at higher visit frequencies. +- **Key Finding**: Insurance reduces the probability mass at zero visits while increasing it in the low-to-moderate visit range (1-5 visits). This represents a redistribution of probability mass from non-users to low-frequency ED users, with minimal effect on frequent visitors. + +The confidence intervals remain wide for both estimators, with minimal differences between simple and ML-adjusted approaches. This limited precision reflects the same fundamental constraint as in the cost analysis: covariates have limited predictive power for ED visit frequency (R² ≈ 0.21). The substantial heterogeneity in treatment effects, combined with weak covariate prediction, means ML adjustment provides minimal efficiency gains over the simpler approach. + +### Stratified Analysis by Household Registration + +The Oregon experiment allows us to examine how treatment effects vary across different household registration patterns. This stratified analysis helps identify heterogeneous treatment effects and provides insights into which populations benefit most from Medicaid enrollment. + +```python +from sklearn.ensemble import RandomForestRegressor + +# Individual Stratum Analysis with Local Estimators +print("\n=== Individual Stratum Analysis (Local Estimators) ===") + +# Helper function to filter data for a specific stratum +def filter_stratum_data(strata_values, stratum_name, X, Z, D, Y): + """Filter and extract data for a specific stratum""" + mask = strata_values == stratum_name + return { + 'X': X[mask], + 'Z': Z[mask], + 'D': D[mask], + 'Y': Y[mask], + 'strata': np.zeros(mask.sum(), dtype=int), # Uniform strata for subset + 'n_total': mask.sum(), + 'n_assigned': (Z[mask] == 1).sum(), + 'n_enrolled': (D[mask] == 1).sum() + } + +# Helper function to estimate LDTE for a stratum +def estimate_stratum_ldte(stratum_data, location_step=3000, folds=3): + """Initialize estimators, fit data, and compute LDTE for a stratum""" + # Initialize estimators + simple_estimator = dte_adj.SimpleLocalDistributionEstimator() + ml_estimator = dte_adj.AdjustedLocalDistributionEstimator( + RandomForestRegressor(n_estimators=10, random_state=42), + folds=folds + ) + + # Fit estimators + simple_estimator.fit(stratum_data['X'], stratum_data['Z'], + stratum_data['D'], stratum_data['Y'], stratum_data['strata']) + ml_estimator.fit(stratum_data['X'], stratum_data['Z'], + stratum_data['D'], stratum_data['Y'], stratum_data['strata']) + + # Define evaluation locations based on stratum's data range + locations = np.arange(stratum_data['Y'].min(), stratum_data['Y'].max(), location_step) + + # Compute LDTE + ldte_simple, lower_simple, upper_simple = simple_estimator.predict_ldte( + target_treatment_arm=1, control_treatment_arm=0, locations=locations + ) + ldte_ml, lower_ml, upper_ml = ml_estimator.predict_ldte( + target_treatment_arm=1, control_treatment_arm=0, locations=locations + ) + + return { + 'simple': {'ldte': ldte_simple, 'lower': lower_simple, 'upper': upper_simple}, + 'ml': {'ldte': ldte_ml, 'lower': lower_ml, 'upper': upper_ml}, + 'locations': locations, + 'sample_size': stratum_data['n_total'], + 'treatment_assignment_size': stratum_data['n_assigned'], + 'treatment_indicator_size': stratum_data['n_enrolled'] + } + +# Get strata values (already consolidated in preprocessing) +strata_consolidated_values = df['strata'].values +unique_consolidated_strata = np.unique(strata_consolidated_values) + +# Analyze each stratum +individual_results = {} +for stratum in unique_consolidated_strata: + print(f"\nAnalyzing stratum: {stratum}") + + # Filter data for this stratum + stratum_data = filter_stratum_data( + strata_consolidated_values, stratum, X, Z, D, Y_ED_CHARG_TOT_ED + ) + + # Print stratum statistics + print(f" Sample size: {stratum_data['n_total']:,}") + print(f" Treatment assignment (Selected): {stratum_data['n_assigned']:,}") + print(f" Treatment indicator (Enrolled): {stratum_data['n_enrolled']:,}") + + # Estimate LDTE for this stratum + individual_results[stratum] = estimate_stratum_ldte( + stratum_data, location_step=2000, folds=3 + ) +``` + +### Visualization: Comparing Overall Population vs Stratified Results + +```python +# Comparison: Overall vs Individual Strata (Local Estimators) +fig, axes = plt.subplots(2, 3, figsize=(24, 12)) + +# Row 1: Simple local estimators +# Overall (all data) +plot(outcome_ed_costs_locations, ldte_simple, lower_simple, upper_simple, + title="ED Costs: Overall Population\n(Simple Local Estimator)", + xlabel="Emergency Department Costs", + ylabel="Local Distribution Treatment Effect", + color="black", ax=axes[0, 0]) + +# Individual strata +col_idx = 1 +for stratum, results in individual_results.items(): + if results is None or col_idx > 2: + continue + + plot(results['locations'], results['simple']['ldte'], + results['simple']['lower'], results['simple']['upper'], + title=f"ED Costs: {stratum}\n(Simple Local Estimator, n={results['sample_size']:,})", + xlabel="Emergency Department Costs", + ylabel="Local Distribution Treatment Effect", + color="blue" if col_idx == 1 else "green", ax=axes[0, col_idx]) + col_idx += 1 + +# Row 2: ML-Adjusted local estimators +# Overall (all data) +plot(outcome_ed_costs_locations, ldte_ml, lower_ml, upper_ml, + title="ED Costs: Overall Population\n(ML-Adjusted Local Estimator)", + xlabel="Emergency Department Costs", + ylabel="Local Distribution Treatment Effect", + color="black", ax=axes[1, 0]) + +# Individual strata +col_idx = 1 +for stratum, results in individual_results.items(): + if results is None or col_idx > 2: + continue + + plot(results['locations'], results['ml']['ldte'], + results['ml']['lower'], results['ml']['upper'], + title=f"ED Costs: {stratum}\n(ML-Adjusted Local Estimator, n={results['sample_size']:,})", + xlabel="Emergency Department Costs", + ylabel="Local Distribution Treatment Effect", + color="red" if col_idx == 1 else "purple", ax=axes[1, col_idx]) + col_idx += 1 + +plt.suptitle("Comparison: Overall Population vs Individual Household Registration Strata (Local Estimators)", fontsize=16) +plt.tight_layout() +plt.show() +``` + +![Oregon Health Insurance Experiment DTE Strata Analysis](../assets/oregon_ldte_costs_strata.png) + +**1. Overall Population vs Stratified Analysis** + +- **Overall Population (Left panels)**: + + - Simple: LDTE ≈ -0.21 at zero costs, converging to zero around $10,000 + - ML-Adjusted: LDTE ≈ -0.15 at zero costs, similar convergence pattern + - Both show consistent rightward distribution shifts across the entire population + +- **Signed Self Up (Middle panels, n=12,982)**: + + - Simple: LDTE ≈ -0.18 at zero costs, converging to zero around $20,000 + - ML-Adjusted: LDTE ≈ -0.20 at zero costs, similar pattern + - Smaller magnitude effects compared to overall population, suggesting this stratum has more moderate responses to insurance + +- **Signed Self Up + Others (Right panels, n=4,068)**: + + - Simple: LDTE ≈ -0.55 at zero costs, converging to zero around $15,000-$20,000 + - ML-Adjusted: LDTE ≈ -0.10 to -0.15 at zero costs, stable pattern with improved confidence intervals + - Much larger magnitude effects in the Simple estimator, indicating households with multiple members show substantially stronger treatment effects + - ML adjustment provides more conservative estimates, potentially controlling for confounding household characteristics + +**2. Heterogeneity Across Strata** + +The stratified analysis reveals substantial treatment effect heterogeneity: + +- **"Signed self up" stratum**: Moderate effects (LDTE ≈ -0.18 to -0.20), suggesting single-person households have more modest increases in ED utilization +- **"Signed self up + others" stratum**: Large effects in Simple estimator (LDTE ≈ -0.55), suggesting multi-person households experience much greater increases in ED access when not adjusting for covariates +- The 3-4x larger effect in the "signed self up + others" group (Simple estimator) indicates that household composition is a critical moderator of insurance impact +- However, ML adjustment substantially reduces this estimate, suggesting that some of the observed effect may be attributable to observable household characteristics rather than pure treatment effects + +**3. Comparison of Estimation Methods** + +- **Overall population**: Both estimators show reasonable confidence intervals, with ML adjustment providing modest improvements in precision and slightly more conservative estimates. +- **"Signed self up" stratum**: Both estimators yield similar point estimates and manageable confidence intervals, suggesting robustness to model specification in this larger subsample. +- **"Signed self up + others" stratum**: + + - The Simple estimator shows the largest treatment effects across all strata (LDTE ≈ -0.55) + - ML adjustment substantially reduces the estimated effect and stabilizes confidence intervals + - This divergence suggests that observable covariates (e.g., household size, age composition, baseline health status) explain a significant portion of the treatment effect heterogeneity + - The improved stability of ML-adjusted estimates indicates successful control for confounding factors that may have been correlated with both treatment assignment and outcomes + +**4. Practical Implications** + +- **Household structure matters**: Multi-person households show substantially larger treatment effects in unadjusted analyses, likely because insurance coverage enables care-seeking for multiple family members. +- **The role of covariates**: The difference between Simple and ML-adjusted estimates in the "signed self up + others" stratum highlights the importance of controlling for household characteristics. The unadjusted effect may overstate the pure treatment effect by conflating insurance provision with pre-existing household differences. +- **Stratification reveals hidden heterogeneity**: The overall population estimate masks substantial variation across household types, demonstrating the value of subgroup analysis. +- **Model specification considerations**: ML adjustment improves estimation stability in smaller strata and provides more defensible causal estimates by controlling for observable confounders. The convergence of all estimates to zero at higher cost levels confirms that the treatment primarily affects the lower tail of the cost distribution. + +### Conclusion + +This analysis of the Oregon Health Insurance Experiment using local distribution treatment effects reveals specific patterns in how Medicaid insurance affects emergency department utilization among compliers: + +**1. Insurance Primarily Shifts the Lower Tail of the Distribution** + +Our LDTE analysis shows that Medicaid insurance reduces the probability of zero ED costs by 12-15 percentage points (LDTE ≈ -0.12 to -0.15 at $0), with effects converging to zero around $10,000. Similarly, for ED visits, insurance reduces zero visits by 12-14 percentage points (LPTE ≈ -0.12 to -0.14 at 0 visits). This indicates insurance primarily enables access for those who would otherwise not use ED services, rather than affecting high-cost or frequent users. + +**2. Probability Mass Redistribution, Not Uniform Increases** + +The LPTE analysis reveals insurance does not uniformly increase ED utilization. Instead, it redistributes probability mass: reducing zero-cost/zero-visit individuals while increasing moderate users ($5,000-$15,000 costs; 1-5 visits). High-cost outliers (>$30,000) and frequent users (>7 visits) show minimal treatment effects, suggesting insurance's impact is concentrated among marginal users. + +**3. Substantial Heterogeneity by Household Composition** + +Stratified analysis uncovers dramatic treatment effect heterogeneity: single-person households ("signed self up") show moderate effects (LDTE ≈ -0.18 to -0.20), while multi-person households ("signed self up + others") exhibit 3-4x larger effects (LDTE ≈ -0.55). This suggests household structure is a critical moderator—insurance enables care-seeking for multiple family members when households include dependents. + +**4. ML Adjustment Effectiveness Depends on Covariate Predictive Power** + +With baseline covariates (pre-randomization ED utilization + demographics, R² ≈ 0.21), ML-adjusted estimators show minimal efficiency gains—confidence intervals remain comparably wide or even slightly wider than simple estimators. However, enhanced feature engineering could improve predictive power, enabling ML adjustment to narrow confidence intervals. + +**5. Policy Implications for Targeted Interventions** + +The distributional analysis reveals that Medicaid's primary benefit is enabling ED access for marginal users who would otherwise forego care, rather than increasing utilization among existing high users. The 3-4x larger effects for multi-person households suggest family coverage may yield substantially greater utilization impacts than individual coverage. These findings have direct implications for healthcare budgeting and targeting: policymakers should anticipate larger ED increases when expanding coverage to families versus individuals, and the primary fiscal impact will come from converting non-users to moderate users rather than increasing costs among existing high users. + +### Next Steps + +**For Your Own Data**: + +- Try with your own randomized experiment data +- Experiment with different ML models (XGBoost, Neural Networks) for adjustment +- Explore stratified estimators for covariate-adaptive randomization designs +- Use multi-task learning (`is_multi_task=True`) for computational efficiency with many locations diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index dc1312a..0000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml new file mode 100644 index 0000000..93c0a7a --- /dev/null +++ b/docs/mkdocs.yml @@ -0,0 +1,95 @@ +site_name: dte_adj +site_description: A Python package for estimating distribution treatment effects +site_url: https://cyberagentailab.github.io/python-dte-adjustment/ +repo_url: https://github.com/CyberAgentAILab/python-dte-adjustment +repo_name: CyberAgentAILab/python-dte-adjustment +copyright: Copyright © 2024 CyberAgent, Inc. + +docs_dir: docs +site_dir: build/html + +theme: + name: material + features: + - navigation.tabs + - navigation.sections + - navigation.top + - navigation.instant + - navigation.tracking + - navigation.indexes + - search.suggest + - search.highlight + - content.code.copy + - content.code.annotate + - toc.follow + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/brightness-4 + name: Switch to light mode + +nav: + - Home: index.md + - Installation: installation.md + - Get Started: get_started.md + - Tutorials: + - tutorials/index.md + - Hillstrom: tutorials/hillstrom.md + - Oregon: tutorials/oregon.md + - API Reference: + - api/index.md + - Simple Randomization: api/simple.md + - Covariate Adaptive Randomization: api/stratified.md + - Local Distribution: api/local.md + - Plotting: api/plot.md + - Contributing: contributing.md + +plugins: + - search + - mkdocstrings: + default_handler: python + handlers: + python: + paths: [..] + options: + docstring_style: numpy + show_source: true + show_root_heading: false + show_root_toc_entry: false + show_signature_annotations: true + separate_signature: true + merge_init_into_class: true + docstring_section_style: table + members_order: source + filters: ["!^_"] + +markdown_extensions: + - admonition + - attr_list + - footnotes + - md_in_html + - toc: + permalink: true + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/CyberAgentAILab/python-dte-adjustment diff --git a/docs/source/api/plot.rst b/docs/source/api/plot.rst deleted file mode 100644 index f394529..0000000 --- a/docs/source/api/plot.rst +++ /dev/null @@ -1,10 +0,0 @@ -Plotting Utilities -================== - -This page documents plotting utilities for visualizing distribution treatment effects, confidence intervals, and other results. - -.. automodule:: dte_adj.plot - :members: - :undoc-members: - :show-inheritance: - :no-index: diff --git a/docs/source/api/simple.rst b/docs/source/api/simple.rst deleted file mode 100644 index a6eda36..0000000 --- a/docs/source/api/simple.rst +++ /dev/null @@ -1,30 +0,0 @@ -Simple Randomization Estimators -=============================== - -This page documents estimators that work with simple randomized experimental designs where treatment assignment is completely randomized. - -These estimators leverage pre-treatment covariates through distributional regression frameworks to improve the precision of distributional treatment effect estimates. The key methodological contribution is using machine learning techniques for variance reduction while maintaining validity as long as nuisance components are reasonably well estimated. - -Byambadalai et al. (2024) [#simple2024]_ propose a regression adjustment method that incorporates covariates into distributional regression, enabling deeper insights beyond average treatment effects by estimating full distributional treatment effects in randomized experiments. - -.. [#simple2024] Byambadalai, U., Oka, T., & Yasui, S. (2024). Estimating Distributional Treatment Effects in Randomized Experiments: Machine Learning for Variance Reduction. arXiv preprint `arXiv:2407.16037 `_. - -SimpleDistributionEstimator ---------------------------- - -.. autoclass:: dte_adj.SimpleDistributionEstimator - :members: - :inherited-members: - :undoc-members: - :show-inheritance: - :no-index: - -AdjustedDistributionEstimator ------------------------------ - -.. autoclass:: dte_adj.AdjustedDistributionEstimator - :members: - :inherited-members: - :undoc-members: - :show-inheritance: - :no-index: diff --git a/docs/source/api/stratified.rst b/docs/source/api/stratified.rst deleted file mode 100644 index 36f068b..0000000 --- a/docs/source/api/stratified.rst +++ /dev/null @@ -1,30 +0,0 @@ -Covariate Adaptive Randomization Estimators -=========================================== - -This page documents estimators that work with stratified experimental designs, particularly for covariate-adaptive randomization (CAR) within strata. - -These estimators are designed to handle stratified block randomization where participants are grouped into strata based on baseline covariates before treatment assignment. The key methodological contribution is leveraging additional covariates beyond strata indicators using machine learning methods to enhance the precision of distributional treatment effect estimates. - -Byambadalai et al. (2025) [#car2025]_ propose a flexible distribution regression framework that achieves the semiparametric efficiency bound for distributional treatment effects under CAR, demonstrating that regression-adjusted estimators can optimally utilize covariate information in stratified designs. - -.. [#car2025] Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). On Efficient Estimation of Distributional Treatment Effects under Covariate-Adaptive Randomization. arXiv preprint `arXiv:2506.05945 `_. - -SimpleStratifiedDistributionEstimator -------------------------------------- - -.. autoclass:: dte_adj.SimpleStratifiedDistributionEstimator - :members: - :inherited-members: - :undoc-members: - :show-inheritance: - :no-index: - -AdjustedStratifiedDistributionEstimator ---------------------------------------- - -.. autoclass:: dte_adj.AdjustedStratifiedDistributionEstimator - :members: - :inherited-members: - :undoc-members: - :show-inheritance: - :no-index: diff --git a/docs/source/api_reference.rst b/docs/source/api_reference.rst deleted file mode 100644 index 5c7b1d1..0000000 --- a/docs/source/api_reference.rst +++ /dev/null @@ -1,38 +0,0 @@ -API Reference -============= - -This section provides comprehensive documentation for all classes and functions in the dte_adj package. The API is organized into logical groups based on functionality and use cases. - -Overview --------- - -The dte_adj package provides several types of estimators for computing distribution treatment effects: - -* **Simple Randomization Estimators**: For estimating distributional effects in simple randomized experiments where treatment assignment is independent of all covariates -* **Covariate Adaptive Randomization Estimators**: For estimating distributional effects under covariate-adaptive randomization (CAR) designs, including stratified block randomization and other adaptive schemes -* **Local Distribution Estimators**: For estimating local distribution treatment effects weighted by treatment propensity within strata -* **Utility Functions**: Helper functions for confidence intervals and statistical computations -* **Plotting Utilities**: Visualization tools for treatment effects and distributions - -For theoretical foundations, see Byambadalai et al. (2024) [#simple2024]_ for simple randomization, Byambadalai et al. (2025) [#car2025]_ for covariate-adaptive randomization, and Byambadalai et al. (2025) [#compliance2025]_ for imperfect compliance scenarios. - -For multi-task learning approaches that train models for all locations simultaneously (using ``is_multi_task=True``), see the neural network framework in [#multitask2025]_. - -.. [#simple2024] Byambadalai, U., Oka, T., & Yasui, S. (2024). Estimating Distributional Treatment Effects in Randomized Experiments: Machine Learning for Variance Reduction. In Proceedings of the 41st International Conference on Machine Learning (ICML'24). `arXiv:2407.16037 `_. - -.. [#car2025] Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). On Efficient Estimation of Distributional Treatment Effects under Covariate-Adaptive Randomization. In Proceedings of the 42nd International Conference on Machine Learning (ICML'25). `arXiv:2506.05945 `_. - -.. [#multitask2025] Hirata, T., Byambadalai, U., Oka, T., Yasui, S., & Uto, S. (2025). Efficient and Scalable Estimation of Distributional Treatment Effects with Multi-Task Neural Networks. arXiv preprint `arXiv:2507.07738 `_. - -.. [#compliance2025] Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). Beyond the Average: Distributional Causal Inference under Imperfect Compliance. arXiv preprint `arXiv:2509.15594 `_. - -Detailed Documentation ----------------------- - -.. toctree:: - :maxdepth: 2 - - api/simple - api/stratified - api/local - api/plot diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index cbd421a..0000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import sys - -sys.path.insert(0, os.path.abspath("../../")) -# Configuration file for the Sphinx documentation builder. -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information - -project = "dte_adj" -copyright = "2024, CyberAgent, Inc." -author = "CyberAgent, Inc" -release = "0.1.9" - -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx.ext.viewcode"] - -templates_path = ["_templates"] -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - -html_theme = "alabaster" -html_static_path = ["_static"] diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst deleted file mode 100644 index f032ef1..0000000 --- a/docs/source/contributing.rst +++ /dev/null @@ -1,4 +0,0 @@ -Contribution Guide -================== - -Regarding how to contribute to this package, please refer to https://github.com/CyberAgentAILab/python-dte-adjustment/blob/main/CONTRIBUTING.md for more details. diff --git a/docs/source/get_started.rst b/docs/source/get_started.rst deleted file mode 100644 index f2c369e..0000000 --- a/docs/source/get_started.rst +++ /dev/null @@ -1,181 +0,0 @@ -Get Started -=========== - -This page contains basic usage of dte_adj library. - -Generate data for training cumulative distribution function: - -.. code-block:: python - - import numpy as np - - def generate_data(n, d_x=100, rho=0.5): - """ - Generate data according to the described data generating process (DGP). - - Args: - n (int): Number of samples. - d_x (int): Number of covariates. Default is 100. - rho (float): Success probability for the Bernoulli distribution. Default is 0.5. - - Returns: - X (np.ndarray): Covariates matrix of shape (n, d_x). - D (np.ndarray): Treatment variable array of shape (n,). - Y (np.ndarray): Outcome variable array of shape (n,). - """ - # Generate covariates X from a uniform distribution on (0, 1) - X = np.random.uniform(0, 1, (n, d_x)) - - # Generate treatment variable D from a Bernoulli distribution with success probability rho - D = np.random.binomial(1, rho, n) - - # Define beta_j and gamma_j according to the problem statement - beta = np.zeros(d_x) - gamma = np.zeros(d_x) - - # Set the first 50 values of beta and gamma to 1 - beta[:50] = 1 - gamma[:50] = 1 - - # Compute the outcome Y - U = np.random.normal(0, 1, n) # Error term - linear_term = np.dot(X, beta) - quadratic_term = np.dot(X**2, gamma) - - # Outcome equation - Y = 5 * D + linear_term + quadratic_term + U - - return X, D, Y - - n = 1000 # Sample size - X, D, Y = generate_data(n) - -Then, let's build an empirical cumulative distribution function (CDF). - -.. code-block:: python - - import dte_adj - from dte_adj.plot import plot - - estimator = dte_adj.SimpleDistributionEstimator() - estimator.fit(X, D, Y) - locations = np.linspace(Y.min(), Y.max(), 20) - cdf = estimator.predict(1, locations) - -Distributional treatment effect (DTE) can be computed easily in the following code. - -.. code-block:: python - - dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="simple") - -A convenience function is available to visualize distribution effects. This method can be used for other distribution parameters including Probability Treatment Effect (PTE) and Quantile Treatment Effect (QTE). - -.. code-block:: python - - plot(locations, dte, lower_bound, upper_bound, title="DTE of simple estimator") - -.. image:: _static/dte_empirical.png - :alt: DTE of empirical estimator - :height: 300px - :width: 450px - :align: center - -To initialize the adjusted distribution function, the base model for conditional distribution function needs to be passed. -In the following example, Logistic Regression is used. Please make sure that your base model implements ``fit`` and ``predict_proba`` methods. - -.. code-block:: python - - from sklearn.linear_model import LogisticRegression - logit = LogisticRegression() - estimator = dte_adj.AdjustedDistributionEstimator(logit, folds=3) - estimator.fit(X, D, Y) - cdf = estimator.predict(1, locations) - -DTE can be computed and visualized in the following code. - -.. code-block:: python - - dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="simple") - plot(locations, dte, lower_bound, upper_bound, title="DTE of adjusted estimator with simple confidence band") - -.. image:: _static/dte_simple.png - :alt: DTE of adjusted estimator with simple confidence band - :height: 300px - :width: 450px - :align: center - -Confidence bands can be computed in different ways. In the following code, moment condition is used to calculate the confidence band. - -.. code-block:: python - - dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="moment") - plot(locations, dte, lower_bound, upper_bound, title="DTE of adjusted estimator with moment confidence band") - -.. image:: _static/dte_moment.png - :alt: DTE of adjusted estimator with moment confidence band - :height: 300px - :width: 450px - :align: center - -Also, an uniform confidence band is used when ``uniform`` is specified for the ``variance_type`` argument. - -.. code-block:: python - - dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="uniform") - plot(locations, dte, lower_bound, upper_bound, title="DTE of adjusted estimator with uniform confidence band") - -.. image:: _static/dte_uniform.png - :alt: DTE of adjusted estimator with uniform confidence band - :height: 300px - :width: 450px - :align: center - -To compute PTE, you can use ``predict_pte`` method. The ``locations`` parameter defines interval boundaries, and the method returns probability treatment effects for each interval. -For each interval, the starting point is not included but the ending point is included. For example, if the `locations` is [0, 1, 2], PTE is computed for `(0, 1]` and `(1, 2]`. - -.. code-block:: python - - pte, lower_bound, upper_bound = estimator.predict_pte(target_treatment_arm=1, control_treatment_arm=0, locations=locations, variance_type="simple") - # Note: pte will have shape (len(locations)-1,) since it computes intervals between locations - plot(locations[:-1], pte, lower_bound, upper_bound, chart_type="bar", title="PTE of adjusted estimator with simple confidence band") - -.. image:: _static/pte_empirical.png - :alt: PTE of adjusted estimator with simple confidence band - :height: 300px - :width: 450px - :align: center - -To compute QTE, you can use ``predict_qte`` method. The confidence band is computed by bootstrap method. - -.. code-block:: python - - quantiles = np.array([0.1 * i for i in range(1, 10)], dtype=np.float32) - qte, lower_bound, upper_bound = estimator.predict_qte(target_treatment_arm=1, control_treatment_arm=0, quantiles=quantiles, n_bootstrap=30) - plot(quantiles, qte, lower_bound, upper_bound, title="QTE of adjusted estimator") - -.. image:: _static/qte.png - :alt: QTE of adjusted estimator - :height: 300px - :width: 450px - :align: center - -You can use any model with ``predict_proba`` or ``predict`` method to adjust the distribution function estimation. -For example, the following code use XGBoost classifier to estimate the conditional distribution. - -.. code-block:: python - - import xgboost as xgb - estimator = dte_adj.AdjustedDistributionEstimator(xgb.XGBClassifier(), folds=3) - estimator.fit(X, D, Y) - cdf = estimator.predict(1, locations) - -``predict_dte`` and ``predict_pte`` methods provide an option to train a model for multiple locations simultaneously. -To enable the feature, pass ``is_multi_task=True``. - -.. code-block:: python - - from sklearn.linear_model import LinearRegression - model = LinearRegression() - estimator = dte_adj.AdjustedDistributionEstimator(model, folds=3) - estimator.fit(X, D, Y) - dte, lower_bound, upper_bound = estimator.predict_dte(target_treatment_arm=1, control_treatment_arm=0, is_multi_task=True, locations=locations, variance_type="moment") diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 806a87a..0000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,62 +0,0 @@ -dte_adj -======= - -A Python Package for Estimating Distribution Treatment Effects --------------------------------------------------------------- - -`dte_adj` is a Python package for estimating distribution treatment effects in randomized experiments. -It provides APIs for conducting regression adjustment to estimate precise distribution functions, enabling deeper insights beyond average treatment effects through machine learning-enhanced estimation methods. - -.. image:: _static/distributional_effect.png - :alt: distributional effect - :height: 400px - :width: 700px - :align: center - -Estimator Types ---------------- - -The package provides several types of estimators for computing distribution treatment effects: - -* **Simple Randomization Estimators**: For estimating distributional effects in simple randomized experiments where treatment assignment is independent of all covariates -* **Covariate Adaptive Randomization Estimators**: For estimating distributional effects under covariate-adaptive randomization (CAR) designs, including stratified block randomization and other adaptive schemes -* **Local Distribution Estimators**: For estimating local distribution treatment effects weighted by treatment propensity within strata - -Theoretical Foundations ------------------------ - -For theoretical foundations, see: - -* **Simple randomization**: Byambadalai et al. (2024) [#simple2024]_ -* **Covariate-adaptive randomization**: Byambadalai et al. (2025) [#car2025]_ -* **Multi-task learning**: Hirata et al. (2025) [#multitask2025]_ -* **Imperfect compliance**: Byambadalai et al. (2025) [#compliance2025]_ - -.. [#simple2024] Byambadalai, U., Oka, T., & Yasui, S. (2024). Estimating Distributional Treatment Effects in Randomized Experiments: Machine Learning for Variance Reduction. In Proceedings of the 41st International Conference on Machine Learning (ICML'24). `arXiv:2407.16037 `_. - -.. [#car2025] Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). On Efficient Estimation of Distributional Treatment Effects under Covariate-Adaptive Randomization. In Proceedings of the 42nd International Conference on Machine Learning (ICML'25). `arXiv:2506.05945 `_. - -.. [#multitask2025] Hirata, T., Byambadalai, U., Oka, T., Yasui, S., & Uto, S. (2025). Efficient and Scalable Estimation of Distributional Treatment Effects with Multi-Task Neural Networks. arXiv preprint `arXiv:2507.07738 `_. - -.. [#compliance2025] Byambadalai, U., Hirata, T., Oka, T., & Yasui, S. (2025). Beyond the Average: Distributional Causal Inference under Imperfect Compliance. arXiv preprint `arXiv:2509.15594 `_. - -.. toctree:: - :maxdepth: 1 - :caption: Contents: - - installation - get_started - tutorials - api_reference - contributing - -Indices and tables -~~~~~~~~~~~~~~~~~~ - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - -License -~~~~~~~ -MIT License diff --git a/docs/source/installation.rst b/docs/source/installation.rst deleted file mode 100644 index f71ff6b..0000000 --- a/docs/source/installation.rst +++ /dev/null @@ -1,30 +0,0 @@ -Installation Guide -================== - -This package can be installed either through PyPI or source code. - -Requirement -~~~~~~~~~~~ - -You need to use Python version 3.10 or higher to use this package. - - -Install from PyPI -~~~~~~~~~~~~~~~~~ - -For installing the package from PyPI, please use the following command. - -.. code-block:: bash - - pip install dte_adj - -Install from source code -~~~~~~~~~~~~~~~~~~~~~~~~ - -For installing the package from the source code, please use the following commands. - -.. code-block:: bash - - git clone https://github.com/CyberAgentAILab/python-dte-adjustment - cd python-dte-adjustment - pip install -e . diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst deleted file mode 100644 index 4f1dc02..0000000 --- a/docs/source/tutorials.rst +++ /dev/null @@ -1,15 +0,0 @@ -Tutorials -========= - -This section provides comprehensive tutorials showing how to use the ``dte_adj`` library to analyze distributional treatment effects in famous randomized control trials. These examples demonstrate the power of looking beyond average treatment effects to understand how interventions affect entire outcome distributions. - -Available Tutorials -------------------- - -.. toctree:: - :maxdepth: 1 - - tutorials/hillstrom - tutorials/oregon - -The tutorials demonstrate practical applications of the ``dte_adj`` library using real-world datasets from famous randomized experiments. Each tutorial provides complete code examples, visualizations, and interpretations of distributional treatment effects. diff --git a/docs/source/tutorials/hillstrom.rst b/docs/source/tutorials/hillstrom.rst deleted file mode 100644 index 6c437e5..0000000 --- a/docs/source/tutorials/hillstrom.rst +++ /dev/null @@ -1,662 +0,0 @@ -Hillstrom Email Marketing Experiment -==================================== - -The Hillstrom email marketing dataset is a classic example from digital marketing, involving 64,000 customers randomly assigned to receive either a men's merchandise email, women's merchandise email, or no email (control). This experiment allows us to examine which email campaign strategy is most effective using revenue as the outcome. - -**Background**: Kevin Hillstrom provided this dataset to demonstrate email marketing analytics. Customers who purchased within the last 12 months were randomly divided into three groups to test targeted email campaigns against a control group. - -**Research Question**: Which email campaign performed best: the men's version or the women's version, and how do the effects vary across the revenue distribution? - -Data Setup and Loading -~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - import numpy as np - import pandas as pd - import matplotlib.pyplot as plt - from sklearn.linear_model import LinearRegression - from sklearn.preprocessing import LabelEncoder - import dte_adj - from dte_adj.plot import plot - - # Load the real Hillstrom dataset - url = "http://www.minethatdata.com/Kevin_Hillstrom_MineThatData_E-MailAnalytics_DataMiningChallenge_2008.03.20.csv" - df = pd.read_csv(url) - - print(f"Dataset shape: {df.shape}") - print(f"Average spend by segment:\n{df.groupby('segment')['spend'].mean()}") - - # Prepare the data for dte_adj analysis - # Create treatment indicator: 0=No E-Mail, 1=Mens E-Mail, 2=Women E-Mail - treatment_mapping = {'No E-Mail': 0, 'Mens E-Mail': 1, 'Women E-Mail': 2} - D = df['segment'].map(treatment_mapping).values - - # Use spend as the outcome variable (revenue) - revenue = df['spend'].values - - zip_code_mapping = {'Surburban': 0, 'Rural': 1, 'Urban': 2} # Note: typo in original data - channel_mapping = {'Phone': 0, 'Web': 1, 'Multichannel': 2} - - # Create feature matrix - features = pd.DataFrame({ - 'recency': df['recency'], - 'history': df['history'], - 'history_segment': df['history_segment'].map(lambda s: int(s[0])), - 'mens': df['mens'], - 'womens': df['womens'], - 'zip_code': df['zip_code'].map(zip_code_mapping), - 'newbie': df['newbie'], - 'channel': df['channel'].map(channel_mapping) - }) - - X = features.values - - print(f"\nDataset size: {len(D):,} customers") - print(f"Control group (No Email): {(D==0).sum():,} ({(D==0).mean():.1%})") - print(f"Men's Email group: {(D==1).sum():,} ({(D==1).mean():.1%})") - print(f"Women's Email group: {(D==2).sum():,} ({(D==2).mean():.1%})") - print("Average Spend by Treatment:") - print(f"No Email: ${revenue[D==0].mean():.2f}") - print(f"Men's Email: ${revenue[D==1].mean():.2f}") - print(f"Women's Email: ${revenue[D==2].mean():.2f}") - - # Also show conversion rates - print("\nConversion Rates:") - print(f"No Email: {df[df['segment']=='No E-Mail']['conversion'].mean():.3f}") - print(f"Men's Email: {df[df['segment']=='Mens E-Mail']['conversion'].mean():.3f}") - print(f"Women's Email: {df[df['segment']=='Women E-Mail']['conversion'].mean():.3f}") - -Email Campaign Effectiveness Analysis -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - # Initialize estimators - simple_estimator = dte_adj.SimpleDistributionEstimator() - ml_estimator = dte_adj.AdjustedDistributionEstimator( - LinearRegression(), - folds=5 - ) - - # Fit estimators on the full dataset - simple_estimator.fit(X, D, revenue) - ml_estimator.fit(X, D, revenue) - - # Define revenue evaluation points - revenue_locations = np.linspace(0, 500, 51) - -Women's Email vs Control Analysis -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -First, let's examine how the Women's email campaign performs compared to no email (control): - -.. code-block:: python - - # Compute DTE: Women's email vs Control - dte_women_ctrl, lower_women_ctrl, upper_women_ctrl = simple_estimator.predict_dte( - target_treatment_arm=2, # Women's email - control_treatment_arm=0, # No email control - locations=revenue_locations, - variance_type="moment" - ) - - # Visualize Women's vs Control using dte_adj's plot function - plot(revenue_locations, dte_women_ctrl, lower_women_ctrl, upper_women_ctrl, - title="Women's Email Campaign vs Control", - xlabel="Spending ($)", ylabel="Distribution Treatment Effect") - -Men's Email vs Control Analysis -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Next, let's examine how the Men's email campaign performs compared to no email (control): - -.. code-block:: python - - # Compute DTE: Men's email vs Control - dte_men_ctrl, lower_men_ctrl, upper_men_ctrl = simple_estimator.predict_dte( - target_treatment_arm=1, # Men's email - control_treatment_arm=0, # No email control - locations=revenue_locations, - variance_type="moment" - ) - - # Visualize Men's vs Control using dte_adj's plot function - plot(revenue_locations, dte_men_ctrl, lower_men_ctrl, upper_men_ctrl, - title="Men's Email Campaign vs Control", - xlabel="Spending ($)", ylabel="Distribution Treatment Effect", color="purple") - -.. image:: ../_static/hillstorm_dte_control.png - :alt: Hillstrom Email Campaigns vs Control Analysis - :width: 500px - :align: center - -Spending Category Effects: Each Campaign vs Control -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Let's also examine how each campaign affects spending in specific intervals using Probability Treatment Effects (PTE): - -.. code-block:: python - - # Compute PTE: Women's email vs Control - pte_women_ctrl, pte_lower_women_ctrl, pte_upper_women_ctrl = simple_estimator.predict_pte( - target_treatment_arm=2, # Women's email - control_treatment_arm=0, # No email control - locations=np.insert(revenue_locations, 0, -1), - variance_type="moment" - ) - - # Compute PTE: Men's email vs Control - pte_men_ctrl, pte_lower_men_ctrl, pte_upper_men_ctrl = simple_estimator.predict_pte( - target_treatment_arm=1, # Men's email - control_treatment_arm=0, # No email control - locations=np.insert(revenue_locations, 0, -1), - variance_type="moment" - ) - - # Visualize PTE results using dte_adj's plot function with bar charts side by side - import matplotlib.pyplot as plt - - # Create subplots for side-by-side comparison - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - # Women's vs Control PTE - plot(revenue_locations[1:], pte_women_ctrl, pte_lower_women_ctrl, pte_upper_women_ctrl, - chart_type="bar", - title="Women's Email vs Control", - xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", - ax=ax1) - - # Men's vs Control PTE - plot(revenue_locations[1:], pte_men_ctrl, pte_lower_men_ctrl, pte_upper_men_ctrl, - chart_type="bar", - title="Men's Email vs Control", - xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", - color="purple", ax=ax2) - - plt.tight_layout() - plt.show() - -The side-by-side PTE analysis produces the following visualization: - -.. image:: ../_static/hillstorm_pte_control.png - :alt: Hillstrom Email Campaigns vs Control PTE Analysis - :width: 800px - :align: center - -These bar charts show how each email campaign affects the probability of customers spending in specific intervals compared to no email: - -**Women's Email vs Control (Left Panel)**: The bar chart reveals specific spending intervals where women's email campaigns increase or decrease customer probability. Positive bars indicate intervals where the campaign increases the likelihood of spending in that range, while negative bars show intervals where it decreases probability. - -**Men's Email vs Control (Right Panel)**: Similarly shows the interval-specific effects of men's email campaigns. The side-by-side comparison allows direct assessment of which campaign is more effective in driving specific spending behaviors. - -**Key Insights from the Hillstrom Email Campaign Analysis**: - -The distributional treatment effects and probability treatment effects reveal several important patterns in how email campaigns affect customer spending behavior: - -1. **Email Campaigns Reduce Zero Spending**: Both men's and women's email campaigns show strong negative effects at the $0 spending level, indicating that email campaigns successfully convert non-purchasers into purchasers. This confirms that email marketing has a clear activation effect. - -2. **Spending Category Redistribution**: The PTE analysis reveals how campaigns redistribute customers across spending intervals. Both campaigns show clear redistribution patterns, with negative effects in the $0 spending category (reducing non-purchase probability) and varying effects across other spending ranges. This redistribution pattern confirms that campaigns shift spending behavior rather than uniformly increasing it across all categories. - -3. **Campaign-Specific Interval Effects**: The side-by-side PTE comparison reveals distinct patterns between campaigns. Women's email campaigns show stronger effects in certain spending intervals (particularly in moderate spending ranges), while men's campaigns display different interval-specific patterns. The visual comparison makes it clear that each campaign has optimal spending ranges where it excels, providing actionable insights for customer segmentation and targeting strategies. - -4. **Statistical Significance Across Intervals**: The confidence intervals in both DTE and PTE analyses reveal that the most reliable effects occur at the zero spending level and in low-to-moderate spending ranges. PTE analysis provides additional granularity by showing which specific spending intervals have statistically significant changes in probability. - -5. **Business Implications**: Email campaigns are most effective at converting non-buyers to buyers and redistributing customers toward moderate purchase amounts. The campaigns have minimal effect on driving high-value purchases ($200+). The PTE analysis provides actionable insights for campaign optimization by identifying which spending intervals are most responsive to each campaign type, enabling more precise targeting and resource allocation strategies. - -This distributional analysis reveals that email marketing's primary value lies in customer activation (reducing zero spending) and encouraging moderate purchase amounts, rather than dramatically increasing high-value purchases. The heterogeneous effects across the spending distribution provide actionable insights for optimizing email campaign strategies and customer segmentation. - -Direct Campaign Comparison: Men's vs Women's Email -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Finally, let's directly compare the two email campaigns to answer the key research question. -This time we estimate the distribution treatment effects with regression adjustment using linear regression for higher precision. - -.. code-block:: python - - # Compute DTE: Women's vs Men's email campaigns - dte_simple, lower_simple, upper_simple = simple_estimator.predict_dte( - target_treatment_arm=2, # Women's email - control_treatment_arm=1, # Men's email (as "control") - locations=revenue_locations, - variance_type="moment" - ) - - dte_ml, lower_ml, upper_ml = ml_estimator.predict_dte( - target_treatment_arm=2, # Women's email - control_treatment_arm=1, # Men's email - locations=revenue_locations, - variance_type="moment" - ) - - # Visualize the distribution treatment effects using dte_adj's built-in plot function - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - # Simple estimator - plot(revenue_locations, dte_simple, lower_simple, upper_simple, - title="Email Campaign Comparison: Women's vs Men's (Simple Estimator)", - xlabel="Spending ($)", ylabel="Distribution Treatment Effect", - color="purple", - ax=ax1) - - # ML-adjusted estimator - plot(revenue_locations, dte_ml, lower_ml, upper_ml, - title="Email Campaign Comparison: Women's vs Men's (ML-Adjusted Estimator)", - xlabel="Spending ($)", ylabel="Distribution Treatment Effect", - ax=ax2) - - plt.tight_layout() - plt.show() - -The analysis produces the following distribution treatment effects visualization: - -.. image:: ../_static/hillstorm_dte.png - :alt: Hillstrom Email Marketing DTE Analysis - :width: 800px - :align: center - -The side-by-side plots show the distribution treatment effects (DTE) comparing Women's vs Men's email campaigns across different spending levels. Key observations: - -**DTE Interpretation**: The predominantly positive DTE values indicate that women's campaigns increase the cumulative probability of customers spending at or below each threshold compared to men's campaigns. This means women's campaigns result in more customers having lower revenue levels, which is unfavorable for business outcomes. - -**Men's Campaign Superiority**: The statistical significance of positive DTE values across most spending levels provides strong evidence that men's campaigns outperform women's campaigns by reducing the probability of customers spending small amounts and encouraging higher revenue per customer. - -**Business Implication**: The DTE analysis clearly demonstrates that men's campaigns are superior for revenue maximization, as they consistently reduce the cumulative probability of low spending levels, effectively shifting customers toward higher revenue categories. - -Revenue Category Analysis with PTE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - # Compute Probability Treatment Effects - pte_simple, pte_lower_simple, pte_upper_simple = simple_estimator.predict_pte( - target_treatment_arm=1, # Women's email - control_treatment_arm=0, # Men's email - locations=np.insert(revenue_locations, 0, -1), - variance_type="moment" - ) - - pte_ml, pte_lower_ml, pte_upper_ml = ml_estimator.predict_pte( - target_treatment_arm=1, # Women's email - control_treatment_arm=0, # Men's email - locations=np.insert(revenue_locations, 0, -1), - variance_type="moment" - ) - - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - # Simple estimator - plot(revenue_locations[1:], pte_simple, pte_lower_simple, pte_upper_simple, - chart_type="bar", - title="Spending Category Effects: Women's vs Men's (Simple Estimator)", - xlabel="Spending Category", ylabel="Probability Treatment Effect", color="purple", - ax=ax1) - - # ML-adjusted estimator - plot(revenue_locations[1:], pte_ml, pte_lower_ml, pte_upper_ml, - chart_type="bar", - title="Spending Category Effects: Women's vs Men's (ML-Adjusted Estimator)", - xlabel="Spending Category", ylabel="Probability Treatment Effect", - ax=ax2) - plt.tight_layout() - plt.show() - -The Probability Treatment Effects analysis produces the following visualization: - -.. image:: ../_static/hillstorm_pte.png - :alt: Hillstrom Email Marketing PTE Analysis - :width: 800px - :align: center - -The side-by-side bar charts show probability treatment effects across different spending intervals, revealing the true story of campaign effectiveness: - -**Critical Finding - Zero Revenue Effect**: Women's campaigns show a positive effect in the $0 revenue category, meaning they increase the probability of customers making no purchase compared to men's campaigns. This is a negative outcome indicating that women's campaigns are less effective at driving any purchase behavior. - -**Spending Category Analysis**: Men's campaigns demonstrate superior performance in driving actual revenue. The negative PTE values in revenue-generating categories for women's campaigns indicate that men's campaigns are more effective at encouraging customers to make purchases and spend meaningful amounts. - -**Revenue Generation Patterns**: Men's campaigns show stronger performance in categories that generate actual revenue, while women's campaigns appear to be associated with higher non-purchase rates. This pattern suggests that men's campaigns are more effective at converting prospects into paying customers. - -**Methodological Confirmation**: Both simple and ML-adjusted estimators confirm this pattern, with the ML-adjusted analysis providing more precise estimates that strengthen the evidence for men's campaign superiority in driving revenue-generating behavior. - -**Strategic Implications**: Men's campaigns should be prioritized for revenue generation and customer conversion goals, as they demonstrate superior ability to drive actual purchases rather than just engagement. - -**Key PTE Findings**: - -1. **Men's Campaigns Drive More Purchases**: The critical finding is that women's campaigns increase the probability of zero revenue (non-purchase) compared to men's campaigns. This means men's campaigns are more effective at converting prospects into paying customers. - -2. **Revenue Generation Superiority**: Men's campaigns show consistently better performance in revenue-generating categories. The negative PTE values for women's campaigns in spending intervals indicate that men's campaigns drive more customers to make actual purchases across most revenue ranges. - -3. **Quantified Business Impact**: The analysis reveals that men's campaigns reduce non-purchase rates and increase the probability of revenue generation. Switching from women's to men's campaigns could improve overall conversion rates and revenue per customer. - -4. **Statistical Significance**: The statistical significance of the zero-revenue effect for women's campaigns provides strong evidence that men's campaigns are superior for business outcomes focused on revenue generation rather than just engagement. - -**Conclusion**: Using the real Hillstrom dataset with 64,000 customers, the distributional analysis reveals nuanced patterns in how email campaigns affect customer spending. The analysis goes beyond simple average comparisons to show how treatment effects vary across the entire spending distribution, providing insights into which customer segments respond best to different campaign types. This demonstrates the power of distribution treatment effect analysis for understanding heterogeneous responses in digital marketing experiments. - -Subgroup Analysis by Purchase History -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Beyond comparing email campaigns overall, we can examine how campaign effectiveness varies by customer purchase history. This analysis segments customers based on their past purchasing behavior: - -- **Men's merchandise purchasers** (``mens=1``): Customers who previously purchased men's merchandise (35,266 customers, 55.1%) -- **Women's merchandise purchasers** (``womens=1``): Customers who previously purchased women's merchandise (35,182 customers, 55.0%) - -Note that these segments overlap (6,448 customers purchased both categories), so a customer can appear in both analyses. - -**Research Question**: Does the effectiveness of men's vs women's email campaigns vary by the type of merchandise customers have historically purchased? - -Defining Subgroups -^^^^^^^^^^^^^^^^^^^ - -.. code-block:: python - - # Define subgroup masks based on purchase history - mens_purchasers = (df['mens'] == 1) - womens_purchasers = (df['womens'] == 1) - - print(f"Men's merchandise purchaser segment: {mens_purchasers.sum():,} customers") - print(f"Women's merchandise purchaser segment: {womens_purchasers.sum():,} customers") - print(f"Overlap: {(mens_purchasers & womens_purchasers).sum():,} customers") - -Average Treatment Effects by Subgroup -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Let's first compute the average treatment effects (ATEs) to quantify the overall impact: - -.. code-block:: python - - # Compute ATEs for each campaign-subgroup combination - # Women's Email Campaign - ate_women_male = (revenue[(D==2) & mens_purchasers].mean() - - revenue[(D==0) & mens_purchasers].mean()) - ate_women_female = (revenue[(D==2) & womens_purchasers].mean() - - revenue[(D==0) & womens_purchasers].mean()) - - # Men's Email Campaign - ate_men_male = (revenue[(D==1) & mens_purchasers].mean() - - revenue[(D==0) & mens_purchasers].mean()) - ate_men_female = (revenue[(D==1) & womens_purchasers].mean() - - revenue[(D==0) & womens_purchasers].mean()) - - print("Average Treatment Effects by Subgroup:") - print("\nWomen's Email Campaign:") - print(f" Men's Merch. Purchasers: ATE = ${ate_women_male:.4f}") - print(f" Women's Merch. Purchasers: ATE = ${ate_women_female:.4f}") - print("\nMen's Email Campaign:") - print(f" Men's Merch. Purchasers: ATE = ${ate_men_male:.4f}") - print(f" Women's Merch. Purchasers: ATE = ${ate_men_female:.4f}") - -Expected output:: - - Average Treatment Effects by Subgroup: - - Women's Email Campaign: - Men's Merch. Purchasers: ATE = $0.2564 - Women's Merch. Purchasers: ATE = $0.5442 - - Men's Email Campaign: - Men's Merch. Purchasers: ATE = $0.8966 - Women's Merch. Purchasers: ATE = $0.8412 - -These results reveal important patterns: - -- **Women's Email Campaign**: Shows 2× stronger effect for women's merchandise purchasers ($0.54) vs men's merchandise purchasers ($0.26) -- **Men's Email Campaign**: Demonstrates consistent strong effects across both segments ($0.84-$0.89) - -While these averages provide a useful summary, they don't tell us *how* customer spending distributions change. The distributional and probability treatment effect analyses that follow reveal the complete picture of campaign effectiveness. - -Distribution Treatment Effects: Women's Email Campaign -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Beyond the average effects, let's examine how the Women's Email campaign shifts the entire spending distribution for each subgroup: - -.. code-block:: python - - # Analyze men's merchandise purchaser segment - estimator_male = dte_adj.SimpleDistributionEstimator() - estimator_male.fit(X[mens_purchasers], D[mens_purchasers], revenue[mens_purchasers]) - - # Analyze women's merchandise purchaser segment - estimator_female = dte_adj.SimpleDistributionEstimator() - estimator_female.fit(X[womens_purchasers], D[womens_purchasers], revenue[womens_purchasers]) - - # Define evaluation points - locations = np.linspace(0, 500, 51) - - # Compute DTE for Women's Email vs Control in each subgroup - dte_women_male, lower_women_male, upper_women_male = estimator_male.predict_dte( - target_treatment_arm=2, # Women's Email - control_treatment_arm=0, # No Email - locations=locations, - variance_type="moment" - ) - - dte_women_female, lower_women_female, upper_women_female = estimator_female.predict_dte( - target_treatment_arm=2, # Women's Email - control_treatment_arm=0, # No Email - locations=locations, - variance_type="moment" - ) - - # Visualize side-by-side - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - plot(locations, dte_women_male, lower_women_male, upper_women_male, - title="Women's Email vs Control\nMen's Merch. Purchasers", - xlabel="Spending ($)", ylabel="Distribution Treatment Effect", - color="purple", ax=ax1) - ax1.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) - - plot(locations, dte_women_female, lower_women_female, upper_women_female, - title="Women's Email vs Control\nWomen's Merch. Purchasers", - xlabel="Spending ($)", ylabel="Distribution Treatment Effect", - color="green", ax=ax2) - ax2.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) - - plt.tight_layout() - plt.show() - -.. image:: ../_static/hillstorm_subgroup_women_dte.png - :alt: Women's Email Campaign Subgroup Analysis - :width: 800px - :align: center - -**Key Finding for Women's Email Campaign**: The distributional treatment effects reveal that women's email campaigns are significantly more effective for the women's merchandise purchaser segment (right panel) compared to the men's merchandise purchaser segment (left panel). The DTE curves show that women's emails reduce the probability of low spending levels (negative DTE at lower thresholds) for women's merchandise purchasers, indicating a shift toward higher spending. In contrast, the men's merchandise purchaser segment shows minimal or non-significant effects across most of the spending distribution, with confidence intervals overlapping zero. - -Distribution Treatment Effects: Men's Email Campaign -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Now let's examine how the Men's Email campaign affects spending distributions: - -.. code-block:: python - - # Compute DTE for Men's Email vs Control in each subgroup - dte_men_male, lower_men_male, upper_men_male = estimator_male.predict_dte( - target_treatment_arm=1, # Men's Email - control_treatment_arm=0, # No Email - locations=locations, - variance_type="moment" - ) - - dte_men_female, lower_men_female, upper_men_female = estimator_female.predict_dte( - target_treatment_arm=1, # Men's Email - control_treatment_arm=0, # No Email - locations=locations, - variance_type="moment" - ) - - # Visualize side-by-side - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - plot(locations, dte_men_male, lower_men_male, upper_men_male, - title="Men's Email vs Control\nMen's Merch. Purchasers", - xlabel="Spending ($)", ylabel="Distribution Treatment Effect", - color="purple", ax=ax1) - ax1.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) - - plot(locations, dte_men_female, lower_men_female, upper_men_female, - title="Men's Email vs Control\nWomen's Merch. Purchasers", - xlabel="Spending ($)", ylabel="Distribution Treatment Effect", - color="green", ax=ax2) - ax2.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) - - plt.tight_layout() - plt.show() - -.. image:: ../_static/hillstorm_subgroup_men_dte.png - :alt: Men's Email Campaign Subgroup Analysis - :width: 800px - :align: center - -**Key Finding for Men's Email Campaign**: In contrast to women's email campaigns, men's email campaigns show consistent effectiveness across both purchase history segments. The DTE curves in both panels show similar patterns, with negative values at lower spending levels indicating reduced probability of low spending for both male and women's merchandise purchasers. This suggests that men's emails have broad appeal regardless of whether customers historically purchased men's or women's merchandise. - -Probability Treatment Effects: Women's Email Campaign -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -While DTE shows how cumulative distributions shift, Probability Treatment Effects (PTE) reveal which specific spending intervals are most affected by the campaign. PTE measures the change in probability mass within each spending category: - -.. code-block:: python - - # Compute PTE for Women's Email vs Control in each subgroup - pte_locations = np.insert(locations, 0, -1) # Add -1 at beginning for intervals - - pte_women_male, pte_lower_women_male, pte_upper_women_male = estimator_male.predict_pte( - target_treatment_arm=2, # Women's Email - control_treatment_arm=0, # No Email - locations=pte_locations, - variance_type="moment" - ) - - pte_women_female, pte_lower_women_female, pte_upper_women_female = estimator_female.predict_pte( - target_treatment_arm=2, # Women's Email - control_treatment_arm=0, # No Email - locations=pte_locations, - variance_type="moment" - ) - - # Visualize side-by-side with bar charts - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - plot(locations, pte_women_male, pte_lower_women_male, pte_upper_women_male, - chart_type="bar", - title="Women's Email vs Control\nMen's Merch. Purchasers", - xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", - color="purple", ax=ax1) - ax1.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) - - plot(locations, pte_women_female, pte_lower_women_female, pte_upper_women_female, - chart_type="bar", - title="Women's Email vs Control\nWomen's Merch. Purchasers", - xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", - color="green", ax=ax2) - ax2.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) - - plt.tight_layout() - plt.show() - -.. image:: ../_static/hillstorm_subgroup_women_pte.png - :alt: Women's Email Campaign PTE Subgroup Analysis - :width: 800px - :align: center - -**Interval-Specific Insights**: The PTE bar charts reveal the mechanism behind the average treatment effect. For women's merchandise purchasers (right panel), women's emails significantly reduce the probability of zero spending (non-purchasers converting to purchasers), which is the primary driver of the positive ATE. However, no significant increase in high spending categories is observed. For men's merchandise purchasers (left panel), the effects are much smaller and less consistent, confirming the limited impact suggested by the ATE and DTE analyses. - -Probability Treatment Effects: Men's Email Campaign -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Let's examine which spending categories are most affected by men's email campaigns: - -.. code-block:: python - - # Compute PTE for Men's Email vs Control in each subgroup - pte_men_male, pte_lower_men_male, pte_upper_men_male = estimator_male.predict_pte( - target_treatment_arm=1, # Men's Email - control_treatment_arm=0, # No Email - locations=pte_locations, - variance_type="moment" - ) - - pte_men_female, pte_lower_men_female, pte_upper_men_female = estimator_female.predict_pte( - target_treatment_arm=1, # Men's Email - control_treatment_arm=0, # No Email - locations=pte_locations, - variance_type="moment" - ) - - # Visualize side-by-side with bar charts - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - plot(locations, pte_men_male, pte_lower_men_male, pte_upper_men_male, - chart_type="bar", - title="Men's Email vs Control\nMen's Merch. Purchasers", - xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", - color="purple", ax=ax1) - ax1.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) - - plot(locations, pte_men_female, pte_lower_men_female, pte_upper_men_female, - chart_type="bar", - title="Men's Email vs Control\nWomen's Merch. Purchasers", - xlabel="Spending Category ($)", ylabel="Probability Treatment Effect", - color="green", ax=ax2) - ax2.axhline(y=0, color='black', linestyle='--', linewidth=0.8, alpha=0.5) - - plt.tight_layout() - plt.show() - -.. image:: ../_static/hillstorm_subgroup_men_pte.png - :alt: Men's Email Campaign PTE Subgroup Analysis - :width: 800px - :align: center - -**Interval-Specific Insights**: Men's email campaigns show similar PTE patterns across both segments (left and right panels). The key mechanism is twofold: (1) significant reduction in zero spending probability (converting non-purchasers to purchasers), and (2) increased probability in the $40-100 spending range. This dual effect—both purchase conversion and mid-range spending increases—occurs consistently across both male and women's merchandise purchaser segments, confirming the broad effectiveness of men's campaigns. - -Key Insights from Subgroup Analysis -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Combining Average Treatment Effects (ATE), Distribution Treatment Effects (DTE), and Probability Treatment Effects (PTE) provides a comprehensive understanding of campaign effectiveness: - -**1. Campaign Targeting Effectiveness (from ATE)** - -- Women's email campaigns show 2× stronger average effects for women's merchandise purchasers ($0.54) vs men's merchandise purchasers ($0.26) -- Men's email campaigns demonstrate consistent strong effects across both segments ($0.89-$0.84) -- This suggests women's campaigns benefit from precise targeting, while men's campaigns have broader appeal - -**2. Distributional Shifts Beyond Averages (from DTE)** - -- For women's emails, the women's merchandise purchaser segment shows negative DTE at lower spending thresholds, indicating a systematic shift away from low-spending behavior -- Men's merchandise purchasers show minimal distributional changes from women's emails, with confidence intervals overlapping zero at most thresholds -- Men's emails produce similar distributional patterns across both segments, confirming broad effectiveness - -**3. Spending Category Changes (from PTE)** - -- PTE analysis reveals *which specific spending intervals* change in response to campaigns, particularly identifying the mechanisms behind average effects -- **Women's emails**: For women's merchandise purchasers, the primary effect is converting non-purchasers to purchasers (significant reduction in zero spending probability). No significant increase in high spending categories was observed. -- **Men's emails**: Show a dual mechanism across both segments: (1) converting non-purchasers to purchasers (zero spending reduction), and (2) increasing purchases in the $40-100 range -- PTE enables identification of behavioral change mechanisms that are invisible in average treatment effects alone—specifically revealing that lift comes primarily from purchase conversion (0→1 effect) rather than spending increases among existing purchasers - -**4. Strategic Implications** - -Based on these findings, several practical implications emerge: - -- **For Women's Campaigns**: Target customers with history of purchasing women's merchandise to maximize ROI. The PTE analysis reveals that effectiveness comes primarily from converting non-purchasers to purchasers among women's merchandise purchaser segments, rather than increasing spending among existing buyers. -- **For Men's Campaigns**: Deploy broadly as they produce consistent positive effects across diverse customer segments. Both male and women's merchandise purchasers show both purchase conversion and mid-range spending increases, suggesting broader appeal. -- **Resource Allocation**: One practical implication is to prioritize precise targeting for gender-specific content (women's emails) but invest confidently in broad deployment for broadly appealing content (men's emails). - -**5. Methodological Value** - -This three-tier analysis demonstrates why distributional methods matter: - -- **ATE alone** would show that both campaigns have positive effects, but with varying magnitudes across subgroups -- **Adding DTE** reveals *how* spending distributions shift, not just average changes -- **Adding PTE** pinpoints *which spending categories* are most affected, enabling precise business decisions - -By examining effects at average, distributional, and interval-specific levels, we gain actionable insights that would be invisible to traditional mean-comparison approaches. This demonstrates the power of distribution treatment effect methods for understanding heterogeneous responses in digital marketing experiments. - -For the complete reproducible code including helper functions and visualizations, see `example/hillstrom.ipynb `_. - -Next Steps -~~~~~~~~~~ - -- Try with your own randomized experiment data -- Experiment with different ML models (XGBoost, Neural Networks) for adjustment -- Explore stratified estimators for covariate-adaptive randomization designs -- Use multi-task learning (``is_multi_task=True``) for computational efficiency with many locations diff --git a/docs/source/tutorials/oregon.rst b/docs/source/tutorials/oregon.rst deleted file mode 100644 index 0f17afe..0000000 --- a/docs/source/tutorials/oregon.rst +++ /dev/null @@ -1,630 +0,0 @@ -Oregon Health Insurance Experiment -==================================== - -The Oregon Health Insurance Experiment is a landmark randomized controlled trial conducted in 2008, where approximately 24,000 low-income adults were randomly assigned to either receive the opportunity to enroll in Medicaid (treatment group) or remain uninsured (control group). This unique natural experiment allows us to examine how public health insurance affects healthcare utilization and costs across the entire distribution. - -**Background**: Due to budget constraints, Oregon decided to expand its Medicaid program through a lottery system, randomly selecting eligible individuals for enrollment opportunities. This created a rare natural experiment with non-compliance (not all selected individuals enrolled) that enables rigorous causal evaluation using Local Distribution Treatment Effects (LDTE) methodology. - -**Research Question**: How does Medicaid assignment (and enrollment) affect healthcare utilization (emergency department visits and costs), accounting for non-compliance, and how do these effects vary across the entire distribution of healthcare outcomes? - -Data Setup and Loading -~~~~~~~~~~~~~~~~~~~~~~~ - -**Data Source**: The Oregon Health Insurance Experiment data used in this tutorial is publicly available through the National Bureau of Economic Research (NBER). You can download the dataset from the official NBER Public Use Data Archive at: https://www.nber.org/research/data/oregon-health-insurance-experiment-data - -The dataset includes multiple files containing information about participants in the experiment: - -- ``oregonhie_descriptive_vars.dta``: Demographic and baseline characteristics -- ``oregonhie_ed_vars.dta``: Emergency department utilization data -- ``oregonhie_inperson_vars.dta``: In-person survey responses -- ``oregonhie_stateprograms_vars.dta``: State program participation data - -This data supports research on how health insurance affects healthcare utilization and is maintained by researchers Amy Finkelstein and Katherine Baicker. Please ensure you comply with the data use agreements when downloading and using this dataset. - -Import Libraries -^^^^^^^^^^^^^^^^ - -First, we import the necessary libraries for data processing, analysis, and visualization: - -.. code-block:: python - - import numpy as np - import pandas as pd - import matplotlib.pyplot as plt - import os - from sklearn.linear_model import LinearRegression - from sklearn.preprocessing import LabelEncoder - import dte_adj - from dte_adj.plot import plot - -Load and Merge Datasets -^^^^^^^^^^^^^^^^^^^^^^^^ - -Next, we load the four separate data files and merge them into a single dataset: - -.. code-block:: python - - # Load the Oregon Health Insurance Experiment dataset - base_path = "OHIE_Public_Use_Files/OHIE_Data" - df_descriptive = pd.read_stata(os.path.join(base_path, "oregonhie_descriptive_vars.dta")) - df_ed = pd.read_stata(os.path.join(base_path, "oregonhie_ed_vars.dta")) - df_inp = pd.read_stata(os.path.join(base_path, "oregonhie_inperson_vars.dta")) - df_state = pd.read_stata(os.path.join(base_path, "oregonhie_stateprograms_vars.dta")) - - # Merge all datasets - df = ( - df_descriptive - .merge(df_ed, on='person_id', how='inner') - .merge(df_inp, on='person_id', how='left') - .merge(df_state, on='person_id', how='inner') - ) - - print(f"Dataset shape: {df.shape}") - print(f"Average num_visit_cens_ed by enrollment:\n{df.groupby('ohp_all_ever_inperson')['num_visit_cens_ed'].mean()}") - print(f"Average ed_charg_tot_ed by enrollment:\n{df.groupby('ohp_all_ever_inperson')['ed_charg_tot_ed'].mean()}") - -Data Preprocessing -^^^^^^^^^^^^^^^^^^ - -Next, we prepare the data for the DTE analysis. This involves creating treatment variables, encoding categorical features, and selecting control variables: - -.. code-block:: python - - # Create treatment assignment (instrumental variable): 0=Not selected, 1=Selected - treatment_assignment_mapping = {'Not selected': 0, 'Selected': 1} - df['Z'] = df['treatment'].map(treatment_assignment_mapping) - - # Create actual treatment indicator: 0=Not enrolled, 1=Enrolled - treatment_mapping = {'NOT enrolled': 0, 'Enrolled': 1} - df['D'] = df['ohp_all_ever_inperson'].map(treatment_mapping) - - # Create strata based on household size - df.rename(columns={'numhh_list': 'strata'}, inplace=True) - df['strata'] = df['strata'].astype(str).replace({ - 'signed self up + 1 additional person': 'signed self up + others', - 'signed self up + 2 additional people': 'signed self up + others' - }) - - # Create feature mappings for categorical variables - gender_mapping = {'Male': 0, 'Female': 1, 'Transgender F to M': 2, 'Transgender M to F': 3} - health_last12_mapping = {'1: Very poor': 1, '2: Poor': 2, '3: Fair': 3, '4: Good': 4, '5: Very good': 5, '6: Excellent': 6} - edu_mapping = {'HS diploma or GED': 0, 'Post HS, not 4-year': 1, 'Less than HS': 2, '4 year degree or more': 3} - - df['age'] = 2008 - df['birthyear_list'] - df['gender_inp'] = df['gender_inp'].map(gender_mapping).astype(float).fillna(-1).astype(int) - df['health_last12_inp'] = df['health_last12_inp'].map(health_last12_mapping).astype(float).fillna(-1).astype(int) - df['edu_inp'] = df['edu_inp'].map(edu_mapping).astype(float).fillna(-1).astype(int) - - # Select control variables: pre-randomization ED utilization variables - ctrl_cols = [col for col in df_ed.columns if 'pre' in col and 'num' in col] + ['gender_inp', 'age', 'health_last12_inp', 'edu_inp', 'charg_tot_pre_ed'] - selected_cols = ['person_id', 'strata', 'ed_charg_tot_ed', 'num_visit_cens_ed', 'Z', 'D'] + ctrl_cols - df = df[selected_cols] - df = df.dropna().reset_index(drop=True) - -Prepare Variables for Analysis -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Finally, we create the feature matrices and outcome variables needed for the Local Distribution Treatment Effect analysis: - -.. code-block:: python - - # Create feature matrix (excluding treatment variables) - X = df[ctrl_cols].values - - Z = df['Z'].astype(int).values # Treatment assignment (instrumental variable) - D = df['D'].astype(int).values # Actual treatment (endogenous variable) - strata = df['strata'].values # Stratification variable - - # Use num_visit_cens_ed and ed_charg_tot_ed as outcome variables - Y_ED_CHARG_TOT_ED = df['ed_charg_tot_ed'].values - Y_NUM_VISIT_CENS_ED = df['num_visit_cens_ed'].values - - print(f"\nDataset size: {len(D):,} people") - print(f"Treatment assignment (Z) - Not selected: {(Z==0).sum():,} ({(Z==0).mean():.1%})") - print(f"Treatment assignment (Z) - Selected: {(Z==1).sum():,} ({(Z==1).mean():.1%})") - print(f"Actual treatment (D) - Not enrolled: {(D==0).sum():,} ({(D==0).mean():.1%})") - print(f"Actual treatment (D) - Enrolled: {(D==1).sum():,} ({(D==1).mean():.1%})") - print("\nCompliance rate (among those assigned to treatment):") - print(f"Compliance rate: {(D[Z==1]==1).mean():.1%}") - print("\nAverage Outcome by Actual Treatment (D):") - print(f"Not enrolled (D=0): {Y_NUM_VISIT_CENS_ED[D==0].mean():.2f} visits, ${Y_ED_CHARG_TOT_ED[D==0].mean():.2f} in ED costs") - print(f"Enrolled (D=1): {Y_NUM_VISIT_CENS_ED[D==1].mean():.2f} visits, ${Y_ED_CHARG_TOT_ED[D==1].mean():.2f} in ED costs") - -Emergency Department Cost Analysis -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - # Initialize LOCAL estimators for non-compliance scenario - simple_local_estimator = dte_adj.SimpleLocalDistributionEstimator() - ml_local_estimator = dte_adj.AdjustedLocalDistributionEstimator( - LinearRegression(), - folds=5 - ) - - # Fit estimators: fit(covariates, treatment_arms, treatment_indicator, outcomes, strata) - # treatment_arms = Z (Treatment assignment), treatment_indicator = D (Actual treatment) - simple_local_estimator.fit(X, Z, D, Y_ED_CHARG_TOT_ED, strata) - ml_local_estimator.fit(X, Z, D, Y_ED_CHARG_TOT_ED, strata) - - # Define evaluation points for emergency department costs - outcome_ed_costs_locations = np.arange(Y_ED_CHARG_TOT_ED.min(), Y_ED_CHARG_TOT_ED.max(), 3000) - -Local Estimator Comparison: Simple vs ML-Adjusted (Costs) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Let's compare the results from both simple and machine learning-adjusted local estimators to examine the robustness of our findings: - -.. code-block:: python - - # Compute LDTE: Treatment vs Control - ldte_simple, lower_simple, upper_simple = simple_local_estimator.predict_ldte( - target_treatment_arm=1, # Z=1 Selected in lottery for Medicaid (treatment assignment) - control_treatment_arm=0, # Z=0 Not selected in lottery for Medicaid (control assignment) - locations=outcome_ed_costs_locations - ) - - ldte_ml, lower_ml, upper_ml = ml_local_estimator.predict_ldte( - target_treatment_arm=1, # Z=1 Selected in lottery for Medicaid (treatment assignment) - control_treatment_arm=0, # Z=0 Not selected in lottery for Medicaid (control assignment) - locations=outcome_ed_costs_locations - ) - - # Visualize the distribution treatment effects using dte_adj's built-in plot function - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - # Visualize Treatment vs Control using dte_adj's plot function - plot(outcome_ed_costs_locations, ldte_simple, lower_simple, upper_simple, - title="ED Costs: Treatment vs Control (Simple Local Estimator)", - xlabel="Emergency Department Costs", - ylabel="Local Distribution Treatment Effect", - color="purple", - ax=ax1) - - plot(outcome_ed_costs_locations, ldte_ml, lower_ml, upper_ml, - title="ED Costs: Treatment vs Control (ML-Adjusted Local Estimator)", - xlabel="Emergency Department Costs", - ylabel="Local Distribution Treatment Effect", - ax=ax2) - - plt.tight_layout() - plt.show() - -The analysis produces the following local distribution treatment effects visualization: - -.. image:: ../_static/oregon_ldte_costs_comparison.png - :alt: Oregon Health Insurance Experiment LDTE Analysis - :width: 800px - :align: center - -**1. LDTE Interpretation and Distribution-Level Insights** - -- **Simple Local Estimator**: Shows LDTE ≈ -0.12 at zero costs, meaning 12 percentage points fewer insured individuals have zero ED costs. The effect converges to zero around $10,000 and remains flat thereafter. -- **ML-Adjusted Local Estimator**: Shows an LDTE ≈ -0.15 at zero costs—a slightly larger (more negative) effect than -0.12—with similar convergence patterns. -- **Key Finding**: Both estimators reveal insurance primarily affects the lower tail (zero to ~$10,000), shifting the distribution rightward. This indicates insurance increases ED access among those who would otherwise not seek care, while having minimal impact on high-cost users. - - -The confidence intervals are not substantially narrower with ML adjustment. Both methods show comparably wide confidence bands, indicating limited efficiency gains. This result reflects the **limited predictive power of available covariates** (R² ≈ 0.21 when predicting ED costs from pre-treatment ED history and demographics). - -ML adjustment provides efficiency gains proportional to covariate predictive power. When covariates weakly predict outcomes (R² < 0.3), as in this case, ML adjustment yields minimal improvements over simple estimation. This is a characteristic of the data—pre-treatment healthcare utilization and basic demographics cannot strongly predict future emergency department costs—not a failure of the ML methodology. - - -Cost Analysis with Local PTE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - # Compute Local Probability Treatment Effects - lpte_simple, lpte_lower_simple, lpte_upper_simple = simple_local_estimator.predict_lpte( - target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) - control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) - locations=np.insert(outcome_ed_costs_locations, 0, -1) - ) - - lpte_ml, lpte_lower_ml, lpte_upper_ml = ml_local_estimator.predict_lpte( - target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) - control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) - locations=np.insert(outcome_ed_costs_locations, 0, -1) - ) - - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - # Simple local estimator - plot(outcome_ed_costs_locations[1:], lpte_simple, lpte_lower_simple, lpte_upper_simple, - chart_type="bar", - title="Effects of Emergency Department Costs (Simple Local Estimator)", - xlabel="Emergency Department Costs", - ylabel="Local Probability Treatment Effect", - color="purple", - ax=ax1) - - # ML-adjusted local estimator - plot(outcome_ed_costs_locations[1:], lpte_ml, lpte_lower_ml, lpte_upper_ml, - chart_type="bar", - title="Effects of Emergency Department Costs (ML-Adjusted Local Estimator)", - xlabel="Emergency Department Costs", - ylabel="Local Probability Treatment Effect", - ax=ax2) - plt.tight_layout() - plt.show() - -The Local Probability Treatment Effects analysis produces the following visualization: - -.. image:: ../_static/oregon_lpte_costs_comparison.png - :alt: Oregon Health Insurance Experiment LPTE Analysis - :width: 800px - :align: center - -**1. LPTE Interpretation and Distribution-Level Insights** - -- **Simple Local Estimator**: Shows mixed effects across cost bins. At zero costs, LPTE ≈ -0.02 (not statistically significant given wide confidence intervals). Small positive effects appear in the $5,000-$10,000 range (LPTE ≈ 0.01-0.02), converging to zero beyond $30,000. -- **ML-Adjusted Local Estimator**: Shows a larger negative effect at zero costs (LPTE ≈ -0.04) and positive effects in the $5,000-$15,000 range (LPTE ≈ 0.01-0.02). Effects converge to zero at higher costs. -- **Key Finding**: Insurance reduces the probability mass at zero costs while increasing it in the moderate cost range ($5,000-$15,000). This represents a redistribution of probability mass from non-users to moderate ED cost users, with minimal effect on high-cost outliers. - -**2. Covariate Adjustment Effects and Confidence Intervals** - -The confidence intervals remain wide for both estimators, though ML adjustment shows slightly more consistent patterns in the moderate cost range. The limited precision suggests: (1) substantial heterogeneity in treatment effects within cost bins, (2) limited predictive power of covariates for specific cost levels, or (3) relatively small sample sizes within individual bins. - -Emergency Department Visits Analysis -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Now let's examine how Medicaid enrollment affects the distribution of emergency department visits (rather than costs): - -.. code-block:: python - - # Initialize local estimators for visits analysis - simple_local_estimator = dte_adj.SimpleLocalDistributionEstimator() - ml_local_estimator = dte_adj.AdjustedLocalDistributionEstimator( - LinearRegression(), - folds=5 - ) - - # Fit local estimators on the full dataset - # Parameters: X, treatment_arms, treatment_indicator, outcomes, strata - simple_local_estimator.fit(X, Z, D, Y_NUM_VISIT_CENS_ED, strata) - ml_local_estimator.fit(X, Z, D, Y_NUM_VISIT_CENS_ED, strata) - - # Define evaluation points for emergency department visits - outcome_ed_visits_locations = np.arange(Y_NUM_VISIT_CENS_ED.min(), Y_NUM_VISIT_CENS_ED.max(), 1) - -Local Estimator Comparison: Simple vs ML-Adjusted (Visits) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Let's compare the results from both simple and machine learning-adjusted local estimators for visits analysis: - -.. code-block:: python - - # Compute LDTE: Treatment vs Control - ldte_simple, lower_simple, upper_simple = simple_local_estimator.predict_ldte( - target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) - control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) - locations=outcome_ed_visits_locations - ) - - ldte_ml, lower_ml, upper_ml = ml_local_estimator.predict_ldte( - target_treatment_arm=1, # Selected for treatment (Enrolled) - control_treatment_arm=0, # Not selected for treatment (Not enrolled) - locations=outcome_ed_visits_locations - ) - - # Visualize the local distribution treatment effects using dte_adj's built-in plot function - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - # Visualize Treatment vs Control using dte_adj's plot function - plot(outcome_ed_visits_locations, ldte_simple, lower_simple, upper_simple, - title="ED Visits: Treatment vs Control (Simple Local Estimator)", - xlabel="Emergency Department Visits", - ylabel="Local Distribution Treatment Effect", - color="purple", - ax=ax1) - - plot(outcome_ed_visits_locations, ldte_ml, lower_ml, upper_ml, - title="ED Visits: Treatment vs Control (ML-Adjusted Local Estimator)", - xlabel="Emergency Department Visits", - ylabel="Local Distribution Treatment Effect", - ax=ax2) - - plt.tight_layout() - plt.show() - -.. image:: ../_static/oregon_ldte_visits_comparison.png - :alt: Oregon Health Insurance Experiment LDTE Visits Analysis - :width: 800px - :align: center - -**1. LDTE Interpretation and Distribution-Level Insights** - -- **Simple Local Estimator**: Shows LDTE ≈ -0.12 at zero visits, meaning 12 percentage points fewer insured individuals have zero ED visits. The effect gradually converges toward zero around 16 visits and remains near zero thereafter. -- **ML-Adjusted Local Estimator**: Shows a more negative effect of LDTE ≈ -0.15 at zero visits (a larger-magnitude reduction), with similar convergence patterns. -- **Key Finding**: Both estimators reveal insurance primarily affects the lower tail (zero to ~7 visits), shifting the distribution rightward. This indicates insurance increases ED utilization among those who would otherwise not visit, while having minimal impact on frequent ED users. - -**2. Covariate Adjustment Effects and Confidence Intervals** - -The confidence intervals are not substantially narrower with ML adjustment. Both methods show comparably wide confidence bands, indicating limited efficiency gains. This suggests: (1) covariates have limited predictive power for ED visit frequency, (2) the linear regression model may be too simple, or (3) the simple estimator is already reasonably efficient. - -Visits Analysis with Local PTE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - # Compute Local Probability Treatment Effects - lpte_simple, lpte_lower_simple, lpte_upper_simple = simple_local_estimator.predict_lpte( - target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) - control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) - locations=np.insert(outcome_ed_visits_locations, 0, -1) - ) - - lpte_ml, lpte_lower_ml, lpte_upper_ml = ml_local_estimator.predict_lpte( - target_treatment_arm=1, # Z=1 Selected for treatment (Enrolled) - control_treatment_arm=0, # Z=0 Not selected for treatment (Not enrolled) - locations=np.insert(outcome_ed_visits_locations, 0, -1) - ) - - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - - # Simple local estimator - plot(outcome_ed_visits_locations[1:], lpte_simple, lpte_lower_simple, lpte_upper_simple, - chart_type="bar", - title="Effects of Emergency Department Visits (Simple Local Estimator)", - xlabel="Emergency Department Visits", - ylabel="Local Probability Treatment Effect", - color="purple", - ax=ax1) - - # ML-adjusted local estimator - plot(outcome_ed_visits_locations[1:], lpte_ml, lpte_lower_ml, lpte_upper_ml, - chart_type="bar", - title="Effects of Emergency Department Visits (ML-Adjusted Local Estimator)", - xlabel="Emergency Department Visits", - ylabel="Local Probability Treatment Effect", - ax=ax2) - - plt.tight_layout() - plt.show() - -.. image:: ../_static/oregon_lpte_visits_comparison.png - :alt: Oregon Health Insurance Experiment LPTE Visits Analysis - :width: 800px - :align: center - -**1. LPTE Interpretation and Distribution-Level Insights** - -- **Simple Local Estimator**: Shows a large negative effect at zero visits (LPTE ≈ -0.12), indicating 12 percentage points fewer insured individuals have zero ED visits. Small positive effects appear in the 1-5 visit range (LPTE ≈ 0.02-0.03), converging to zero beyond 7 visits. -- **ML-Adjusted Local Estimator**: Shows a larger negative effect at zero visits (LPTE ≈ -0.14) and positive effects in the 1-5 visit range (LPTE ≈ 0.03-0.04). Effects converge to zero at higher visit frequencies. -- **Key Finding**: Insurance reduces the probability mass at zero visits while increasing it in the low-to-moderate visit range (1-5 visits). This represents a redistribution of probability mass from non-users to low-frequency ED users, with minimal effect on frequent visitors. - - -The confidence intervals remain wide for both estimators, with minimal differences between simple and ML-adjusted approaches. This limited precision reflects the same fundamental constraint as in the cost analysis: covariates have limited predictive power for ED visit frequency (R² ≈ 0.21). The substantial heterogeneity in treatment effects, combined with weak covariate prediction, means ML adjustment provides minimal efficiency gains over the simpler approach. - - -Stratified Analysis by Household Registration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The Oregon experiment allows us to examine how treatment effects vary across different household registration patterns. This stratified analysis helps identify heterogeneous treatment effects and provides insights into which populations benefit most from Medicaid enrollment. - -.. code-block:: python - - from sklearn.ensemble import RandomForestRegressor - - # Individual Stratum Analysis with Local Estimators - print("\n=== Individual Stratum Analysis (Local Estimators) ===") - - # Helper function to filter data for a specific stratum - def filter_stratum_data(strata_values, stratum_name, X, Z, D, Y): - """Filter and extract data for a specific stratum""" - mask = strata_values == stratum_name - return { - 'X': X[mask], - 'Z': Z[mask], - 'D': D[mask], - 'Y': Y[mask], - 'strata': np.zeros(mask.sum(), dtype=int), # Uniform strata for subset - 'n_total': mask.sum(), - 'n_assigned': (Z[mask] == 1).sum(), - 'n_enrolled': (D[mask] == 1).sum() - } - - # Helper function to estimate LDTE for a stratum - def estimate_stratum_ldte(stratum_data, location_step=3000, folds=3): - """Initialize estimators, fit data, and compute LDTE for a stratum""" - # Initialize estimators - simple_estimator = dte_adj.SimpleLocalDistributionEstimator() - ml_estimator = dte_adj.AdjustedLocalDistributionEstimator( - RandomForestRegressor(n_estimators=10, random_state=42), - folds=folds - ) - - # Fit estimators - simple_estimator.fit(stratum_data['X'], stratum_data['Z'], - stratum_data['D'], stratum_data['Y'], stratum_data['strata']) - ml_estimator.fit(stratum_data['X'], stratum_data['Z'], - stratum_data['D'], stratum_data['Y'], stratum_data['strata']) - - # Define evaluation locations based on stratum's data range - locations = np.arange(stratum_data['Y'].min(), stratum_data['Y'].max(), location_step) - - # Compute LDTE - ldte_simple, lower_simple, upper_simple = simple_estimator.predict_ldte( - target_treatment_arm=1, control_treatment_arm=0, locations=locations - ) - ldte_ml, lower_ml, upper_ml = ml_estimator.predict_ldte( - target_treatment_arm=1, control_treatment_arm=0, locations=locations - ) - - return { - 'simple': {'ldte': ldte_simple, 'lower': lower_simple, 'upper': upper_simple}, - 'ml': {'ldte': ldte_ml, 'lower': lower_ml, 'upper': upper_ml}, - 'locations': locations, - 'sample_size': stratum_data['n_total'], - 'treatment_assignment_size': stratum_data['n_assigned'], - 'treatment_indicator_size': stratum_data['n_enrolled'] - } - - # Get strata values (already consolidated in preprocessing) - strata_consolidated_values = df['strata'].values - unique_consolidated_strata = np.unique(strata_consolidated_values) - - # Analyze each stratum - individual_results = {} - for stratum in unique_consolidated_strata: - print(f"\nAnalyzing stratum: {stratum}") - - # Filter data for this stratum - stratum_data = filter_stratum_data( - strata_consolidated_values, stratum, X, Z, D, Y_ED_CHARG_TOT_ED - ) - - # Print stratum statistics - print(f" Sample size: {stratum_data['n_total']:,}") - print(f" Treatment assignment (Selected): {stratum_data['n_assigned']:,}") - print(f" Treatment indicator (Enrolled): {stratum_data['n_enrolled']:,}") - - # Estimate LDTE for this stratum - individual_results[stratum] = estimate_stratum_ldte( - stratum_data, location_step=2000, folds=3 - ) - -Visualization: Comparing Overall Population vs Stratified Results -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - # Comparison: Overall vs Individual Strata (Local Estimators) - fig, axes = plt.subplots(2, 3, figsize=(24, 12)) - - # Row 1: Simple local estimators - # Overall (all data) - plot(outcome_ed_costs_locations, ldte_simple, lower_simple, upper_simple, - title="ED Costs: Overall Population\n(Simple Local Estimator)", - xlabel="Emergency Department Costs", - ylabel="Local Distribution Treatment Effect", - color="black", ax=axes[0, 0]) - - # Individual strata - col_idx = 1 - for stratum, results in individual_results.items(): - if results is None or col_idx > 2: - continue - - plot(results['locations'], results['simple']['ldte'], - results['simple']['lower'], results['simple']['upper'], - title=f"ED Costs: {stratum}\n(Simple Local Estimator, n={results['sample_size']:,})", - xlabel="Emergency Department Costs", - ylabel="Local Distribution Treatment Effect", - color="blue" if col_idx == 1 else "green", ax=axes[0, col_idx]) - col_idx += 1 - - # Row 2: ML-Adjusted local estimators - # Overall (all data) - plot(outcome_ed_costs_locations, ldte_ml, lower_ml, upper_ml, - title="ED Costs: Overall Population\n(ML-Adjusted Local Estimator)", - xlabel="Emergency Department Costs", - ylabel="Local Distribution Treatment Effect", - color="black", ax=axes[1, 0]) - - # Individual strata - col_idx = 1 - for stratum, results in individual_results.items(): - if results is None or col_idx > 2: - continue - - plot(results['locations'], results['ml']['ldte'], - results['ml']['lower'], results['ml']['upper'], - title=f"ED Costs: {stratum}\n(ML-Adjusted Local Estimator, n={results['sample_size']:,})", - xlabel="Emergency Department Costs", - ylabel="Local Distribution Treatment Effect", - color="red" if col_idx == 1 else "purple", ax=axes[1, col_idx]) - col_idx += 1 - - plt.suptitle("Comparison: Overall Population vs Individual Household Registration Strata (Local Estimators)", fontsize=16) - plt.tight_layout() - plt.show() - -.. image:: ../_static/oregon_ldte_costs_strata.png - :alt: Oregon Health Insurance Experiment DTE Strata Analysis - :width: 800px - :align: center - -**1. Overall Population vs Stratified Analysis** - -- **Overall Population (Left panels)**: - - - Simple: LDTE ≈ -0.21 at zero costs, converging to zero around $10,000 - - ML-Adjusted: LDTE ≈ -0.15 at zero costs, similar convergence pattern - - Both show consistent rightward distribution shifts across the entire population - -- **Signed Self Up (Middle panels, n=12,982)**: - - - Simple: LDTE ≈ -0.18 at zero costs, converging to zero around $20,000 - - ML-Adjusted: LDTE ≈ -0.20 at zero costs, similar pattern - - Smaller magnitude effects compared to overall population, suggesting this stratum has more moderate responses to insurance - -- **Signed Self Up + Others (Right panels, n=4,068)**: - - - Simple: LDTE ≈ -0.55 at zero costs, converging to zero around $15,000-$20,000 - - ML-Adjusted: LDTE ≈ -0.10 to -0.15 at zero costs, stable pattern with improved confidence intervals - - Much larger magnitude effects in the Simple estimator, indicating households with multiple members show substantially stronger treatment effects - - ML adjustment provides more conservative estimates, potentially controlling for confounding household characteristics - -**2. Heterogeneity Across Strata** - -The stratified analysis reveals substantial treatment effect heterogeneity: - -- **"Signed self up" stratum**: Moderate effects (LDTE ≈ -0.18 to -0.20), suggesting single-person households have more modest increases in ED utilization -- **"Signed self up + others" stratum**: Large effects in Simple estimator (LDTE ≈ -0.55), suggesting multi-person households experience much greater increases in ED access when not adjusting for covariates -- The 3-4x larger effect in the "signed self up + others" group (Simple estimator) indicates that household composition is a critical moderator of insurance impact -- However, ML adjustment substantially reduces this estimate, suggesting that some of the observed effect may be attributable to observable household characteristics rather than pure treatment effects - -**3. Comparison of Estimation Methods** - -- **Overall population**: Both estimators show reasonable confidence intervals, with ML adjustment providing modest improvements in precision and slightly more conservative estimates. -- **"Signed self up" stratum**: Both estimators yield similar point estimates and manageable confidence intervals, suggesting robustness to model specification in this larger subsample. -- **"Signed self up + others" stratum**: - - - The Simple estimator shows the largest treatment effects across all strata (LDTE ≈ -0.55) - - ML adjustment substantially reduces the estimated effect and stabilizes confidence intervals - - This divergence suggests that observable covariates (e.g., household size, age composition, baseline health status) explain a significant portion of the treatment effect heterogeneity - - The improved stability of ML-adjusted estimates indicates successful control for confounding factors that may have been correlated with both treatment assignment and outcomes - -**4. Practical Implications** - -- **Household structure matters**: Multi-person households show substantially larger treatment effects in unadjusted analyses, likely because insurance coverage enables care-seeking for multiple family members. -- **The role of covariates**: The difference between Simple and ML-adjusted estimates in the "signed self up + others" stratum highlights the importance of controlling for household characteristics. The unadjusted effect may overstate the pure treatment effect by conflating insurance provision with pre-existing household differences. -- **Stratification reveals hidden heterogeneity**: The overall population estimate masks substantial variation across household types, demonstrating the value of subgroup analysis. -- **Model specification considerations**: ML adjustment improves estimation stability in smaller strata and provides more defensible causal estimates by controlling for observable confounders. The convergence of all estimates to zero at higher cost levels confirms that the treatment primarily affects the lower tail of the cost distribution. - -Conclusion -~~~~~~~~~~ - -This analysis of the Oregon Health Insurance Experiment using local distribution treatment effects reveals specific patterns in how Medicaid insurance affects emergency department utilization among compliers: - -**1. Insurance Primarily Shifts the Lower Tail of the Distribution** - -Our LDTE analysis shows that Medicaid insurance reduces the probability of zero ED costs by 12-15 percentage points (LDTE ≈ -0.12 to -0.15 at $0), with effects converging to zero around $10,000. Similarly, for ED visits, insurance reduces zero visits by 12-14 percentage points (LPTE ≈ -0.12 to -0.14 at 0 visits). This indicates insurance primarily enables access for those who would otherwise not use ED services, rather than affecting high-cost or frequent users. - -**2. Probability Mass Redistribution, Not Uniform Increases** - -The LPTE analysis reveals insurance does not uniformly increase ED utilization. Instead, it redistributes probability mass: reducing zero-cost/zero-visit individuals while increasing moderate users ($5,000-$15,000 costs; 1-5 visits). High-cost outliers (>$30,000) and frequent users (>7 visits) show minimal treatment effects, suggesting insurance's impact is concentrated among marginal users. - -**3. Substantial Heterogeneity by Household Composition** - -Stratified analysis uncovers dramatic treatment effect heterogeneity: single-person households ("signed self up") show moderate effects (LDTE ≈ -0.18 to -0.20), while multi-person households ("signed self up + others") exhibit 3-4x larger effects (LDTE ≈ -0.55). This suggests household structure is a critical moderator—insurance enables care-seeking for multiple family members when households include dependents. - -**4. ML Adjustment Effectiveness Depends on Covariate Predictive Power** - -With baseline covariates (pre-randomization ED utilization + demographics, R² ≈ 0.21), ML-adjusted estimators show minimal efficiency gains—confidence intervals remain comparably wide or even slightly wider than simple estimators. However, enhanced feature engineering could improve predictive power, enabling ML adjustment to narrow confidence intervals. - - -**5. Policy Implications for Targeted Interventions** - -The distributional analysis reveals that Medicaid's primary benefit is enabling ED access for marginal users who would otherwise forego care, rather than increasing utilization among existing high users. The 3-4x larger effects for multi-person households suggest family coverage may yield substantially greater utilization impacts than individual coverage. These findings have direct implications for healthcare budgeting and targeting: policymakers should anticipate larger ED increases when expanding coverage to families versus individuals, and the primary fiscal impact will come from converting non-users to moderate users rather than increasing costs among existing high users. - -Next Steps -~~~~~~~~~~ - -**For Your Own Data**: - -- Try with your own randomized experiment data -- Experiment with different ML models (XGBoost, Neural Networks) for adjustment -- Explore stratified estimators for covariate-adaptive randomization designs -- Use multi-task learning (``is_multi_task=True``) for computational efficiency with many locations diff --git a/pyproject.toml b/pyproject.toml index e50615a..1915c21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,9 @@ dependencies = [ dev = [ "build>=1.2.2,<1.6.0", "ruff>=0.12.2,<0.17.0", - "sphinx>=7.3.7,<8.2.0", + "mkdocs>=1.6,<2.0", + "mkdocs-material>=9.5,<10.0", + "mkdocstrings[python]>=0.26,<0.30", "scikit-learn>=1.5,<1.9", "pre-commit>=4.0.1,<4.7.0", "pandas>=2.0", @@ -49,7 +51,9 @@ dev-dependencies = [ "build>=1.2.2,<1.6.0", "build>=1.2.2,<1.4.0", "ruff>=0.12.2,<0.17.0", - "sphinx>=7.3.7,<8.2.0", + "mkdocs>=1.6,<2.0", + "mkdocs-material>=9.5,<10.0", + "mkdocstrings[python]>=0.26,<0.30", "scikit-learn>=1.5,<1.9", "pre-commit>=4.0.1,<4.7.0", "pandas>=2.0", diff --git a/uv.lock b/uv.lock index 6771af6..4c9dee2 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", @@ -12,21 +12,25 @@ resolution-markers = [ ] [[package]] -name = "alabaster" -version = "0.7.16" +name = "babel" +version = "2.17.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/3e/13dd8e5ed9094e734ac430b5d0eb4f2bb001708a8b7856cbf8e084e001ba/alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65", size = 23776, upload-time = "2024-01-10T00:56:10.189Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/6b/d52e42361e1aa00709585ecc30b3f9684b3ab62530771402248b1b1d6240/babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", size = 9951852, upload-time = "2025-02-01T15:17:41.026Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/32/34/d4e1c02d3bee589efb5dfa17f88ea08bdb3e3eac12bc475462aec52ed223/alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92", size = 13511, upload-time = "2024-01-10T00:56:08.388Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, ] [[package]] -name = "babel" -version = "2.17.0" +name = "backrefs" +version = "8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/6b/d52e42361e1aa00709585ecc30b3f9684b3ab62530771402248b1b1d6240/babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", size = 9951852, upload-time = "2025-02-01T15:17:41.026Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/56/4744bcd0c82184e80c52b0ac4076c261a8ffa1f1b343ff2f6e89ce0e1cef/backrefs-8.0.tar.gz", hash = "sha256:b556cd7d36c3a3a2f256b89590b176b8eddfb73bcfaee3a3ddd84ea66d21ce50", size = 7013081, upload-time = "2026-07-26T19:54:24.638Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, + { url = "https://files.pythonhosted.org/packages/e3/fd/9bf53b6a6f6f519ffaac765df2f2a25e5c2fc6d32cfd2b2747099e72c911/backrefs-8.0-py310-none-any.whl", hash = "sha256:4a627b817fd2dce43b79ab48da63613340509381cd8ce0897078a0bce79a2ab8", size = 380377, upload-time = "2026-07-26T19:54:17.457Z" }, + { url = "https://files.pythonhosted.org/packages/e1/29/4bd7ae72a2634da00379c2b3bcc5439e7c94620235c6afea8af15229a973/backrefs-8.0-py311-none-any.whl", hash = "sha256:f0c35cf0102ba6b6070c12a492be3c1c1d3f5839529784b9a9565d6d04569a01", size = 392169, upload-time = "2026-07-26T19:54:18.782Z" }, + { url = "https://files.pythonhosted.org/packages/29/13/232505664e8e2a0c7a2eb0c505cfade9d715538f89a5d62bc4c272968f62/backrefs-8.0-py312-none-any.whl", hash = "sha256:87f0fae8c5f207fe9f4b2887efc71d42f4900ac78faa1af08d675ef303692dc5", size = 398084, upload-time = "2026-07-26T19:54:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/8a/69/47a3dc20abc4fa5486655fde681bd55e63211b46c886d8c02223d6468431/backrefs-8.0-py313-none-any.whl", hash = "sha256:601ce68ca12385dbda06ce264406b4c4210cf5b79fd0fd627592365c92f29a88", size = 400040, upload-time = "2026-07-26T19:54:21.194Z" }, + { url = "https://files.pythonhosted.org/packages/1c/cf/e5f9b68a5b0e939a2fb933a66c20180d0c9241bf8927f7a47fa48c1675e9/backrefs-8.0-py314-none-any.whl", hash = "sha256:9ec96efa080938be92323e8e730e57718c9c88eb15ad70bbef4e1766df591408", size = 411903, upload-time = "2026-07-26T19:54:23.221Z" }, ] [[package]] @@ -124,6 +128,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" }, ] +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -219,18 +235,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] -[[package]] -name = "docutils" -version = "0.21.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444, upload-time = "2024-04-23T18:57:18.24Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, -] - [[package]] name = "dte-adj" -version = "0.1.9" +version = "0.1.10" source = { editable = "." } dependencies = [ { name = "matplotlib" }, @@ -244,39 +251,45 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "build" }, + { name = "mkdocs" }, + { name = "mkdocs-material" }, + { name = "mkdocstrings", extra = ["python"] }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, { name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, { name = "polars" }, { name = "pre-commit" }, { name = "ruff" }, { name = "scikit-learn" }, - { name = "sphinx" }, ] [package.dev-dependencies] dev = [ { name = "build" }, + { name = "mkdocs" }, + { name = "mkdocs-material" }, + { name = "mkdocstrings", extra = ["python"] }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, { name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, { name = "polars" }, { name = "pre-commit" }, { name = "ruff" }, { name = "scikit-learn" }, - { name = "sphinx" }, ] [package.metadata] requires-dist = [ { name = "build", marker = "extra == 'dev'", specifier = ">=1.2.2,<1.6.0" }, { name = "matplotlib", specifier = ">=3.7" }, + { name = "mkdocs", marker = "extra == 'dev'", specifier = ">=1.6,<2.0" }, + { name = "mkdocs-material", marker = "extra == 'dev'", specifier = ">=9.5,<10.0" }, + { name = "mkdocstrings", extras = ["python"], marker = "extra == 'dev'", specifier = ">=0.26,<0.30" }, { name = "numpy", specifier = ">=1.26" }, { name = "pandas", marker = "extra == 'dev'", specifier = ">=2.0" }, { name = "polars", marker = "extra == 'dev'", specifier = ">=1.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.0.1,<4.7.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.12.2,<0.16.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.12.2,<0.17.0" }, { name = "scikit-learn", marker = "extra == 'dev'", specifier = ">=1.5,<1.9" }, { name = "scipy", specifier = ">=1.13.1" }, - { name = "sphinx", marker = "extra == 'dev'", specifier = ">=7.3.7,<8.2.0" }, { name = "tqdm", specifier = ">=4.60" }, ] provides-extras = ["dev"] @@ -285,12 +298,14 @@ provides-extras = ["dev"] dev = [ { name = "build", specifier = ">=1.2.2,<1.4.0" }, { name = "build", specifier = ">=1.2.2,<1.6.0" }, + { name = "mkdocs", specifier = ">=1.6,<2.0" }, + { name = "mkdocs-material", specifier = ">=9.5,<10.0" }, + { name = "mkdocstrings", extras = ["python"], specifier = ">=0.26,<0.30" }, { name = "pandas", specifier = ">=2.0" }, { name = "polars", specifier = ">=1.0" }, { name = "pre-commit", specifier = ">=4.0.1,<4.7.0" }, - { name = "ruff", specifier = ">=0.12.2,<0.16.0" }, + { name = "ruff", specifier = ">=0.12.2,<0.17.0" }, { name = "scikit-learn", specifier = ">=1.5,<1.9" }, - { name = "sphinx", specifier = ">=7.3.7,<8.2.0" }, ] [[package]] @@ -359,6 +374,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/6c/10280af05b44fafd1dff69422805061fa1af29270bc52dce031ac69540bf/fonttools-4.60.2-py3-none-any.whl", hash = "sha256:73cf92eeda67cf6ff10c8af56fc8f4f07c1647d989a979be9e388a49be26552a", size = 1144610, upload-time = "2025-12-09T13:38:09.5Z" }, ] +[[package]] +name = "ghp-import" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, +] + +[[package]] +name = "griffe" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "griffecli" }, + { name = "griffelib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/a0/927cf9416f527f623f4ccd9220337fca0cd85303b7e82adeab1e53669cd3/griffe-2.2.0.tar.gz", hash = "sha256:9c0dd9a7feda9e169d783507d777c6fb2f2fc2919b139747a5b4f5357517a611", size = 246229, upload-time = "2026-08-16T14:04:57.232Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/51/c9f9cb144e2bd21cbcb6ebd7978ecb5b9f5a375e78313744158338c8039c/griffe-2.2.0-py3-none-any.whl", hash = "sha256:db20672b31c3de2a1af5a18a3b3d8c186ccc2425e10290486b4f0364d25929e9", size = 5070, upload-time = "2026-08-16T14:04:51.971Z" }, +] + +[[package]] +name = "griffecli" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "griffelib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/75/3cf2fb1ae21fb55a3b917b45d24426a620ee916f4bdbd03f350c69d3d892/griffecli-2.2.0.tar.gz", hash = "sha256:b9e763131218eb19887ed52de20fe944252d811bfb4a3e339874908a588f0044", size = 58066, upload-time = "2026-08-16T14:04:55.984Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/83/04652213d82afeecc8e8a31231eeb87cc484dcc055fd96ecb0bfd9860405/griffecli-2.2.0-py3-none-any.whl", hash = "sha256:8ebb7ae1dcc2617f5d39f00e4c238dd6b5cd0b13157b0ce22c1ea4ca8b4c5d1d", size = 11480, upload-time = "2026-08-16T14:04:53.14Z" }, +] + +[[package]] +name = "griffelib" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/b4/a767e91c606deefc447a96eaf59edd77397960b1d677dffd833ee8449831/griffelib-2.2.0.tar.gz", hash = "sha256:e1bc36fe9cd21d4b6b659b456346755e4cfdc5676c0a5214083126ee12612b3c", size = 227048, upload-time = "2026-08-16T14:04:58.383Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/b6/f65ac785d4ac90dcf7c831ac6256f5dd4a19780f4e1575b2c0d6eeebe319/griffelib-2.2.0-py3-none-any.whl", hash = "sha256:d71c3bc2bbed9f958488634fe788b843a9f705d6d2838ca32cd6c25eeb64dfc4", size = 166779, upload-time = "2026-08-16T14:04:54.365Z" }, +] + [[package]] name = "identify" version = "2.6.12" @@ -377,15 +439,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, ] -[[package]] -name = "imagesize" -version = "1.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/84/62473fb57d61e31fef6e36d64a179c8781605429fd927b5dd608c997be31/imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a", size = 1280026, upload-time = "2022-07-01T12:21:05.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", size = 8769, upload-time = "2022-07-01T12:21:02.467Z" }, -] - [[package]] name = "importlib-metadata" version = "8.7.0" @@ -506,6 +559,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/1d/50ad811d1c5dae091e4cf046beba925bcae0a610e79ae4c538f996f63ed5/kiwisolver-1.4.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:65ea09a5a3faadd59c2ce96dc7bf0f364986a315949dc6374f04396b0d60e09b", size = 71762, upload-time = "2024-12-24T18:30:48.903Z" }, ] +[[package]] +name = "markdown" +version = "3.10.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/6f/da4c6aea59b3001f2e8c0ec7497475aadaf3b021c10cab5b2858f0f32b26/markdown-3.10.3.tar.gz", hash = "sha256:3589362618f743188b4d955b874402bc814f4f83f544dc207719f4baa7d9c45f", size = 372596, upload-time = "2026-07-30T19:05:29.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/69/4a5af2bc115a9a33fefe51709749de8262be3f9ba063d1753a837cdbc49c/markdown-3.10.3-py3-none-any.whl", hash = "sha256:fa6c92a00a4a3c98b22728c64a935ae1928250ae65058a6ded814d2cc29a4cea", size = 110757, upload-time = "2026-07-30T19:05:27.883Z" }, +] + [[package]] name = "markupsafe" version = "3.0.2" @@ -617,6 +679,135 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/b9/59e120d24a2ec5fc2d30646adb2efb4621aab3c6d83d66fb2a7a182db032/matplotlib-3.10.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb73d8aa75a237457988f9765e4dfe1c0d2453c5ca4eabc897d4309672c8e014", size = 8594298, upload-time = "2025-05-08T19:10:51.738Z" }, ] +[[package]] +name = "mergedeep" +version = "1.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, +] + +[[package]] +name = "mkdocs" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "ghp-import" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mergedeep" }, + { name = "mkdocs-get-deps" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "watchdog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, +] + +[[package]] +name = "mkdocs-autorefs" +version = "1.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mkdocs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/c0/f641843de3f612a6b48253f39244165acff36657a91cc903633d456ae1ac/mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197", size = 56588, upload-time = "2026-02-10T15:23:55.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089", size = 25530, upload-time = "2026-02-10T15:23:53.817Z" }, +] + +[[package]] +name = "mkdocs-get-deps" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mergedeep" }, + { name = "platformdirs" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, +] + +[[package]] +name = "mkdocs-material" +version = "9.7.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel" }, + { name = "backrefs" }, + { name = "colorama" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "mkdocs" }, + { name = "mkdocs-material-extensions" }, + { name = "paginate" }, + { name = "pygments" }, + { name = "pymdown-extensions" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/cd/c05d3a530ba7934f144fb45f7203cd236adc25c7bdcc34673d202f4b0278/mkdocs_material-9.7.7.tar.gz", hash = "sha256:c0649c065b1b0512d60aad8c10f947f8e455284475239b364b610f2deb4d0855", size = 4097923, upload-time = "2026-07-17T16:21:33.156Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/21/17c1bc9e6f47c972ad66fb2ac2568f99f90f1207eeb6fc3b34d094dba7b5/mkdocs_material-9.7.7-py3-none-any.whl", hash = "sha256:8ea9bb1737a5b524a5f9dcf2e1b4ebda8274ae3008aa7845720a97083bef708f", size = 9305438, upload-time = "2026-07-17T16:21:30.017Z" }, +] + +[[package]] +name = "mkdocs-material-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847, upload-time = "2023-11-22T19:09:45.208Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" }, +] + +[[package]] +name = "mkdocstrings" +version = "0.29.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mkdocs" }, + { name = "mkdocs-autorefs" }, + { name = "pymdown-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/e8/d22922664a627a0d3d7ff4a6ca95800f5dde54f411982591b4621a76225d/mkdocstrings-0.29.1.tar.gz", hash = "sha256:8722f8f8c5cd75da56671e0a0c1bbed1df9946c0cef74794d6141b34011abd42", size = 1212686, upload-time = "2025-03-31T08:33:11.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/14/22533a578bf8b187e05d67e2c1721ce10e3f526610eebaf7a149d557ea7a/mkdocstrings-0.29.1-py3-none-any.whl", hash = "sha256:37a9736134934eea89cbd055a513d40a020d87dfcae9e3052c2a6b8cd4af09b6", size = 1631075, upload-time = "2025-03-31T08:33:09.661Z" }, +] + +[package.optional-dependencies] +python = [ + { name = "mkdocstrings-python" }, +] + +[[package]] +name = "mkdocstrings-python" +version = "1.16.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "griffe" }, + { name = "mkdocs-autorefs" }, + { name = "mkdocstrings" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/ed/b886f8c714fd7cccc39b79646b627dbea84cd95c46be43459ef46852caf0/mkdocstrings_python-1.16.12.tar.gz", hash = "sha256:9b9eaa066e0024342d433e332a41095c4e429937024945fea511afe58f63175d", size = 206065, upload-time = "2025-06-03T12:52:49.276Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/dd/a24ee3de56954bfafb6ede7cd63c2413bb842cc48eb45e41c43a05a33074/mkdocstrings_python-1.16.12-py3-none-any.whl", hash = "sha256:22ded3a63b3d823d57457a70ff9860d5a4de9e8b1e482876fc9baabaf6f5f374", size = 124287, upload-time = "2025-06-03T12:52:47.819Z" }, +] + [[package]] name = "nodeenv" version = "1.9.1" @@ -789,6 +980,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "paginate" +version = "0.5.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252, upload-time = "2024-08-25T14:17:24.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, +] + [[package]] name = "pandas" version = "2.3.3" @@ -922,6 +1122,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/b0/34937815889fa982613775e4b97fddd13250f11012d769949c5465af2150/pandas-3.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:108dd1790337a494aa80e38def654ca3f0968cf4f362c85f44c15e471667102d", size = 9452085, upload-time = "2026-02-17T22:20:14.331Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "pillow" version = "12.3.0" @@ -1078,6 +1287,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pymdown-extensions" +version = "11.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/a9/5f0c535ba3b08fe09270c16808e053a968868242ecbd5676d4e3a488bf28/pymdown_extensions-11.0.1.tar.gz", hash = "sha256:dd2905ae6fc5b75582fafb139a1266ffc754705efa902aa50067fa7ff4f94ec0", size = 857113, upload-time = "2026-07-02T17:59:22.955Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl", hash = "sha256:db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2", size = 269455, upload-time = "2026-07-02T17:59:21.271Z" }, +] + [[package]] name = "pyparsing" version = "3.2.3" @@ -1161,6 +1383,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, ] +[[package]] +name = "pyyaml-env-tag" +version = "1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, +] + [[package]] name = "requests" version = "2.33.0" @@ -1379,97 +1613,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "snowballstemmer" -version = "3.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/75/a7/9810d872919697c9d01295633f5d574fb416d47e535f258272ca1f01f447/snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895", size = 105575, upload-time = "2025-05-09T16:34:51.843Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064", size = 103274, upload-time = "2025-05-09T16:34:50.371Z" }, -] - -[[package]] -name = "sphinx" -version = "7.3.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils" }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/0a/b88033900b1582f5ed8f880263363daef968d1cd064175e32abfd9714410/sphinx-7.3.7.tar.gz", hash = "sha256:a4a7db75ed37531c05002d56ed6948d4c42f473a36f46e1382b0bd76ca9627bc", size = 7094808, upload-time = "2024-04-19T04:44:48.297Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/fa/130c32ed94cf270e3d0b9ded16fb7b2c8fea86fa7263c29a696a30c1dde7/sphinx-7.3.7-py3-none-any.whl", hash = "sha256:413f75440be4cacf328f580b4274ada4565fb2187d696a84970c23f77b64d8c3", size = 3335650, upload-time = "2024-04-19T04:44:43.839Z" }, -] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, -] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, -] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, -] - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, -] - -[[package]] -name = "sphinxcontrib-qthelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, -] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, -] - [[package]] name = "threadpoolctl" version = "3.6.0" @@ -1572,6 +1715,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/2a/dc2228b2888f51192c7dc766106cd475f1b768c10caaf9727659726f7391/virtualenv-20.36.1-py3-none-any.whl", hash = "sha256:575a8d6b124ef88f6f51d56d656132389f961062a9177016a50e4f507bbcc19f", size = 6008258, upload-time = "2026-01-09T18:20:59.425Z" }, ] +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/56/90994d789c61df619bfc5ce2ecdabd5eeff564e1eb47512bd01b5e019569/watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26", size = 96390, upload-time = "2024-11-01T14:06:24.793Z" }, + { url = "https://files.pythonhosted.org/packages/55/46/9a67ee697342ddf3c6daa97e3a587a56d6c4052f881ed926a849fcf7371c/watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112", size = 88389, upload-time = "2024-11-01T14:06:27.112Z" }, + { url = "https://files.pythonhosted.org/packages/44/65/91b0985747c52064d8701e1075eb96f8c40a79df889e59a399453adfb882/watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3", size = 89020, upload-time = "2024-11-01T14:06:29.876Z" }, + { url = "https://files.pythonhosted.org/packages/e0/24/d9be5cd6642a6aa68352ded4b4b10fb0d7889cb7f45814fb92cecd35f101/watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c", size = 96393, upload-time = "2024-11-01T14:06:31.756Z" }, + { url = "https://files.pythonhosted.org/packages/63/7a/6013b0d8dbc56adca7fdd4f0beed381c59f6752341b12fa0886fa7afc78b/watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2", size = 88392, upload-time = "2024-11-01T14:06:32.99Z" }, + { url = "https://files.pythonhosted.org/packages/d1/40/b75381494851556de56281e053700e46bff5b37bf4c7267e858640af5a7f/watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c", size = 89019, upload-time = "2024-11-01T14:06:34.963Z" }, + { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, + { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, + { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, + { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, + { url = "https://files.pythonhosted.org/packages/30/ad/d17b5d42e28a8b91f8ed01cb949da092827afb9995d4559fd448d0472763/watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881", size = 87902, upload-time = "2024-11-01T14:06:53.119Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ca/c3649991d140ff6ab67bfc85ab42b165ead119c9e12211e08089d763ece5/watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11", size = 88380, upload-time = "2024-11-01T14:06:55.19Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + [[package]] name = "zipp" version = "3.23.0"