Custom templating solution (#5)

Template analysis notebook to apply to other projects.

Author: rossbar

.gitignore

@@ -4,6 +4,7 @@ __pycache__/*
 *.egg-info
 _data/*
 site/_build/*
+site/_generated/*
 site/content/_generated/*
 # Ignore text version of issues table
 top_issues_table.md

site/Makefile

@@ -12,9 +12,36 @@ BUILDDIR      = _build
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-.PHONY: help Makefile
+_generated/numpy.md: analysis_template.md
+	python tools/generate_from_template.py numpy
 
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+_generated/scipy.md: analysis_template.md
+	python tools/generate_from_template.py scipy
+
+_generated/matplotlib.md: analysis_template.md
+	python tools/generate_from_template.py matplotlib
+
+_generated/pandas.md: analysis_template.md
+	python tools/generate_from_template.py pandas
+
+_generated/scikit-learn.md: analysis_template.md
+	python tools/generate_from_template.py scikit-learn
+
+_generated/scikit-image.md: analysis_template.md
+	python tools/generate_from_template.py scikit-image
+
+_generated/networkx.md: analysis_template.md
+	python tools/generate_from_template.py networkx
+
+html : _generated/numpy.md _generated/scipy.md _generated/matplotlib.md _generated/pandas.md _generated/scikit-learn.md _generated/scikit-image.md _generated/networkx.md
+	$(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) $(O)
+
+clean: 
+	rm -rf _build _generated
+
+.PHONY: help clean
+
+## Catch-all target: route all unknown targets to Sphinx using the new
+## "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+#%: Makefile
+#	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

site/numpy_timeseries.md site/analysis_template.mdsite/numpy_timeseries.md renamed to site/analysis_template.md

@@ -11,7 +11,7 @@ kernelspec:
   name: python3
 ---
 
-# NumPy Report
+# `{{ project }}`
 
 ```{code-cell} ipython3
 ---
@@ -48,7 +48,7 @@ output_notebook()
 
 %TODO improve handling of datetimes (super annoying)
 
-A snapshot of the development on the NumPy project.
+A snapshot of the development on the {{ project }} project.
 
 ## Issues
 
@@ -59,10 +59,10 @@ A snapshot of the development on the NumPy project.
 query_date = np.datetime64("2020-01-01 00:00:00")
 
 # Load data
-with open("../_data/issues.json", "r") as fh:
+with open("../../devstats-data/{{ project }}_issues.json", "r") as fh:
     issues = [item["node"] for item in json.loads(fh.read())]
 
-glue("query_date", str(query_date.astype("M8[D]")))
+glue("{{ project }}_query_date", str(query_date.astype("M8[D]")))
 ```
 
 ### New issues
@@ -86,17 +86,17 @@ new_issue_lifetime = np.array(
     ],
 ).astype("m8[h]")  # in hours
 
-glue("num_new_issues", len(newly_created))
-glue("num_new_issues_closed", percent_val(len(new_issues_closed), len(newly_created)))
-glue("new_issue_median_lifetime", f"{np.median(new_issue_lifetime)}")
+glue("{{ project }}_num_new_issues", len(newly_created))
+glue("{{ project }}_num_new_issues_closed", percent_val(len(new_issues_closed), len(newly_created)))
+glue("{{ project }}_new_issue_median_lifetime", f"{np.median(new_issue_lifetime)}")
 ```
 
-{glue:text}`num_new_issues` new issues have been opened since
-{glue:text}`query_date`, of which {glue:text}`num_new_issues_closed` have been
+{glue:text}`{{ project }}_num_new_issues` new issues have been opened since
+{glue:text}`{{ project }}_query_date`, of which {glue:text}`{{ project }}_num_new_issues_closed` have been
 closed.
 
 The median lifetime of new issues that were created and closed in this period
-is {glue:text}`new_issue_median_lifetime`.
+is {glue:text}`{{ project }}_new_issue_median_lifetime`.
 
 % TODO: replace with bokeh or some other live-plot
 % TODO: for any remaining static/mpl plots, set default params for things
@@ -185,19 +185,19 @@ num_issues_commented_by_cutoff = np.array(
 # usually sits before it's at least commented on
 
 glue(
-    "num_new_issues_responded",
+    "{{ project }}_num_new_issues_responded",
     percent_val(len(commented_issues), len(newly_created_day_old))
 )
 
-glue("new_issues_at_least_1_day_old", len(newly_created_day_old))
-glue("median_response_time", f"{median_time_til_first_response:1.0f}")
+glue("{{ project }}_new_issues_at_least_1_day_old", len(newly_created_day_old))
+glue("{{ project }}_median_response_time", f"{median_time_til_first_response:1.0f}")
 ```
 
-Of the {glue:text}`new_issues_at_least_1_day_old` issues that are at least 24
-hours old, {glue:text}`num_new_issues_responded` of them have been commented
+Of the {glue:text}`{{ project }}_new_issues_at_least_1_day_old` issues that are at least 24
+hours old, {glue:text}`{{ project }}_num_new_issues_responded` of them have been commented
 on.
 The median time until an issue is first responded to is
-{glue:text}`median_response_time` hours.
+{glue:text}`{{ project }}_median_response_time` hours.
 
 ```{code-cell} ipython3
 ---
@@ -257,7 +257,7 @@ first_commenter_tab.sort_values(
 tags: [hide-input]
 ---
 
-with open("../_data/prs.json", "r") as fh:
+with open("../../devstats-data/{{ project }}_prs.json", "r") as fh:
     prs = [item["node"] for item in json.loads(fh.read())]
 
 ### Filters
@@ -413,9 +413,9 @@ show(p)
 
 ### Where contributions come from
 
-There have been a total of {glue:text}`num_merged_prs_with_known_authors`
-merged PRs[^only_active] submitted by {glue:text}`num_unique_authors_of_merged_prs`
-unique authors. {glue:text}`num_flyby` of these are "fly-by" PRs, i.e.
+There have been a total of {glue:text}`{{ project }}_num_merged_prs_with_known_authors`
+merged PRs[^only_active] submitted by {glue:text}`{{ project }}_num_unique_authors_of_merged_prs`
+unique authors. {glue:text}`{{ project }}_num_flyby` of these are "fly-by" PRs, i.e.
 PRs from users who have contributed to the project once (to-date).
 
 
@@ -436,9 +436,9 @@ num_merged_prs_per_author = np.array(
 
 num_flybys = np.sum(num_merged_prs_per_author == 1)
 
-glue("num_merged_prs_with_known_authors", len(merged_prs))
-glue("num_unique_authors_of_merged_prs", len(contributions_by_author))
-glue("num_flyby", percent_val(num_flybys, len(num_merged_prs_per_author)))
+glue("{{ project }}_num_merged_prs_with_known_authors", len(merged_prs))
+glue("{{ project }}_num_unique_authors_of_merged_prs", len(contributions_by_author))
+glue("{{ project }}_num_flyby", percent_val(num_flybys, len(num_merged_prs_per_author)))
 ```
 
 ```{code-cell} ipython3
@@ -477,7 +477,7 @@ described as:
 
 For this analysis, we will consider merged PRs as the metric for contribution.
 Considering all merged PRs over the lifetime of the project, the pony factor
-is: {glue:text}`pony_factor`.
+is: {glue:text}`{{ project }}_pony_factor`.
 
 % TODO: pandas-ify to improve sorting
 
@@ -510,7 +510,7 @@ ax.hlines(
 )
 ax.legend();
 
-glue("pony_factor", pony_factor)
+glue("{{ project }}_pony_factor", pony_factor)
 ```
 
 % TODO: Add:

site/conf.py

@@ -37,7 +37,8 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', "analysis_template.md"]
+execution_excludepatterns = ["analysis_template.md"]
 
 
 # -- Options for HTML output -------------------------------------------------

site/index.md

@@ -5,9 +5,8 @@ This website tracks the recent development history of the Scientific Python
 ecosystem.
 
 ```{toctree}
----
-maxdepth: 1
----
+:maxdepth: 1
+:hidden: true
 
-numpy_timeseries
+project_reports
 ```

site/project_reports.md

@@ -0,0 +1,9 @@
+Project Reports
+===============
+
+```{toctree}
+:glob:
+:maxdepth: 1
+
+_generated/*
+```

site/tools/generate_from_template.py

@@ -0,0 +1,14 @@
+import sys
+import os
+
+if len(sys.argv) != 2:
+    raise ValueError("CLI usage: python generate_from_template.py <project_name>")
+
+project = sys.argv[1]
+
+os.makedirs("_generated", exist_ok=True)
+
+with open("analysis_template.md", "r") as fh:
+    template = fh.read()
+with open(f"_generated/{project}.md", "w") as fh:
+    fh.write(template.replace("{{ project }}", project))