Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
268 changes: 268 additions & 0 deletions backend/tests/test_scientific_fitting.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
"""Independent published oracles for the Fitting and Inference scientific pack."""

import copy
import json
import math
from pathlib import Path

import pytest

from graphlink_plugins.scientific.catalog import PACKS, operations
from graphlink_plugins.scientific.core import VERSION, readable_result, run_operation
from graphlink_plugins.scientific.packs import fitting

FIXTURE = json.loads(Path(__file__).with_name("test_scientific_fitting_nist.json").read_text(encoding="utf-8"))


def _run(operation_id, inputs):
operation = next(o for o in operations("fitting") if o.id == operation_id)
return run_operation("fitting", operation, inputs)


def _certified(actual, reference, digits):
"""Compare all *published* significant digits, not an arbitrary tolerance."""
assert format(actual, f".{digits - 1}e") == format(reference, f".{digits - 1}e"), (actual, reference)


def test_manifest_and_all_examples_have_verifiable_receipts():
import tomllib

assert PACKS["fitting"][0] == "Fitting and Inference"
manifest_path = Path(__file__).resolve().parents[2] / "plugins" / "science_fitting" / "plugin.toml"
with manifest_path.open("rb") as f:
manifest = tomllib.load(f)
assert manifest["plugin"]["id"] == "science_fitting"
assert manifest["plugin"]["version"] == VERSION
assert manifest["scopes"]["grants"] == ["graph.read", "graph.mutate"]
assert manifest["persistence"]["passive_node_kinds"] == ["workbench"]
assert {op.id for op in operations("fitting")} == {
"weighted_linear_least_squares", "nonlinear_least_squares", "welch_t_test", "chi_square_gof"
}
for operation in operations("fitting"):
result = _run(operation.id, operation.example)
assert result["provenance"]["packId"] == "fitting"
assert result["provenance"]["packVersion"] == VERSION
assert readable_result({"version": 1, "latest": result}, "fitting")
corrupted = copy.deepcopy(result)
key = next(iter(corrupted["values"]))
corrupted["values"][key] = "corrupted"
assert not readable_result({"version": 1, "latest": corrupted}, "fitting")


def test_longley_all_15_certified_digits_from_the_raw_nist_rows():
item = FIXTURE["longley"]
rows = item["data_y_x1_x2_x3_x4_x5_x6"]
values = _run("weighted_linear_least_squares", {
"predictors": [row[1:] for row in rows], "response": [row[0] for row in rows],
"weights": [1] * len(rows),
})
result = values["values"]
for i, expected in enumerate(item["coefficients"]):
_certified(result[f"coefficient_{i}"], expected, item["digits"])
for i, expected in enumerate(item["standard_errors"]):
_certified(result[f"standard_error_{i}"], expected, item["digits"])
_certified(result["residual_standard_deviation"], item["residual_standard_deviation"], item["digits"])
_certified(result["r_squared"], item["r_squared"], item["digits"])
assert result["degrees_of_freedom"] == 9
assert len(values["table"]["rows"]) == 7 # 7 x 7 coefficient covariance
for i in range(7):
for j in range(7):
_certified(values["table"]["rows"][i][j + 1],
values["table"]["rows"][j][i + 1], 14)
assert math.sqrt(values["table"]["rows"][i][i + 1]) == pytest.approx(result[f"standard_error_{i}"], rel=1e-14)
assert len(values["plot"]["y"]) == 16
assert abs(math.fsum(values["plot"]["y"])) < 1e-7
assert result["weighted_residual_sum_squares"] == pytest.approx(836424.055505915, rel=2e-15)


@pytest.mark.parametrize("case", ["misra1a", "thurber", "mgh09"])
@pytest.mark.parametrize("start_number", [0, 1])
def test_nist_nonlinear_all_11_certified_digits_from_both_starts(case, start_number):
item = FIXTURE[case]
rows = item["data_y_x"]
result = _run("nonlinear_least_squares", {
"model": item["model"], "x": [row[1] for row in rows], "y": [row[0] for row in rows],
"weights": [1] * len(rows), "initial_parameters": item["starts"][start_number],
})
values = result["values"]
assert values["converged"] == "true"
assert 1 <= values["iterations"] < 1201
assert values["accepted_steps"] > 0
assert values["degrees_of_freedom"] == len(rows) - len(item["parameters"])
for i, expected in enumerate(item["parameters"]):
_certified(values[f"parameter_{i}"], expected, item["digits"])
for i, expected in enumerate(item["standard_errors"]):
_certified(values[f"standard_error_{i}"], expected, item["digits"])
_certified(values["weighted_residual_sum_squares"], item["residual_sum_squares"], item["digits"])
assert values["reduced_chi_square"] == pytest.approx(
values["weighted_residual_sum_squares"] / values["degrees_of_freedom"], rel=1e-14
)
assert len(result["plot"]["y"]) == len(rows)
for j in range(len(item["parameters"])):
estimate = values[f"parameter_{j}"]
assert values[f"ci95_lower_{j}"] < estimate < values[f"ci95_upper_{j}"]
assert values[f"ci95_upper_{j}"] - estimate == pytest.approx(
fitting._student_t_ppf(0.975, values["degrees_of_freedom"]) * values[f"standard_error_{j}"],
rel=1e-12, abs=1e-14,
)


@pytest.mark.parametrize("model,parameters,x", [
("exponential", [2.0, -0.3, 4.0], list(range(1, 12))),
("power_law", [3.0, 1.25], list(range(1, 12))),
("gaussian", [2.0, 10.0, 0.5, 1.25], [i / 2 for i in range(-10, 13)]),
("polynomial", [2.0, -1.0, 0.5, 0.125], list(range(-5, 7))),
])
def test_other_nonlinear_families_recover_known_parameters(model, parameters, x):
# Generated from the declared equation and analytic Jacobian, without
# calling the fitter; add small deterministic noise for nonzero SEs.
y = [float(fitting._model(model, fitting._d(xx), list(map(fitting._d, parameters)))[0])
+ (i % 3 - 1) * 0.0001 for i, xx in enumerate(x)]
initial = [v * 1.02 if v else 0.02 for v in parameters]
if model == "gaussian":
initial[2] += 0.1
result = _run("nonlinear_least_squares", {
"model": model, "x": x, "y": y, "weights": [1] * len(x), "initial_parameters": initial,
})
for i, expected in enumerate(parameters):
assert result["values"][f"parameter_{i}"] == pytest.approx(expected, rel=0.003, abs=0.002)
assert result["values"][f"standard_error_{i}"] > 0
assert result["values"]["weighted_residual_sum_squares"] < 1e-6


def test_analytic_model_jacobians_agree_with_centered_finite_differences():
examples = [
("exponential", 2, [2.0, -0.3, 4.0]), ("saturating_exponential", 10, [25, 0.01]),
("power_law", 2, [3, 1.5]), ("gaussian", 0.4, [1, 3, 0.5, 1.3]),
("polynomial", 2, [1, -2, 3]), ("thurber", 0.4, [1000, 1000, 400, 40, 0.7, 0.3, 0.03]),
("mgh09", 0.5, [0.2, 0.2, 0.1, 0.15]),
]
for name, xx, params in examples:
dparams = list(map(fitting._d, params))
_, derivatives = fitting._model(name, fitting._d(xx), dparams)
for j, derivative in enumerate(derivatives):
step = fitting._d(1e-6 * max(abs(params[j]), 1))
higher, lower = dparams.copy(), dparams.copy()
higher[j] += step
lower[j] -= step
high = fitting._model(name, fitting._d(xx), higher)[0]
low = fitting._model(name, fitting._d(xx), lower)[0]
numerical = (high - low) / (2 * step)
assert float(derivative) == pytest.approx(float(numerical), rel=1e-9, abs=1e-9), (name, j)


def test_distribution_functions_against_nist_published_tables():
# https://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm
assert fitting._student_t_cdf(2.228, 10) == pytest.approx(0.975, abs=2e-5)
assert fitting._student_t_cdf(-2.228, 10) == pytest.approx(0.025, abs=2e-5)
assert fitting._student_t_ppf(.975, 10) == pytest.approx(2.228, abs=0.0005)
assert fitting._student_t_ppf(.025, 10) == pytest.approx(-2.228, abs=0.0005)
# https://www.itl.nist.gov/div898/handbook/eda/section3/eda3674.htm
assert fitting._chi_square_cdf(18.307, 10) == pytest.approx(0.95, abs=2e-5)
assert fitting._chi_square_cdf(3.841, 1) == pytest.approx(0.95, abs=2e-5)
assert fitting._normal_cdf(1.959963984540054) == pytest.approx(.975, abs=2e-14)
assert fitting._normal_cdf(0) == .5
assert fitting._normal_cdf(-20) == pytest.approx(0, abs=3e-89)
assert fitting._gamma_q(1, 5) == pytest.approx(math.exp(-5), rel=2e-13)
assert fitting._gamma_q(1, 0) == 1
assert fitting._beta_i(.5, .5, .1) == pytest.approx(2 / math.pi * math.asin(math.sqrt(.1)), rel=2e-14)
assert fitting._beta_i(.5, .5, .9) == pytest.approx(2 / math.pi * math.asin(math.sqrt(.9)), rel=2e-14)
assert fitting._beta_i(1, 1, 0) == 0
assert fitting._beta_i(1, 1, 1) == 1
assert fitting._student_t_cdf(0, 8) == .5
# Independent central Taylor coefficient: t-density at the origin.
# Using I(df/(df+t²)) here would round to 1 and return exactly 0.5.
df, tiny_t = 1000, 1e-8
density_zero = math.exp(math.lgamma((df + 1) / 2) - math.lgamma(df / 2)) / math.sqrt(df * math.pi)
expected_cdf = 0.5 + density_zero * tiny_t
assert fitting._student_t_cdf(tiny_t, df) == pytest.approx(expected_cdf, abs=5e-15)
assert fitting._student_t_cdf(-tiny_t, df) == pytest.approx(1 - expected_cdf, abs=5e-15)
assert fitting._student_t_cdf(1e200, 10) == 1


def test_welch_two_sided_p_and_chi_square_p():
welch = _run("welch_t_test", {"sample_a": [1, 3, 5, 7], "sample_b": [2, 4, 6, 8]})["values"]
# Equal sample variance in this case: t = -sqrt(0.3), df = 6.
assert welch["t_statistic"] == pytest.approx(-math.sqrt(.3), rel=1e-14)
assert welch["degrees_of_freedom"] == pytest.approx(6, rel=1e-14)
assert welch["p_value"] == pytest.approx(
fitting._beta_i(3, .5, 6 / (6 + .3)), rel=1e-14
)
assert 0 < welch["p_value"] < 1
chi = _run("chi_square_gof", {"observed": [24, 36, 40], "expected": [30, 30, 40]})
assert chi["values"]["chi_square"] == pytest.approx(2.4, rel=1e-14)
assert chi["values"]["degrees_of_freedom"] == 2
assert chi["values"]["p_value"] == pytest.approx(math.exp(-1.2), rel=1e-13)
assert sum(row[-1] for row in chi["table"]["rows"]) == pytest.approx(2.4)


@pytest.mark.parametrize("op,inputs", [
("weighted_linear_least_squares", {"predictors": [[1], [2]], "response": [2, 4], "weights": [1, 1]}),
("weighted_linear_least_squares", {"predictors": [[1], [1], [1]], "response": [2, 4, 6], "weights": [1, 1, 1]}),
("weighted_linear_least_squares", {"predictors": [[1], [2], [3]], "response": [2, 4, 6], "weights": [1, 0, 0]}),
("weighted_linear_least_squares", {"predictors": [[1], [2], [3]], "response": [2, 4, 6], "weights": [1, -1, 1]}),
("weighted_linear_least_squares", {"predictors": [[1], [2], [3]], "response": [2, 4], "weights": [1, 1, 1]}),
("nonlinear_least_squares", {"model": "bogus", "x": [1, 2, 3], "y": [2, 4, 6], "weights": [1, 1, 1], "initial_parameters": [1, 2]}),
("nonlinear_least_squares", {"model": "power_law", "x": [0, 1, 2], "y": [2, 3, 4], "weights": [1, 1, 1], "initial_parameters": [1, 2]}),
("nonlinear_least_squares", {"model": "gaussian", "x": [1, 2, 3, 4, 5], "y": [1, 2, 4, 2, 1], "weights": [1] * 5, "initial_parameters": [1, 3, 2, 0]}),
("nonlinear_least_squares", {"model": "mgh09", "x": [0, 1, 2, 3, 4], "y": [1, 2, 3, 4, 5], "weights": [1] * 5, "initial_parameters": [1, 2, 3]}),
("nonlinear_least_squares", {"model": "polynomial", "x": [1, 2, 3], "y": [1, 2, 3], "weights": [1] * 3, "initial_parameters": [1, 1, 1]}),
("welch_t_test", {"sample_a": [1], "sample_b": [1, 3]}),
("welch_t_test", {"sample_a": [1, 1], "sample_b": [2, 2]}),
("chi_square_gof", {"observed": [1, 2], "expected": [1, 1]}),
("chi_square_gof", {"observed": [1, -1], "expected": [1, 1]}),
("chi_square_gof", {"observed": [1.5, 2.5], "expected": [2, 2]}),
("chi_square_gof", {"observed": [2, 2], "expected": [0, 4]}),
("chi_square_gof", {"observed": [10], "expected": [10]}),
("weighted_linear_least_squares", {"predictors": [[0], [1], [2]], "response": [0, 1, float("nan")], "weights": [1, 1, 1]}),
])
def test_invalid_inputs_rejected_by_operation_and_shared_validator(op, inputs):
with pytest.raises(ValueError):
_run(op, inputs)


@pytest.mark.parametrize("function,args", [
(fitting._gamma_q, (0, 1)), (fitting._gamma_q, (2, -1)),
(fitting._beta_i, (-1, 1, .5)), (fitting._beta_i, (1, 2, 1.1)),
(fitting._student_t_cdf, (1, 0)), (fitting._student_t_ppf, (1, 2)),
])
def test_invalid_distribution_domains_are_rejected(function, args):
with pytest.raises(ValueError):
function(*args)


def test_weighted_linear_matches_independent_hand_calculation():
# Fit y = 1 + 2*x with a nonzero noisy endpoint and one ignored row.
# All normal-equation sums below can be verified by hand.
result = _run("weighted_linear_least_squares", {
"predictors": [[-1], [0], [1], [2], [10]],
"response": [-1, 1, 3, 6, 1e5], "weights": [1, 1, 1, 1, 0],
})
values = result["values"]
assert values["positive_weight_observations"] == 4
assert values["degrees_of_freedom"] == 2
assert values["coefficient_0"] == pytest.approx(1.1)
assert values["coefficient_1"] == pytest.approx(2.3)
assert values["residual_4"] == pytest.approx(1e5 - 24.1)
assert values["r_squared"] <= 1
assert values["r_squared"] >= 0


def test_constant_response_r_squared_undefined_and_singular_nonlinear_rejected():
linear = _run("weighted_linear_least_squares", {
"predictors": [[0], [1], [2], [3]],
"response": [4, 4, 4, 4], "weights": [1] * 4,
})
assert "r_squared" not in linear["values"]
with pytest.raises(ValueError, match="rank deficient|unidentifiable"):
_run("nonlinear_least_squares", {
"model": "polynomial", "x": [1, 1, 1, 1],
"y": [1, 2, 3, 4], "weights": [1] * 4,
"initial_parameters": [0, 0],
})


def test_float_underflow_is_rejected_not_silently_rounded_to_zero():
from decimal import Decimal
with pytest.raises(ValueError, match="numeric range"):
fitting._as_float(Decimal("1e-500"), "Tiny covariance")
55 changes: 55 additions & 0 deletions backend/tests/test_scientific_fitting_nist.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{
"source": "https://www.itl.nist.gov/div898/strd/",
"longley": {
"data_y_x1_x2_x3_x4_x5_x6": [
[60323,83.0,234289,2356,1590,107608,1947],
[61122,88.5,259426,2325,1456,108632,1948],
[60171,88.2,258054,3682,1616,109773,1949],
[61187,89.5,284599,3351,1650,110929,1950],
[63221,96.2,328975,2099,3099,112075,1951],
[63639,98.1,346999,1932,3594,113270,1952],
[64989,99.0,365385,1870,3547,115094,1953],
[63761,100.0,363112,3578,3350,116219,1954],
[66019,101.2,397469,2904,3048,117388,1955],
[67857,104.6,419180,2822,2857,118734,1956],
[68169,108.4,442769,2936,2798,120445,1957],
[66513,110.8,444546,4681,2637,121950,1958],
[68655,112.6,482704,3813,2552,123366,1959],
[69564,114.2,502601,3931,2514,125368,1960],
[69331,115.7,518173,4806,2572,127852,1961],
[70551,116.9,554894,4007,2827,130081,1962]
],
"coefficients": [-3482258.63459582,15.0618722713733,-0.0358191792925910,-2.02022980381683,-1.03322686717359,-0.0511041056535807,1829.15146461355],
"standard_errors": [890420.383607373,84.9149257747669,0.0334910077722432,0.488399681651699,0.214274163161675,0.226073200069370,455.478499142212],
"residual_standard_deviation": 304.854073561965,
"r_squared": 0.995479004577296,
"digits": 15
},
"misra1a": {
"model": "saturating_exponential",
"data_y_x": [[10.07,77.6],[14.73,114.9],[17.94,141.1],[23.93,190.8],[29.61,239.9],[35.18,289.0],[40.02,332.8],[44.82,378.4],[50.76,434.8],[55.05,477.3],[61.01,536.8],[66.40,593.1],[75.47,689.1],[81.78,760.0]],
"starts": [[500,0.0001],[250,0.0005]],
"parameters": [238.94212918,0.00055015643181],
"standard_errors": [2.7070075241,0.0000072668688436],
"residual_sum_squares": 0.12455138894,
"digits": 11
},
"thurber": {
"model": "thurber",
"data_y_x": [[80.574,-3.067],[84.248,-2.981],[87.264,-2.921],[87.195,-2.912],[89.076,-2.840],[89.608,-2.797],[89.868,-2.702],[90.101,-2.699],[92.405,-2.633],[95.854,-2.481],[100.696,-2.363],[101.060,-2.322],[401.672,-1.501],[390.724,-1.460],[567.534,-1.274],[635.316,-1.212],[733.054,-1.100],[759.087,-1.046],[894.206,-0.915],[990.785,-0.714],[1090.109,-0.566],[1080.914,-0.545],[1122.643,-0.400],[1178.351,-0.309],[1260.531,-0.109],[1273.514,-0.103],[1288.339,0.010],[1327.543,0.119],[1353.863,0.377],[1414.509,0.790],[1425.208,0.963],[1421.384,1.006],[1442.962,1.115],[1464.350,1.572],[1468.705,1.841],[1447.894,2.047],[1457.628,2.200]],
"starts": [[1000,1000,400,40,0.7,0.3,0.03],[1300,1500,500,75,1,0.4,0.05]],
"parameters": [1288.1396800,1491.0792535,583.23836877,75.416644291,0.96629502864,0.39797285797,0.049727297349],
"standard_errors": [4.6647963344,39.571156086,28.698696102,5.5675370270,0.031333340687,0.014984928198,0.0065842344623],
"residual_sum_squares": 5642.7082397,
"digits": 11
},
"mgh09": {
"model": "mgh09",
"data_y_x": [[0.1957,4.0],[0.1947,2.0],[0.1735,1.0],[0.1600,0.5],[0.0844,0.25],[0.0627,0.167],[0.0456,0.125],[0.0342,0.1],[0.0323,0.0833],[0.0235,0.0714],[0.0246,0.0625]],
"starts": [[25,39,41.5,39],[0.25,0.39,0.415,0.39]],
"parameters": [0.19280693458,0.19128232873,0.12305650693,0.13606233068],
"standard_errors": [0.011435312227,0.19633220911,0.080842031232,0.090025542308],
"residual_sum_squares": 0.00030750560385,
"digits": 11
}
}
Loading
Loading