"""
Poisson deviance statistics: per-gate terms, the summed and reduced deviance, and its
expectation (and variance) under the model for photon counts.
"""
from functools import lru_cache
from typing import Any
import numpy as np
from scipy.special import gammaln, xlogy
NONPOSITIVE_MODES = ("zero", "nan", "floor")
MU_MIN = 1e-8
MU_SWITCH = 1e3
N_GRID = 4401
[docs]
def poisson_deviance_terms(
model: Any,
data: Any,
floor: float = 1e-12,
nonpositive: str = "zero",
data_floor: float = 1e-9,
) -> Any:
"""
Per-gate Poisson deviance ``2 (mu - d + d ln(d / mu))`` with ``mu`` floored at `floor`.
`nonpositive` sets the ``d ln(d / mu)`` term of gates with ``d <= 0``: ``"zero"`` (0),
``"nan"`` (scipy ``xlogy``: 0 for ``d = 0``, NaN for ``d < 0``) or ``"floor"``
(``d ln(max(d, data_floor) / mu)`` on every gate).
"""
if nonpositive not in NONPOSITIVE_MODES:
raise ValueError(
f"nonpositive must be one of {NONPOSITIVE_MODES}, got {nonpositive!r}."
)
mu = np.clip(np.asarray(model, dtype=float), floor, None)
d = np.asarray(data, dtype=float)
if nonpositive == "nan":
return 2.0 * (xlogy(d, d / mu) - (d - mu))
if nonpositive == "floor":
return 2.0 * (mu - d + d * np.log(np.clip(d, data_floor, None) / mu))
positive = d > 0
log_term = np.where(positive, d * np.log(np.where(positive, d, 1.0) / mu), 0.0)
return 2.0 * (mu - d + log_term)
[docs]
def poisson_deviance(model: Any, data: Any, axis: int = -1, **kwargs: Any) -> Any:
"""
Poisson deviance: :func:`poisson_deviance_terms` summed along `axis` (gates with
``d <= 0`` contribute ``2 * (mu - d)`` by default). For counts drawn from the model,
``deviance / (sum E[D_k] - p)`` averages 1 (see :func:`reduced_poisson_deviance`).
"""
return np.sum(poisson_deviance_terms(model, data, **kwargs), axis=axis)
[docs]
@lru_cache(maxsize=4)
def poisson_deviance_moment_table(
mu_min: float = MU_MIN, mu_max: float = MU_SWITCH, n_grid: int = N_GRID
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""``(ln mu, E[d], Var[d])`` of the per-gate deviance on a log grid of ``mu``, exact by summation."""
mu_grid = np.logspace(np.log10(mu_min), np.log10(mu_max), n_grid)
E = np.empty(n_grid)
V = np.empty(n_grid)
for i, m in enumerate(mu_grid):
half_width = 14.0 * np.sqrt(m) + 30.0
y = np.arange(
max(0, int(m - half_width)), int(np.ceil(m + half_width)) + 1, dtype=float
)
p = np.exp(xlogy(y, m) - m - gammaln(y + 1.0))
d = poisson_deviance_terms(m, y, nonpositive="nan")
E[i] = np.sum(p * d)
V[i] = np.sum(p * (d - E[i]) ** 2)
return np.log(mu_grid), E, V
[docs]
def poisson_deviance_moments(model: Any) -> tuple[Any, Any]:
"""
Exact ``(E[d], Var[d])`` of the per-gate deviance for ``d ~ Poisson(mu)``, element-wise:
tabulated below ``mu = 1e3`` (0 at ``mu = 0``), then ``E = 1 + 1/(6 mu) + 1/(6 mu^2)``
and ``Var = 2 + 2/(3 mu) + 4/(3 mu^2)``.
"""
mu = np.clip(np.asarray(model, dtype=float), 0.0, None)
log_grid, E_tab, V_tab = poisson_deviance_moment_table()
safe = np.where(mu > 0, mu, MU_MIN)
log_mu = np.log(safe)
small = mu < MU_SWITCH
positive = mu > 0
E = np.where(
small,
np.where(positive, np.interp(log_mu, log_grid, E_tab, left=0.0), 0.0),
1.0 + 1.0 / (6.0 * safe) + 1.0 / (6.0 * safe**2),
)
V = np.where(
small,
np.where(positive, np.interp(log_mu, log_grid, V_tab, left=0.0), 0.0),
2.0 + 2.0 / (3.0 * safe) + 4.0 / (3.0 * safe**2),
)
return E, V
[docs]
def expected_poisson_deviance(model: Any) -> Any:
"""
Expected Poisson deviance ``E[2 * (mu - d + d * ln(d / mu))]`` of a gate with
expected count `mu` (element-wise), for ``d ~ Poisson(mu)``: about ``2 mu
ln(1/mu)`` for ``mu -> 0``, peaks near 1.15 at ``mu ~ 1`` and tends to
``1 + 1/(6 mu)`` for large ``mu`` (:func:`poisson_deviance_moments`).
"""
return poisson_deviance_moments(model)[0]
[docs]
def deviance_dof(model: Any, n_params: int, weights: Any = None, axis: int = -1) -> Any:
"""Expected deviance minus fitted parameters, ``sum_k w_k E[d_k] - n_params``: the reduced-deviance denominator."""
expected = expected_poisson_deviance(model)
if weights is not None:
expected = weights * expected
return np.sum(expected, axis=axis) - n_params
[docs]
def reduced_poisson_deviance(
model: Any, data: Any, n_params: int, axis: int = -1
) -> tuple[Any, Any]:
"""
``(D, D_reduced)``: the Poisson deviance along `axis` and the deviance divided
by its expectation under the model minus the number of fitted parameters,
``D / max(sum E[D_k] - n_params, 1)``, which averages 1 for a correct model.
"""
deviance = poisson_deviance(model, data, axis=axis)
return deviance, deviance / np.maximum(
deviance_dof(model, n_params, axis=axis), 1.0
)