Monitoring and MLOps

import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression

rng = np.random.default_rng(11)
predictors = ["age", "egfr", "albuminuria"]

def month_of_patients(n, assay_bias=0.0):
    d = pd.DataFrame({"age": rng.normal(60, 10, n).round(),
                      "true_egfr": rng.normal(70, 20, n).clip(15, 120),
                      "albuminuria": rng.binomial(1, 0.3, n)})
    logit = -2.0 + 0.02 * (d.age - 60) - 0.04 * (d.true_egfr - 70) + 1.0 * d.albuminuria
    d["outcome"] = rng.binomial(1, 1 / (1 + np.exp(-logit)))
    d["egfr"] = (d.true_egfr + assay_bias).round()
    return d

development = month_of_patients(8000)
model = LogisticRegression(max_iter=1000).fit(development[predictors], development.outcome)

def psi(expected, actual, bins=10):
    """Population stability index: how far a variable's distribution has moved from baseline."""
    edges = np.quantile(expected, np.linspace(0, 1, bins + 1))
    edges[0], edges[-1] = -np.inf, np.inf
    e = np.histogram(expected, edges)[0] / len(expected)
    a = np.histogram(actual, edges)[0] / len(actual)
    e, a = np.clip(e, 1e-6, None), np.clip(a, 1e-6, None)
    return float(np.sum((a - e) * np.log(a / e)))

print("month  eGFR PSI  observed  expected  O/E   alert")
for month in range(1, 13):
    d = month_of_patients(600, assay_bias=15 if month >= 7 else 0)   # new lab assay from month 7
    expected = model.predict_proba(d[predictors])[:, 1].mean()
    observed = d.outcome.mean()
    drift = psi(development.egfr, d.egfr)
    alerts = [a for a, bad in [("input drift", drift > 0.2),
                               ("calibration", not 0.8 <= observed / expected <= 1.25)] if bad]
    print(f"{month:>5}  {drift:8.2f}  {observed:8.1%}  {expected:8.1%}  "
          f"{observed / expected:4.2f}  {', '.join(alerts)}")

# month  eGFR PSI  observed  expected  O/E   alert
#     1      0.01     18.0%     19.8%  0.91
#     2      0.01     22.3%     19.1%  1.17
#     3      0.01     18.8%     19.3%  0.98
#     4      0.02     20.8%     19.3%  1.08
#     5      0.01     20.3%     19.2%  1.06
#     6      0.02     19.7%     17.5%  1.12
#     7      0.54     19.3%     12.3%  1.57  input drift, calibration
#     8      0.50     19.0%     12.7%  1.49  input drift, calibration
#     9      0.57     18.7%     12.4%  1.51  input drift, calibration
#    10      0.55     18.7%     12.1%  1.54  input drift, calibration
#    11      0.64     20.2%     11.7%  1.72  input drift, calibration
#    12      0.61     20.7%     11.8%  1.76  input drift, calibration