End-to-end RAG evaluation

TESTS = [   # (question, section that answers it or None, key facts a correct answer must contain)
    ("When should metformin be stopped?", "s2", ["below 30"]),
    ("Can metformin be started with an eGFR of 40?", "s2", ["below 45"]),
    ("What should I do about a hypo?", "s4", ["15 g", "15 minutes"]),
    ("How is a low glucose treated?", "s4", ["15 g"]),
    ("How often should HbA1c be checked after a change in treatment?", "s1", ["3 months"]),
    ("How often is foot sensation tested?", "s3", ["once a year"]),
    ("When should an ulcer be referred urgently?", "s3", ["spreading redness"]),
    ("When does eye screening start?", "s5", ["at diagnosis"]),
    ("What dose of semaglutide should I start?", None, []),
    ("Is it safe to fast during Ramadan with diabetes?", None, []),
]

def evaluate(**settings):
    retrieved = correct = refused_right = answerable = unanswerable = 0
    for q, gold, facts in TESTS:
        r = answer(q, **settings)
        cited = r["citations"][0].split(":")[0] if r["citations"] else None
        if gold is None:
            unanswerable += 1
            refused_right += cited is None
            continue
        answerable += 1
        retrieved += cited == gold
        correct += cited == gold and all(f in r["answer"] for f in facts)
    print(f"{str(settings):<42} retrieval {retrieved}/{answerable}  correct {correct}/{answerable}  "
          f"declined unanswerable {refused_right}/{unanswerable}")

evaluate(max_sentences=2)
evaluate(max_sentences=None)
evaluate(max_sentences=None, min_score=0.45)

# {'max_sentences': 2}                       retrieval 8/8  correct 6/8  declined unanswerable 2/2
# {'max_sentences': None}                    retrieval 8/8  correct 8/8  declined unanswerable 2/2
# {'max_sentences': None, 'min_score': 0.45} retrieval 8/8  correct 8/8  declined unanswerable 2/2

scores = [(answer(q, min_score=0.0)["retrieval_score"], gold is not None, q) for q, gold, _ in TESTS]
for s, answerable, q in sorted(scores):
    print(f"{s:.2f}  {'answerable  ' if answerable else 'out of scope'}  {q}")
lowest_answerable = min(s for s, a, _ in scores if a)
highest_unanswerable = max(s for s, a, _ in scores if not a)
print(f"\nMargin: lowest answerable {lowest_answerable:.2f}, highest out-of-scope {highest_unanswerable:.2f}")

# 0.31  out of scope  Is it safe to fast during Ramadan with diabetes?
# 0.32  out of scope  What dose of semaglutide should I start?
# 0.49  answerable    When does eye screening start?
# 0.51  answerable    When should an ulcer be referred urgently?
# 0.53  answerable    When should metformin be stopped?
# 0.55  answerable    Can metformin be started with an eGFR of 40?
# 0.63  answerable    What should I do about a hypo?
# 0.66  answerable    How often is foot sensation tested?
# 0.67  answerable    How is a low glucose treated?
# 0.77  answerable    How often should HbA1c be checked after a change in treatment?
#
# Margin: lowest answerable 0.49, highest out-of-scope 0.32