Hallucination and grounding

reviews = [
    {"note_sentences": 14, "hallucinations": [], "omissions": ["allergy"]},
    {"note_sentences": 11, "hallucinations": [("dose changed", "major")], "omissions": []},
    {"note_sentences": 16, "hallucinations": [("extra symptom", "minor")],
     "omissions": ["follow-up date", "smoking status"]},
    {"note_sentences": 12, "hallucinations": [], "omissions": []},
]
sentences = sum(r["note_sentences"] for r in reviews)
halluc = [h for r in reviews for h in r["hallucinations"]]
omissions = sum(len(r["omissions"]) for r in reviews)
major = sum(1 for _, severity in halluc if severity == "major")
print(f"{sentences} sentences reviewed")
print(f"Hallucination rate {len(halluc) / sentences:.1%} ({major} major)")
print(f"Omissions: {omissions} across {len(reviews)} notes")
print(f"Notes with at least one error: "
      f"{sum(1 for r in reviews if r['hallucinations'] or r['omissions'])} of {len(reviews)}")

# 53 sentences reviewed
# Hallucination rate 3.8% (1 major)
# Omissions: 3 across 4 notes
# Notes with at least one error: 3 of 4

import re

source = """Synthetic consultation. Patient reports tiredness for three weeks.
On metformin 1 g twice daily. HbA1c last month 8.4%. Blood pressure today 138/86.
No chest pain. Plan: add a second medicine after discussing options; recheck HbA1c in 3 months."""

summary_sentences = [
    "Three weeks of tiredness.",
    "Takes metformin 1 g twice daily.",
    "HbA1c 8.4% last month; blood pressure 138/86.",
    "Started on gliclazide 80 mg.",          # not in the source
    "Reports chest pain on exertion.",       # contradicts the source
    "Recheck HbA1c in 6 months.",            # wrong interval
]

MEDICINES = {"metformin", "gliclazide", "insulin", "sitagliptin", "empagliflozin"}
src = source.lower()
src_numbers = set(re.findall(r"\d+(?:\.\d+)?", src))

def unsupported_details(sentence):
    s = sentence.lower()
    problems = [f"number {n}" for n in re.findall(r"\d+(?:\.\d+)?", s) if n not in src_numbers]
    problems += [f"medicine {m}" for m in MEDICINES if m in s and m not in src]
    return problems

for sentence in summary_sentences:
    problems = unsupported_details(sentence)
    print(("FLAG  " if problems else "ok    ") + sentence, problems or "")

# ok    Three weeks of tiredness.
# ok    Takes metformin 1 g twice daily.
# ok    HbA1c 8.4% last month; blood pressure 138/86.
# FLAG  Started on gliclazide 80 mg. ['number 80', 'medicine gliclazide']
# ok    Reports chest pain on exertion.
# FLAG  Recheck HbA1c in 6 months. ['number 6']