Chunking clinical guidelines

PROTOCOL = """# Synthetic diabetes clinic protocol (teaching document, not clinical guidance)

## 1. Monitoring glucose control
Check HbA1c every 3 months after any change in treatment. Once targets are met and treatment is stable, check HbA1c every 6 months. A common target for many non-pregnant adults is below 7%, adjusted for the individual.

## 2. Kidney function and metformin
Check eGFR and urine albumin-to-creatinine ratio at least once a year. Metformin should not be started if eGFR is below 45. If eGFR falls below 30 during treatment, stop metformin.

## 3. Foot care
Examine both feet at every review. Test sensation with a 10 g monofilament at least once a year. Refer urgently if there is an ulcer with spreading redness, a suspected infection of the bone, or a cold, pale foot.

## 4. Low blood glucose
A glucose below 70 mg/dL is low. Treat with 15 g of fast-acting carbohydrate and recheck after 15 minutes. Repeat if glucose is still below 70 mg/dL. Review any medicine that may have caused the low glucose.

## 5. Eye screening
Arrange retinal screening at diagnosis of type 2 diabetes and then at the interval set by the local screening programme.
"""

def fixed_size_chunks(text, size=40):
    words = text.split()
    return [" ".join(words[i:i + size]) for i in range(0, len(words), size)]

chunks = fixed_size_chunks(PROTOCOL)
print(f"{len(chunks)} chunks of up to 40 words\n")
for i, c in enumerate(chunks[2:5], start=2):
    print(f"[{i}] {c}\n")

# 5 chunks of up to 40 words
#
# [2] eGFR falls below 30 during treatment, stop metformin. ## 3. Foot care Examine both feet at every review. Test sensation with a 10 g monofilament at least once a year. Refer urgently if there is an ulcer with spreading redness,
#
# [3] a suspected infection of the bone, or a cold, pale foot. ## 4. Low blood glucose A glucose below 70 mg/dL is low. Treat with 15 g of fast-acting carbohydrate and recheck after 15 minutes. Repeat if glucose is still
#
# [4] below 70 mg/dL. Review any medicine that may have caused the low glucose. ## 5. Eye screening Arrange retinal screening at diagnosis of type 2 diabetes and then at the interval set by the local screening programme.

import re

def section_chunks(text):
    """Split on '## ' headings; keep the heading with its text so each chunk stands alone."""
    title = text.splitlines()[0].lstrip("# ").strip()
    parts = re.split(r"\n(?=## )", text)[1:]
    return [{"id": f"s{i + 1}", "text": f"{title} > {p.strip()}"} for i, p in enumerate(parts)]

sections = section_chunks(PROTOCOL)
print(f"{len(sections)} section chunks")
print(sections[1]["text"])

# 5 section chunks
# Synthetic diabetes clinic protocol (teaching document, not clinical guidance) > ## 2. Kidney function and metformin
# Check eGFR and urine albumin-to-creatinine ratio at least once a year. Metformin should not be started if eGFR is below 45. If eGFR falls below 30 during treatment, stop metformin.

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def top_chunk(chunk_texts, question):
    vec = TfidfVectorizer(stop_words="english").fit(chunk_texts + [question])
    scores = cosine_similarity(vec.transform([question]), vec.transform(chunk_texts))[0]
    return chunk_texts[scores.argmax()]

question = "When should metformin be stopped because of kidney function?"
print("Fixed-size chunk retrieved:\n ", top_chunk(fixed_size_chunks(PROTOCOL), question))
print("\nSection chunk retrieved:\n ", top_chunk([s["text"] for s in section_chunks(PROTOCOL)], question))

# Fixed-size chunk retrieved:
#   target for many non-pregnant adults is below 7%, adjusted for the individual. ## 2. Kidney function and metformin Check eGFR and urine albumin-to-creatinine ratio at least once a year. Metformin should not be started if eGFR is below 45. If
#
# Section chunk retrieved:
#   Synthetic diabetes clinic protocol (teaching document, not clinical guidance) > ## 2. Kidney function and metformin
# Check eGFR and urine albumin-to-creatinine ratio at least once a year. Metformin should not be started if eGFR is below 45. If eGFR falls below 30 during treatment, stop metformin.