import { CASES } from "./cases";
import { assess, extractLocal, toInput } from "./context";
import type { Evaluation, Review, Source } from "../src/types";
const all = (sources: Source[]): Review =>
Object.fromEntries(extractLocal(sources).facts.map((f) => [f.id, "include"]));
const check = (sources: Source[]) =>
assess(sources, all(sources), extractLocal(sources).fingerprint, false);
const blocks = (fn: () => unknown) => {
try {
fn();
return false;
} catch {
return true;
}
};
export function runEvaluation(): Evaluation {
const start = performance.now();
const tests: Evaluation["tests"] = [];
const run = (name: string, group: string, test: () => boolean) => {
try {
tests.push({ name, group, passed: test() });
} catch {
tests.push({ name, group, passed: false });
}
};
const renal = CASES[0].sources;
run("Every extracted fact points to an exact source line", "Provenance", () =>
extractLocal(renal).facts.every(
(f) =>
renal.find((s) => s.id === f.sourceId)!.text.split("\n")[f.line - 1] ===
f.quote,
),
);
run(
"Kidney context adds the existing metformin / eGFR rule",
"Clinical-rule integration",
() => check(renal).addedRuleIds.includes("lab-metformin-egfr-45"),
);
run(
"The same medicine list without labs has no renal finding",
"Clinical-rule integration",
() =>
!check(renal).baseline.findings.some((f) =>
f.ruleId.startsWith("lab-metformin"),
),
);
run(
"A non-triggering eGFR value does not produce the low-eGFR rule",
"Clinical-rule integration",
() =>
!check(
renal.map((s) => ({
...s,
text: s.text.replace("value: 38", "value: 90"),
})),
).addedRuleIds.some((id) => id.startsWith("lab-metformin")),
);
run(
"Cross-record penicillin history adds an allergy finding",
"Clinical-rule integration",
() => check(CASES[1].sources).addedRuleIds.includes("allergy-penicillin"),
);
run(
"Unknown allergies remain unknown",
"Uncertainty",
() => toInput(extractLocal(renal).facts).allergiesStatus === "not_sure",
);
run(
"Condition history remains explicitly outside rule coverage",
"Uncertainty",
() =>
check(renal).contextual.what_we_couldnt_check.some(
(f) => f.reason === "condition",
),
);
run("Unreviewed facts block assessment", "Review gate", () =>
blocks(() => assess(renal, {}, extractLocal(renal).fingerprint, false)),
);
run("Source edits invalidate the previous fingerprint", "Review gate", () =>
blocks(() =>
assess(
renal.map((s) => ({ ...s, text: s.text + "\nAge: 63" })),
all(renal),
extractLocal(renal).fingerprint,
false,
),
),
);
run(
"Conflicting medication strengths block assessment",
"Reconciliation",
() => blocks(() => check(CASES[2].sources)),
);
run(
"Explicit exclusion resolves the medication discrepancy",
"Reconciliation",
() => {
const sources = CASES[2].sources;
const draft = extractLocal(sources);
const review = all(sources);
review[
draft.facts.find(
(f) => f.kind === "medicine" && f.sourceId === "fatima-clinic",
)!.id
] = "exclude";
return (
assess(sources, review, draft.fingerprint, false).excludedFacts
.length === 1
);
},
);
run("Unsupported lab units block inclusion", "Input validation", () =>
blocks(() =>
check(
renal.map((s) => ({
...s,
text: s.text.replace("mL/min/1.73m2", "mg/dL"),
})),
),
),
);
run("Missing lab dates block inclusion", "Input validation", () =>
blocks(() =>
check(
renal.map((s) => ({
...s,
text: s.text.replace("; date: 2026-10-12", ""),
})),
),
),
);
run(
"Clinical sign-off remains pending",
"Evidence boundary",
() => check(renal).contextual.rulesReviewed === false,
);
return {
tests,
passed: tests.filter((t) => t.passed).length,
total: tests.length,
durationMs: Math.round(performance.now() - start),
runAt: new Date().toISOString(),
};
}