marmar-context / web / context / server / evaluation.ts
evaluation.ts
Raw
import { CASES } from "./cases";
import { assess, extractLocal, toInput } from "./context";
import type { Evaluation, Review, Source } from "../src/types";

const all = (sources: Source[]): Review =>
  Object.fromEntries(extractLocal(sources).facts.map((f) => [f.id, "include"]));
const check = (sources: Source[]) =>
  assess(sources, all(sources), extractLocal(sources).fingerprint, false);
const blocks = (fn: () => unknown) => {
  try {
    fn();
    return false;
  } catch {
    return true;
  }
};
export function runEvaluation(): Evaluation {
  const start = performance.now();
  const tests: Evaluation["tests"] = [];
  const run = (name: string, group: string, test: () => boolean) => {
    try {
      tests.push({ name, group, passed: test() });
    } catch {
      tests.push({ name, group, passed: false });
    }
  };
  const renal = CASES[0].sources;
  run("Every extracted fact points to an exact source line", "Provenance", () =>
    extractLocal(renal).facts.every(
      (f) =>
        renal.find((s) => s.id === f.sourceId)!.text.split("\n")[f.line - 1] ===
        f.quote,
    ),
  );
  run(
    "Kidney context adds the existing metformin / eGFR rule",
    "Clinical-rule integration",
    () => check(renal).addedRuleIds.includes("lab-metformin-egfr-45"),
  );
  run(
    "The same medicine list without labs has no renal finding",
    "Clinical-rule integration",
    () =>
      !check(renal).baseline.findings.some((f) =>
        f.ruleId.startsWith("lab-metformin"),
      ),
  );
  run(
    "A non-triggering eGFR value does not produce the low-eGFR rule",
    "Clinical-rule integration",
    () =>
      !check(
        renal.map((s) => ({
          ...s,
          text: s.text.replace("value: 38", "value: 90"),
        })),
      ).addedRuleIds.some((id) => id.startsWith("lab-metformin")),
  );
  run(
    "Cross-record penicillin history adds an allergy finding",
    "Clinical-rule integration",
    () => check(CASES[1].sources).addedRuleIds.includes("allergy-penicillin"),
  );
  run(
    "Unknown allergies remain unknown",
    "Uncertainty",
    () => toInput(extractLocal(renal).facts).allergiesStatus === "not_sure",
  );
  run(
    "Condition history remains explicitly outside rule coverage",
    "Uncertainty",
    () =>
      check(renal).contextual.what_we_couldnt_check.some(
        (f) => f.reason === "condition",
      ),
  );
  run("Unreviewed facts block assessment", "Review gate", () =>
    blocks(() => assess(renal, {}, extractLocal(renal).fingerprint, false)),
  );
  run("Source edits invalidate the previous fingerprint", "Review gate", () =>
    blocks(() =>
      assess(
        renal.map((s) => ({ ...s, text: s.text + "\nAge: 63" })),
        all(renal),
        extractLocal(renal).fingerprint,
        false,
      ),
    ),
  );
  run(
    "Conflicting medication strengths block assessment",
    "Reconciliation",
    () => blocks(() => check(CASES[2].sources)),
  );
  run(
    "Explicit exclusion resolves the medication discrepancy",
    "Reconciliation",
    () => {
      const sources = CASES[2].sources;
      const draft = extractLocal(sources);
      const review = all(sources);
      review[
        draft.facts.find(
          (f) => f.kind === "medicine" && f.sourceId === "fatima-clinic",
        )!.id
      ] = "exclude";
      return (
        assess(sources, review, draft.fingerprint, false).excludedFacts
          .length === 1
      );
    },
  );
  run("Unsupported lab units block inclusion", "Input validation", () =>
    blocks(() =>
      check(
        renal.map((s) => ({
          ...s,
          text: s.text.replace("mL/min/1.73m2", "mg/dL"),
        })),
      ),
    ),
  );
  run("Missing lab dates block inclusion", "Input validation", () =>
    blocks(() =>
      check(
        renal.map((s) => ({
          ...s,
          text: s.text.replace("; date: 2026-10-12", ""),
        })),
      ),
    ),
  );
  run(
    "Clinical sign-off remains pending",
    "Evidence boundary",
    () => check(renal).contextual.rulesReviewed === false,
  );
  return {
    tests,
    passed: tests.filter((t) => t.passed).length,
    total: tests.length,
    durationMs: Math.round(performance.now() - start),
    runAt: new Date().toISOString(),
  };
}