marmar-context / web / context / scripts / load-evaluation.mjs
load-evaluation.mjs
Raw
import fs from "node:fs/promises";
import { setTimeout as delay } from "node:timers/promises";
const base = "http://127.0.0.1:4191/api/";
const soak = process.argv.includes("--soak");
async function api(path, body) {
  const start = performance.now();
  const r = await fetch(base + path, {
    ...(body
      ? {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify(body),
        }
      : {}),
    signal: AbortSignal.timeout(28000),
  });
  return {
    status: r.status,
    ms: Math.round(performance.now() - start),
    data: await r.json(),
  };
}
const status = (await api("status")).data;
if (!status.modelConfigured || !status.modelLoad)
  throw new Error("Connect live model on updated server first.");
const report = {
  runAt: new Date().toISOString(),
  scope:
    "Bounded local live Azure load test, synthetic data, fresh calls. Not production capacity certification.",
  model: status.model,
  rows: [],
  health: [],
};
let polling = true;
const start = performance.now();
const poll = (async () => {
  while (polling) {
    report.health.push(await api("status"));
    await delay(200);
  }
})();
async function flow(id, phase) {
  const sources = [
    {
      id: "load-" + id,
      title: "Synthetic load " + id,
      kind: "Synthetic",
      date: "2026-10-23",
      text: `Patient: Load Test ${id}\nAge: 62\nShe takes Metformin 500 mg twice daily.\nHer eGFR was 38 mL/min/1.73m2 on 2026-10-12.`,
    },
  ];
  const x = await api("extract", {
    sources,
    syntheticOnly: true,
    useModel: true,
    freshModel: true,
  });
  const row = { id, phase, status: x.status, extractionMs: x.ms };
  report.rows.push(row);
  if (x.status !== 200) {
    row.error = x.data.error;
    return;
  }
  const d = x.data;
  row.calls = d.metrics.providerCalls;
  row.attempts = d.attempts;
  const usable = d.facts.filter((f) => !f.problem);
  const expected =
    usable.some((f) => f.kind === "patient" && f.label === `Load Test ${id}`) &&
    usable.some((f) => f.kind === "medicine" && f.label === "Metformin") &&
    usable.some((f) => f.kind === "lab" && f.label === "eGFR 38");
  const review = Object.fromEntries(
    d.facts.map((f) => [f.id, f.problem ? "exclude" : "include"]),
  );
  const a = await api("assess", {
    sources,
    syntheticOnly: true,
    draftId: d.id,
    review,
    fingerprint: d.fingerprint,
    acknowledgeUnparsed: true,
  });
  let exported = false;
  if (a.status === 200) {
    const r = await api("reports/" + a.data.reportId);
    exported =
      r.status === 200 &&
      JSON.stringify(r.data.sources) === JSON.stringify(sources) &&
      r.data.extraction.id === d.id &&
      r.data.assessment.contextual.rulesReviewed === false;
  }
  row.correct =
    expected &&
    a.status === 200 &&
    a.data.contextual.findings.some(
      (f) => f.ruleId === "lab-metformin-egfr-45",
    ) &&
    exported;
  row.exported = exported;
  if (!row.correct) row.diagnostic = { draft: d, assessment: a };
}
try {
  await Promise.all(
    Array.from({ length: 4 }, async (_, worker) => {
      for (let n = 0; n < (soak ? 7 : 8); n++) {
        await flow("w" + worker + "n" + n, "steady");
        if (n < (soak ? 6 : 7)) await delay(soak ? 20000 : 1500);
      }
    }),
  );
  await Promise.all(
    Array.from({ length: 12 }, (_, n) => flow("burst" + n, "burst")),
  );
  await flow("recovery", "recovery");
} finally {
  polling = false;
  await poll;
}
report.durationMs = Math.round(performance.now() - start);
const okay = report.rows.filter((r) => r.status === 200);
const times = okay.map((r) => r.extractionMs).sort((a, b) => a - b);
report.summary = {
  requests: report.rows.length,
  completed: okay.length,
  correct: okay.filter((r) => r.correct).length,
  busy: report.rows.filter((r) => r.status === 503).length,
  otherFailures: report.rows.filter((r) => ![200, 503].includes(r.status))
    .length,
  providerCalls: okay.reduce((n, r) => n + r.calls, 0),
  medianMs: times[Math.floor(times.length / 2)],
  p95Ms: times[Math.ceil(times.length * 0.95) - 1],
  maxActive: Math.max(...report.health.map((h) => h.data.modelLoad.active)),
  maxHealthMs: Math.max(...report.health.map((h) => h.ms)),
  healthFailures: report.health.filter((h) => h.status !== 200).length,
};
report.passed =
  report.rows
    .filter((r) => r.phase === "steady")
    .every((r) => r.status === 200 && r.correct) &&
  okay.every((r) => r.correct) &&
  report.summary.maxActive <= 4 &&
  report.summary.busy === 8 &&
  report.summary.otherFailures === 0 &&
  report.rows.find((r) => r.phase === "recovery")?.correct === true;
await fs.writeFile(
  process.argv[2] || "validation/live-load-2026-10-24.json",
  JSON.stringify(report, null, 2) + "\n",
);
console.log(
  JSON.stringify({
    passed: report.passed,
    durationMs: report.durationMs,
    ...report.summary,
  }),
);
process.exitCode = report.passed ? 0 : 1;