marmar-context / web / context / scripts / fault-evaluation.mjs
fault-evaluation.mjs
Raw
import fs from "node:fs/promises";
import { setTimeout as delay } from "node:timers/promises";
const modeFile =
  process.env.CONTEXT_FAULT_FILE || "/tmp/marmar-context-fault-mode.json";
const base = "http://127.0.0.1:4198/api/";
const request = async (path, body, signal) => {
  const t = performance.now();
  const r = await fetch(base + path, {
    ...(body
      ? {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify(body),
        }
      : {}),
    signal: signal ?? AbortSignal.timeout(28000),
  });
  return {
    status: r.status,
    ms: Math.round(performance.now() - t),
    retryAfter: r.headers.get("retry-after"),
    data: await r.json(),
  };
};
const payload = (id = "fault") => ({
  sources: [
    {
      id,
      title: "Synthetic fault test",
      kind: "Synthetic",
      date: "2026-10-23",
      text: "Patient: Fault Test\nShe takes Metformin 500 mg twice daily.",
    },
  ],
  useModel: true,
  freshModel: true,
  syntheticOnly: true,
});
const setMode = async (kind, ms) => {
  await fs.writeFile(
    modeFile,
    JSON.stringify({ kind, ms, epoch: crypto.randomUUID() })
  );
  await request("model", {
    endpoint:
      "https://synthetic.services.ai.azure.com/openai/v1/chat/completions",
    key: "synthetic-not-a-credential",
    model: "Fault-test-provider",
    auth: "api-key",
  });
};
const stats = async () =>
  JSON.parse(await fs.readFile(modeFile + ".stats", "utf8"));
const report = {
  runAt: new Date().toISOString(),
  scope: "Isolated simulated-provider HTTP failure tests. No Azure calls.",
  checks: [],
};
const check = (name, passed, detail) => {
  report.checks.push({ name, passed: !!passed, detail });
  console.log(JSON.stringify(report.checks.at(-1)));
};
for (const [kind, calls] of [
  ["network", 2],
  ["503", 2],
  ["401", 1],
  ["malformed", 2],
  ["truncated", 1],
  ["null", 1],
]) {
  await setMode(kind);
  const r = await request("extract", payload());
  const s = await stats();
  check(
    kind + " safely fails without draft or leaked details",
    r.status === 502 &&
      !r.data.id &&
      !r.data.facts &&
      !JSON.stringify(r.data).includes("PRIVATE") &&
      s.calls === calls &&
      s.active === 0,
    { ...r, provider: s }
  );
}
for (const kind of ["recover", "rate-short"]) {
  await setMode(kind);
  const r = await request("extract", payload());
  check(kind + " recovers once", r.status === 200 && r.data.attempts === 2, {
    status: r.status,
    ms: r.ms,
    provider: await stats(),
  });
}
await setMode("rate-date");
const rate = await request("extract", payload());
const again = await request("extract", payload());
check(
  "HTTP-date cooldown suppresses repeated provider calls",
  rate.status === 503 &&
    Number(rate.retryAfter) > 20 &&
    again.status === 503 &&
    (await stats()).calls === 1,
  { rate, again }
);
await setMode("slow", 1500);
const burst = await Promise.all(
  Array.from({ length: 12 }, (_, i) =>
    request("extract", payload("burst-" + i))
  )
);
const load = await stats();
check(
  "12-request burst bounded at four model slots",
  burst.filter((r) => r.status === 200).length === 4 &&
    burst.filter((r) => r.status === 503).length === 8 &&
    load.peak === 4 &&
    load.active === 0,
  {
    statuses: burst.map((r) => r.status),
    maxRejectedMs: Math.max(
      ...burst.filter((r) => r.status === 503).map((r) => r.ms)
    ),
    provider: load,
  }
);
await setMode("hang");
const controller = new AbortController();
const pending = request("extract", payload(), controller.signal).catch((e) => ({
  cancelled: e.name === "AbortError",
}));
await delay(250);
controller.abort();
await pending;
await delay(200);
const cancelled = await stats();
check(
  "client disconnect cancels provider work",
  cancelled.aborted === 1 && cancelled.active === 0,
  cancelled
);
for (const kind of ["hang", "body-hang"]) {
  await setMode(kind);
  const pending = request("extract", payload());
  await delay(500);
  const health = await request("status");
  const local = await request("extract", {
    ...payload("structured"),
    useModel: false,
  });
  check(
    kind + " keeps status and local reader responsive",
    health.status === 200 &&
      health.ms < 500 &&
      local.status === 200 &&
      local.ms < 500,
    { healthMs: health.ms, localMs: local.ms }
  );
  const r = await pending;
  check(
    kind + " obeys overall deadline",
    r.status === 502 && r.ms >= 20000 && r.ms < 24000 && !r.data.id,
    { ...r, provider: await stats() }
  );
  await setMode("good");
  const recovered = await request("extract", payload());
  check(
    kind + " releases capacity for recovery",
    recovered.status === 200 && (await stats()).active === 0,
    { status: recovered.status, ms: recovered.ms }
  );
}
report.passed = report.checks.filter((c) => c.passed).length;
report.total = report.checks.length;
await fs.writeFile(
  process.argv[2] || "validation/fault-load-2026-10-24.json",
  JSON.stringify(report, null, 2) + "\n"
);
await setMode("slow", 8000); // Keep isolated test UI available for manual cancellation checks.
process.exitCode = report.passed === report.total ? 0 : 1;