marmar-context / web / context / test / model.test.ts
model.test.ts
Raw
import { afterEach, describe, expect, it, vi } from "vitest";
import { extract, configureAzureSession } from "../server/model";
import { CASES } from "../server/cases";
const sources = [
  {
    id: "synthetic",
    title: "Test note",
    kind: "Synthetic",
    date: "2026-10-23",
    text: "Patient: Amina Stone\nShe takes Metformin 500 mg twice daily.",
  },
];
const valid = {
  facts: [[2, "Medication: Metformin; strength: 500 mg; frequency: 2/day"]],
};
const reply = (content: unknown, finish = "stop") => ({
  ok: true,
  status: 200,
  headers: new Headers(),
  json: async () => ({
    choices: [{ finish_reason: finish, message: { content } }],
    usage: { prompt_tokens: 100, completion_tokens: 30 },
  }),
});
const configured = () =>
  configureAzureSession({
    endpoint:
      "https://synthetic.services.ai.azure.com/openai/v1/chat/completions",
    key: "synthetic-test-key",
    model: "test-Llama",
    auth: "api-key",
  });
afterEach(() => {
  vi.unstubAllGlobals();
  vi.useRealTimers();
});
describe("compact provider contract", () => {
  it("makes no provider call for complete labelled records", async () => {
    configured();
    const fetcher = vi.fn();
    vi.stubGlobal("fetch", fetcher);
    const d = await extract(CASES[0].sources, true, true);
    expect(d.strategy).toBe("structured");
    expect(d.mode).toBe("local");
    expect(d.metrics?.providerCalls).toBe(0);
    expect(fetcher).not.toHaveBeenCalled();
  });
  it("sends only unparsed lines, reattaches original provenance and preserves structured fields", async () => {
    configured();
    const fetcher = vi.fn().mockResolvedValue(reply(JSON.stringify(valid)));
    vi.stubGlobal("fetch", fetcher);
    const d = await extract(sources, true, true);
    const request = JSON.parse(fetcher.mock.calls[0][1].body);
    expect(JSON.parse(request.messages[1].content)).toEqual([
      [2, sources[0].text.split("\n")[1]],
    ]);
    expect(d.facts).toHaveLength(2);
    expect(d.facts[1].quote).toBe(sources[0].text.split("\n")[1]);
    expect(d.metrics?.outputTokens).toBe(30);
    expect(fetcher.mock.calls[0][1].headers["api-key"]).toBe(
      "synthetic-test-key",
    );
  });
  it("retries malformed JSON once and records calls and total token usage", async () => {
    configured();
    const fetcher = vi
      .fn()
      .mockResolvedValueOnce(reply("not JSON"))
      .mockResolvedValueOnce(reply(JSON.stringify(valid)));
    vi.stubGlobal("fetch", fetcher);
    const d = await extract(sources, true, true);
    expect(d.attempts).toBe(2);
    expect(d.metrics?.providerCalls).toBe(2);
    expect(d.metrics?.outputTokens).toBe(60);
  });
  it("does not report partial token totals after a network failure", async () => {
    configured();
    const fetcher = vi
      .fn()
      .mockRejectedValueOnce(new Error("network"))
      .mockResolvedValueOnce(reply(JSON.stringify(valid)));
    vi.stubGlobal("fetch", fetcher);
    const d = await extract(sources, true, true);
    expect(d.metrics?.providerCalls).toBe(2);
    expect(d.metrics?.inputTokens).toBeNull();
    expect(d.metrics?.outputTokens).toBeNull();
  });
  it("stops after two malformed replies", async () => {
    configured();
    const fetcher = vi.fn().mockResolvedValue(reply("not JSON"));
    vi.stubGlobal("fetch", fetcher);
    await expect(extract(sources, true, true)).rejects.toThrow("both attempts");
    expect(fetcher).toHaveBeenCalledTimes(2);
  });
  it("retries a schema mismatch without accepting partial values", async () => {
    configured();
    const fetcher = vi
      .fn()
      .mockResolvedValueOnce(reply('{"facts":[{"bad":"shape"}]}'))
      .mockResolvedValueOnce(reply(JSON.stringify(valid)));
    vi.stubGlobal("fetch", fetcher);
    expect((await extract(sources, true, true)).attempts).toBe(2);
  });
  it.each([
    ["content_filter", "blocked"],
    ["length", "output limit"],
  ])("never retries %s responses", async (finish, message) => {
    configured();
    const fetcher = vi.fn().mockResolvedValue(reply('{"facts":[]}', finish));
    vi.stubGlobal("fetch", fetcher);
    await expect(extract(sources, true, true)).rejects.toThrow(message);
    expect(fetcher).toHaveBeenCalledTimes(1);
  });
  it("rejects a non-string provider response", async () => {
    configured();
    vi.stubGlobal(
      "fetch",
      vi.fn().mockResolvedValue(reply({ unexpected: "object" })),
    );
    await expect(extract(sources, true, true)).rejects.toThrow("no text");
  });
  it("retries transient network failure once without returning provider secrets", async () => {
    configured();
    const fetcher = vi.fn().mockRejectedValue(new Error("secret response"));
    vi.stubGlobal("fetch", fetcher);
    await expect(extract(sources, true, true)).rejects.toThrow("bounded retry");
    expect(fetcher).toHaveBeenCalledTimes(2);
  });
  it("retries a transient HTTP failure once", async () => {
    configured();
    const fetcher = vi
      .fn()
      .mockResolvedValueOnce({ ok: false, status: 503, headers: new Headers() })
      .mockResolvedValueOnce(reply(JSON.stringify(valid)));
    vi.stubGlobal("fetch", fetcher);
    const d = await extract(sources, true, true);
    expect(d.metrics?.providerCalls).toBe(2);
    expect(d.metrics?.inputTokens).toBeNull();
  });
  it.each([401, 403])(
    "never retries authorization failure %s",
    async (status) => {
      configured();
      const fetcher = vi
        .fn()
        .mockResolvedValue({ ok: false, status, headers: new Headers() });
      vi.stubGlobal("fetch", fetcher);
      await expect(extract(sources, true, true)).rejects.toThrow("rejected");
      expect(fetcher).toHaveBeenCalledTimes(1);
    },
  );
  it("honors rate limiting beyond the retry budget", async () => {
    configured();
    const fetcher = vi
      .fn()
      .mockResolvedValue({
        ok: false,
        status: 429,
        headers: new Headers({ "Retry-After": "10" }),
      });
    vi.stubGlobal("fetch", fetcher);
    await expect(extract(sources, true, true)).rejects.toThrow("busy");
    expect(fetcher).toHaveBeenCalledTimes(1);
  });
  it("bounds retries when transport consumes the total time budget", async () => {
    configured();
    const fetcher = vi.fn().mockImplementation(async () => {
      vi.spyOn(performance, "now").mockReturnValue(99999999);
      throw new Error("timeout");
    });
    vi.stubGlobal("fetch", fetcher);
    try {
      await expect(extract(sources, true, true)).rejects.toThrow("time budget");
      expect(fetcher).toHaveBeenCalledTimes(1);
    } finally {
      vi.restoreAllMocks();
    }
  });
  it("does not retry invalid source references", async () => {
    configured();
    const fetcher = vi
      .fn()
      .mockResolvedValue(reply('{"facts":[[99,"Age: 62"]]}'));
    vi.stubGlobal("fetch", fetcher);
    await expect(extract(sources, true, true)).rejects.toThrow(
      "unknown source",
    );
    expect(fetcher).toHaveBeenCalledTimes(1);
  });
  it("reuses a validated draft only for unchanged sources and records a cache hit", async () => {
    configured();
    const fetcher = vi.fn().mockResolvedValue(reply(JSON.stringify(valid)));
    vi.stubGlobal("fetch", fetcher);
    const first = await extract(sources, true);
    const second = await extract(sources, true);
    expect(first.mode).toBe("model");
    expect(second.strategy).toBe("cached");
    expect(second.metrics?.providerCalls).toBe(0);
    expect(fetcher).toHaveBeenCalledTimes(1);
    second.facts[0].label = "Mutated";
    expect((await extract(sources, true)).facts[0].label).toBe("Amina Stone");
    await extract(sources, true, true);
    expect(fetcher).toHaveBeenCalledTimes(2);
  });
  it("invalidates cached results after changing sources or reconnecting", async () => {
    configured();
    const fetcher = vi.fn().mockResolvedValue(reply(JSON.stringify(valid)));
    vi.stubGlobal("fetch", fetcher);
    await extract(sources, true);
    await extract(
      sources.map((s) => ({ ...s, title: "Revised title" })),
      true,
    );
    configured();
    await extract(sources, true);
    expect(fetcher).toHaveBeenCalledTimes(3);
  });
  it("expires cached results after three minutes", async () => {
    configured();
    vi.useFakeTimers();
    const fetcher = vi.fn().mockResolvedValue(reply(JSON.stringify(valid)));
    vi.stubGlobal("fetch", fetcher);
    await extract(sources, true);
    vi.setSystemTime(Date.now() + 181000);
    await extract(sources, true);
    expect(fetcher).toHaveBeenCalledTimes(2);
  });
});