marmar-context / web / context / scripts / api-evaluation.mjs
api-evaluation.mjs
Raw
import fs from "node:fs/promises";
const origin = "http://127.0.0.1:4191";
async function req(path, body, headers = {}) {
  const r = await fetch(origin + "/api/" + path, {
    method: body === undefined ? "GET" : "POST",
    headers: {
      ...(body === undefined ? {} : { "Content-Type": "application/json" }),
      ...headers,
    },
    body:
      body === undefined
        ? undefined
        : typeof body === "string"
          ? body
          : JSON.stringify(body),
  });
  return { status: r.status, body: await r.json() };
}
const cases = (await req("cases")).body;
const sources = cases[0].sources;
const results = [];
async function check(name, fn) {
  try {
    results.push({ name, passed: Boolean(await fn()) });
  } catch (e) {
    results.push({ name, passed: false, error: e.message });
  }
}
await check("Status exposes no credential fields", async () => {
  const r = await req("status");
  return (
    r.status === 200 &&
    !/"(?:key|apiKey|api_key|endpoint|authorization)"/i.test(
      JSON.stringify(r.body),
    )
  );
});
await check(
  "Unrelated web origins cannot call local APIs",
  async () =>
    (
      await req(
        "extract",
        { sources, syntheticOnly: true },
        { Origin: "https://unrelated.example" },
      )
    ).status === 403,
);
await check(
  "Real-data acknowledgement rejected",
  async () =>
    (await req("extract", { sources, syntheticOnly: false })).status === 400,
);
await check(
  "Non-JSON bodies rejected",
  async () =>
    (await req("extract", "plain text", { "Content-Type": "text/plain" }))
      .status === 415,
);
await check(
  "Malformed JSON rejected",
  async () => (await req("extract", "{")).status === 400,
);
await check(
  "Duplicate source IDs rejected",
  async () =>
    (
      await req("extract", {
        sources: [sources[0], sources[0]],
        syntheticOnly: true,
      })
    ).status === 400,
);
await check(
  "Too many source records rejected",
  async () =>
    (
      await req("extract", {
        sources: Array.from({ length: 9 }, (_, i) => ({
          ...sources[0],
          id: "record-" + i,
        })),
        syntheticOnly: true,
      })
    ).status === 400,
);
const draft = (
  await req("extract", { sources, syntheticOnly: true, useModel: false })
).body;
const review = Object.fromEntries(draft.facts.map((f) => [f.id, "include"]));
const payload = {
  sources,
  syntheticOnly: true,
  draftId: draft.id,
  review,
  fingerprint: draft.fingerprint,
  acknowledgeUnparsed: false,
};
await check(
  "Unknown draft IDs rejected",
  async () =>
    (
      await req("assess", {
        ...payload,
        draftId: "00000000-0000-4000-8000-000000000000",
      })
    ).status === 400,
);
await check(
  "Unreviewed facts rejected",
  async () => (await req("assess", { ...payload, review: {} })).status === 400,
);
await check(
  "Client-injected facts rejected",
  async () =>
    (
      await req("assess", {
        ...payload,
        review: { ...review, invented: "include" },
      })
    ).status === 400,
);
await check(
  "Source edits invalidate assessment",
  async () =>
    (
      await req("assess", {
        ...payload,
        sources: sources.map((s) => ({ ...s, text: s.text + "\nAge: 99" })),
      })
    ).status === 400,
);
const assessment = await req("assess", payload);
await check(
  "Local assessment remains explicitly local and unsigned",
  async () =>
    assessment.status === 200 &&
    assessment.body.extraction.mode === "local" &&
    assessment.body.contextual.rulesReviewed === false,
);
await check("Export is generated from the stored reviewed draft", async () => {
  const r = await req("reports/" + assessment.body.reportId);
  return (
    r.status === 200 &&
    r.body.extraction.id === draft.id &&
    r.body.assessment.reportId === assessment.body.reportId &&
    JSON.stringify(r.body.review) === JSON.stringify(review)
  );
});
const report = {
  runAt: new Date().toISOString(),
  scope: "Local API boundary and evidence-integrity tests",
  passed: results.filter((r) => r.passed).length,
  total: results.length,
  tests: results,
};
await fs.writeFile(
  process.argv[2] || "validation/api-2026-10-23.json",
  JSON.stringify(report, null, 2) + "\n",
);
console.log(JSON.stringify(report, null, 2));
process.exitCode = report.passed === report.total ? 0 : 1;