import fs from "node:fs/promises";
import { createInterface } from "node:readline";
import { spawn } from "node:child_process";
// Optional system TLS transport for hosts where Node's direct egress times out.
// Credentials travel through stdin, never argv, files or diagnostic output.
async function request(url, options = {}) {
if (process.env.CONTEXT_TEST_CURL !== "1") return fetch(url, options);
const config = [
"silent",
"show-error",
"include",
"max-time = 50",
"connect-timeout = 15",
`url = ${JSON.stringify(url)}`,
`request = ${JSON.stringify(options.method || "GET")}`,
...Object.entries(options.headers || {}).map(
([key, value]) => `header = ${JSON.stringify(key + ": " + value)}`
),
...(options.body ? [`data = ${JSON.stringify(options.body)}`] : []),
].join("\n");
const output = await new Promise((resolve, reject) => {
const child = spawn("curl", ["--config", "-"], {
stdio: ["pipe", "pipe", "pipe"],
});
let out = "";
child.stdout.on("data", (data) => (out += data));
child.stderr.resume();
child.on("error", () => reject(new Error("Test transport failed")));
child.on("close", (code) =>
code ? reject(new Error(`Test transport exited ${code}`)) : resolve(out)
);
child.stdin.end(config + "\n");
});
const boundary = output.lastIndexOf("\r\n\r\n");
const blocks = output.slice(0, boundary).split("\r\n\r\n");
const lines = blocks.at(-1).split("\r\n");
const status = Number(lines.shift().split(" ")[1]);
const headers = new Headers();
for (const line of lines) {
const colon = line.indexOf(":");
if (colon > 0)
headers.append(line.slice(0, colon), line.slice(colon + 1).trim());
}
return {
status,
ok: status >= 200 && status < 300,
headers,
json: async () => JSON.parse(output.slice(boundary + 4)),
};
}
const origin = process.env.CONTEXT_TEST_ORIGIN || "http://127.0.0.1:4191";
if (!["http://127.0.0.1:4191", "https://context.marmar.life"].includes(origin))
throw new Error("Unexpected test destination");
let cookie, otherCookie, cookieFlags;
if (origin.startsWith("https:")) {
console.log(
"Enter existing preview password on stdin; it will not be logged or saved."
);
const input = createInterface({ input: process.stdin, terminal: false });
let password = await new Promise((resolve) => input.once("line", resolve));
input.close();
const login = await request(origin + "/api/auth", {
method: "POST",
headers: { Origin: origin, "Content-Type": "application/json" },
body: JSON.stringify({ password }),
});
if (!login.ok) throw new Error(`Preview login failed (${login.status})`);
cookie = login.headers.get("set-cookie")?.split(";")[0];
cookieFlags = login.headers.get("set-cookie")?.split(";").slice(1).join(";");
if (!cookie) throw new Error("Preview login returned no session");
if (process.env.CONTEXT_TEST_SECURITY_ONLY === "1") {
const other = await request(origin + "/api/auth", {
method: "POST",
headers: { Origin: origin, "Content-Type": "application/json" },
body: JSON.stringify({ password }),
});
if (!other.ok) throw new Error("Second reviewer login failed");
otherCookie = other.headers.get("set-cookie")?.split(";")[0];
}
password = undefined;
}
const corpus = JSON.parse(
await fs.readFile(
process.argv[3] ||
new URL("../test/fixtures/messy-flow-corpus.json", import.meta.url),
"utf8"
)
);
const api = async (path, body) => {
const r = await request(
origin + "/api/" + path,
body
? {
method: "POST",
headers: {
"Content-Type": "application/json",
Origin: origin,
...(cookie ? { Cookie: cookie } : {}),
},
body: JSON.stringify(body),
signal: AbortSignal.timeout(30000),
}
: { headers: { Origin: origin, ...(cookie ? { Cookie: cookie } : {}) } }
);
return { status: r.status, ok: r.ok, data: await r.json() };
};
const status = (await api("status")).data;
if (!status.modelConfigured)
throw new Error("Connect the Azure model through the interface first.");
if (process.env.CONTEXT_TEST_SECURITY_ONLY === "1") {
if (!cookie || !otherCookie) throw new Error("Hosted login required");
const checks = [];
const check = (name, pass) => checks.push({ name, passed: Boolean(pass) });
check(
"Secure HttpOnly host cookie",
/Secure/i.test(cookieFlags) &&
/HttpOnly/i.test(cookieFlags) &&
/SameSite=Strict/i.test(cookieFlags) &&
!/Domain=/i.test(cookieFlags)
);
check("Separate reviewer identities", cookie !== otherCookie);
const cases = (await api("cases")).data;
const sources = cases[0].sources;
const extracted = await api("extract", {
sources,
syntheticOnly: true,
useModel: false,
});
check("Hosted structured extraction stored", extracted.ok);
const draft = extracted.data;
const payload = {
sources,
syntheticOnly: true,
draftId: draft.id,
fingerprint: draft.fingerprint,
review: Object.fromEntries(draft.facts.map((f) => [f.id, "include"])),
acknowledgeUnparsed: true,
};
const assessment = await api("assess", payload);
check("Hosted assessment committed", assessment.ok);
check(
"Owner can retrieve report",
(await api("reports/" + assessment.data.reportId)).ok
);
const originalCookie = cookie;
cookie = otherCookie;
check(
"Another reviewer cannot read report",
(await api("reports/" + assessment.data.reportId)).status === 400
);
check(
"Another reviewer cannot assess draft",
(await api("assess", payload)).status === 400
);
cookie = originalCookie + "tampered";
check("Tampered session rejected", (await api("status")).status === 401);
cookie = originalCookie;
check(
"Hosted model settings locked",
(await api("model", {})).status === 403
);
check(
"Non-synthetic request rejected",
(await api("extract", { sources, syntheticOnly: false, useModel: false }))
.status === 400
);
const evidence = {
runAt: new Date().toISOString(),
origin,
scope: "Hosted synthetic persistence and access checks; no model calls",
checks,
};
await fs.writeFile(
process.argv[2] || "validation/netlify-security.json",
JSON.stringify(evidence, null, 2) + "\n"
);
console.log(JSON.stringify(evidence));
process.exit(checks.every((x) => x.passed) ? 0 : 1);
}
const clean = (s) => s.toLowerCase().replace(/\s+/g, "").replaceAll("²", "2");
const match = (f, g) =>
f.kind === g[0] &&
clean(f.label) === clean(g[1]) &&
g
.slice(2)
.every((v) =>
clean((f.normalizedLine ?? "") + " " + f.detail).includes(clean(v))
);
const report = {
origin,
runAt: new Date().toISOString(),
scope:
"New author-written synthetic challenge cases. Live model, no cache. Not independent clinical validation.",
model: status.model,
cases: [],
};
for (const item of corpus) {
// Keep the six-per-minute reviewer quota and shared preview capacity intact.
if (cookie && report.cases.length)
await new Promise((resolve) => setTimeout(resolve, 12000));
const start = performance.now();
const row = { id: item.id, checks: [], passed: false };
const check = (name, passed, detail) =>
row.checks.push({
name,
passed: Boolean(passed),
...(detail === undefined ? {} : { detail }),
});
const sources = item.notes.map((text, i) => ({
id: `${item.id}-${i}`,
title: `Synthetic source ${i + 1}`,
kind: "Synthetic note",
date: "2026-10-23",
text,
}));
row.sources = sources;
try {
const x = await api("extract", {
sources,
syntheticOnly: true,
useModel: true,
freshModel: true,
});
row.extractionStatus = x.status;
if (!x.ok) throw new Error(x.data.error);
const d = x.data;
row.draft = d;
check(
"Fresh live model used",
d.mode === "model" &&
d.strategy === "narrative" &&
d.metrics?.providerCalls > 0 &&
!d.metrics?.cacheHit
);
const usable = d.facts.filter((f) => !f.problem);
const missing = item.required.filter(
(g) => !usable.some((f) => match(f, g))
);
const prohibited = (item.withheld ?? []).filter((g) =>
usable.some((f) => match(f, g))
);
const allowed = [...item.required, ...(item.optional ?? [])];
const unexpected = usable.filter((f) => !allowed.some((g) => match(f, g)));
check("Required facts recovered", missing.length === 0, missing);
check("Prohibited facts not accepted", prohibited.length === 0, prohibited);
check("No unexpected usable facts", unexpected.length === 0, unexpected);
check(
"Every quote matches original source",
d.facts.every(
(f) =>
sources
.find((s) => s.id === f.sourceId)
?.text.split(/\r?\n/)
[f.line - 1]?.trim() === f.quote.trim()
)
);
const review = Object.fromEntries(
d.facts.map((f) => [f.id, f.problem ? "exclude" : "include"])
);
const payload = {
sources,
syntheticOnly: true,
draftId: d.id,
review,
fingerprint: d.fingerprint,
acknowledgeUnparsed: true,
};
const unreviewed = await api("assess", { ...payload, review: {} });
check("Missing review blocked", !unreviewed.ok);
const unacknowledged = await api("assess", {
...payload,
acknowledgeUnparsed: false,
});
check(
"Original-note acknowledgement required",
!unacknowledged.ok && /original notes/i.test(unacknowledged.data.error)
);
let a = await api("assess", payload);
row.initialAssessment = a;
if (item.blocked) {
check(
"Conflicting evidence blocked",
!a.ok && a.data.error.includes(item.blocked)
);
}
if (item.identityProbe) {
const identityReview = { ...review };
for (const f of d.facts)
if (f.sourceId === sources[1].id && f.kind === "patient")
identityReview[f.id] = "exclude";
const probe = await api("assess", { ...payload, review: identityReview });
row.identityProbe = probe;
check(
"Removing only wrong identity remains blocked",
!probe.ok && /different patient/.test(probe.data.error)
);
}
if (item.exclude) {
for (const f of d.facts)
if (
item.exclude.some(
(e) =>
f.sourceId === sources[e.source].id &&
(!e.kind || e.kind === f.kind) &&
(!e.detailIncludes || f.detail.includes(e.detailIncludes))
)
)
review[f.id] = "exclude";
a = await api("assess", payload);
}
row.finalAssessment = a;
check("Assessment completed after explicit review", a.ok);
if (a.ok) {
const rules = a.data.contextual.findings.map((f) => f.ruleId);
check(
"Expected engine result",
(!item.rule || rules.includes(item.rule)) &&
(item.rules || []).every((id) => rules.includes(id)) &&
(!item.absentRule || !rules.includes(item.absentRule)),
rules
);
const exported = await api("reports/" + a.data.reportId);
row.export = exported;
check(
"Export preserves reviewed evidence",
exported.ok &&
exported.data.extraction.id === d.id &&
JSON.stringify(exported.data.sources) === JSON.stringify(sources) &&
JSON.stringify(exported.data.review) === JSON.stringify(review) &&
exported.data.assessment.contextual.rulesReviewed === false
);
}
const stale = await api("assess", {
...payload,
sources: sources.map((s, i) =>
i ? s : { ...s, text: s.text + "\nEdited after review." }
),
});
check(
"Edited sources invalidate old draft",
!stale.ok && /changed/i.test(stale.data.error)
);
} catch (e) {
row.error = e.message;
check("Flow completed", false, e.message);
}
row.totalMs = Math.round(performance.now() - start);
row.passed = row.checks.every((c) => c.passed);
report.cases.push(row);
console.log(
JSON.stringify({
id: row.id,
passed: row.passed,
ms: row.totalMs,
providerCalls: row.draft?.metrics?.providerCalls,
failures: row.checks.filter((c) => !c.passed),
error: row.error,
})
);
}
report.passed = report.cases.filter((c) => c.passed).length;
report.total = report.cases.length;
report.providerCalls = report.cases.reduce(
(s, c) => s + (c.draft?.metrics?.providerCalls ?? 0),
0
);
const timings = report.cases
.filter((c) => c.draft)
.map((c) => c.draft.durationMs)
.sort((a, b) => a - b);
report.extractionMedianMs = timings[Math.floor(timings.length / 2)];
report.extractionP95Ms = timings[Math.ceil(timings.length * 0.95) - 1];
await fs.writeFile(
process.argv[2] || "validation/messy-flow-2026-10-23.json",
JSON.stringify(report, null, 2) + "\n"
);
console.log(
JSON.stringify({
passed: report.passed,
total: report.total,
providerCalls: report.providerCalls,
medianMs: report.extractionMedianMs,
p95Ms: report.extractionP95Ms,
})
);
process.exitCode = report.passed === report.total ? 0 : 1;