import fs from "node:fs/promises"; import { createInterface } from "node:readline"; import { spawn } from "node:child_process"; // Optional system TLS transport for hosts where Node's direct egress times out. // Credentials travel through stdin, never argv, files or diagnostic output. async function request(url, options = {}) { if (process.env.CONTEXT_TEST_CURL !== "1") return fetch(url, options); const config = [ "silent", "show-error", "include", "max-time = 50", "connect-timeout = 15", `url = ${JSON.stringify(url)}`, `request = ${JSON.stringify(options.method || "GET")}`, ...Object.entries(options.headers || {}).map( ([key, value]) => `header = ${JSON.stringify(key + ": " + value)}` ), ...(options.body ? [`data = ${JSON.stringify(options.body)}`] : []), ].join("\n"); const output = await new Promise((resolve, reject) => { const child = spawn("curl", ["--config", "-"], { stdio: ["pipe", "pipe", "pipe"], }); let out = ""; child.stdout.on("data", (data) => (out += data)); child.stderr.resume(); child.on("error", () => reject(new Error("Test transport failed"))); child.on("close", (code) => code ? reject(new Error(`Test transport exited ${code}`)) : resolve(out) ); child.stdin.end(config + "\n"); }); const boundary = output.lastIndexOf("\r\n\r\n"); const blocks = output.slice(0, boundary).split("\r\n\r\n"); const lines = blocks.at(-1).split("\r\n"); const status = Number(lines.shift().split(" ")[1]); const headers = new Headers(); for (const line of lines) { const colon = line.indexOf(":"); if (colon > 0) headers.append(line.slice(0, colon), line.slice(colon + 1).trim()); } return { status, ok: status >= 200 && status < 300, headers, json: async () => JSON.parse(output.slice(boundary + 4)), }; } const origin = process.env.CONTEXT_TEST_ORIGIN || "http://127.0.0.1:4191"; if (!["http://127.0.0.1:4191", "https://context.marmar.life"].includes(origin)) throw new Error("Unexpected test destination"); let cookie, otherCookie, cookieFlags; if (origin.startsWith("https:")) { console.log( "Enter existing preview password on stdin; it will not be logged or saved." ); const input = createInterface({ input: process.stdin, terminal: false }); let password = await new Promise((resolve) => input.once("line", resolve)); input.close(); const login = await request(origin + "/api/auth", { method: "POST", headers: { Origin: origin, "Content-Type": "application/json" }, body: JSON.stringify({ password }), }); if (!login.ok) throw new Error(`Preview login failed (${login.status})`); cookie = login.headers.get("set-cookie")?.split(";")[0]; cookieFlags = login.headers.get("set-cookie")?.split(";").slice(1).join(";"); if (!cookie) throw new Error("Preview login returned no session"); if (process.env.CONTEXT_TEST_SECURITY_ONLY === "1") { const other = await request(origin + "/api/auth", { method: "POST", headers: { Origin: origin, "Content-Type": "application/json" }, body: JSON.stringify({ password }), }); if (!other.ok) throw new Error("Second reviewer login failed"); otherCookie = other.headers.get("set-cookie")?.split(";")[0]; } password = undefined; } const corpus = JSON.parse( await fs.readFile( process.argv[3] || new URL("../test/fixtures/messy-flow-corpus.json", import.meta.url), "utf8" ) ); const api = async (path, body) => { const r = await request( origin + "/api/" + path, body ? { method: "POST", headers: { "Content-Type": "application/json", Origin: origin, ...(cookie ? { Cookie: cookie } : {}), }, body: JSON.stringify(body), signal: AbortSignal.timeout(30000), } : { headers: { Origin: origin, ...(cookie ? { Cookie: cookie } : {}) } } ); return { status: r.status, ok: r.ok, data: await r.json() }; }; const status = (await api("status")).data; if (!status.modelConfigured) throw new Error("Connect the Azure model through the interface first."); if (process.env.CONTEXT_TEST_SECURITY_ONLY === "1") { if (!cookie || !otherCookie) throw new Error("Hosted login required"); const checks = []; const check = (name, pass) => checks.push({ name, passed: Boolean(pass) }); check( "Secure HttpOnly host cookie", /Secure/i.test(cookieFlags) && /HttpOnly/i.test(cookieFlags) && /SameSite=Strict/i.test(cookieFlags) && !/Domain=/i.test(cookieFlags) ); check("Separate reviewer identities", cookie !== otherCookie); const cases = (await api("cases")).data; const sources = cases[0].sources; const extracted = await api("extract", { sources, syntheticOnly: true, useModel: false, }); check("Hosted structured extraction stored", extracted.ok); const draft = extracted.data; const payload = { sources, syntheticOnly: true, draftId: draft.id, fingerprint: draft.fingerprint, review: Object.fromEntries(draft.facts.map((f) => [f.id, "include"])), acknowledgeUnparsed: true, }; const assessment = await api("assess", payload); check("Hosted assessment committed", assessment.ok); check( "Owner can retrieve report", (await api("reports/" + assessment.data.reportId)).ok ); const originalCookie = cookie; cookie = otherCookie; check( "Another reviewer cannot read report", (await api("reports/" + assessment.data.reportId)).status === 400 ); check( "Another reviewer cannot assess draft", (await api("assess", payload)).status === 400 ); cookie = originalCookie + "tampered"; check("Tampered session rejected", (await api("status")).status === 401); cookie = originalCookie; check( "Hosted model settings locked", (await api("model", {})).status === 403 ); check( "Non-synthetic request rejected", (await api("extract", { sources, syntheticOnly: false, useModel: false })) .status === 400 ); const evidence = { runAt: new Date().toISOString(), origin, scope: "Hosted synthetic persistence and access checks; no model calls", checks, }; await fs.writeFile( process.argv[2] || "validation/netlify-security.json", JSON.stringify(evidence, null, 2) + "\n" ); console.log(JSON.stringify(evidence)); process.exit(checks.every((x) => x.passed) ? 0 : 1); } const clean = (s) => s.toLowerCase().replace(/\s+/g, "").replaceAll("²", "2"); const match = (f, g) => f.kind === g[0] && clean(f.label) === clean(g[1]) && g .slice(2) .every((v) => clean((f.normalizedLine ?? "") + " " + f.detail).includes(clean(v)) ); const report = { origin, runAt: new Date().toISOString(), scope: "New author-written synthetic challenge cases. Live model, no cache. Not independent clinical validation.", model: status.model, cases: [], }; for (const item of corpus) { // Keep the six-per-minute reviewer quota and shared preview capacity intact. if (cookie && report.cases.length) await new Promise((resolve) => setTimeout(resolve, 12000)); const start = performance.now(); const row = { id: item.id, checks: [], passed: false }; const check = (name, passed, detail) => row.checks.push({ name, passed: Boolean(passed), ...(detail === undefined ? {} : { detail }), }); const sources = item.notes.map((text, i) => ({ id: `${item.id}-${i}`, title: `Synthetic source ${i + 1}`, kind: "Synthetic note", date: "2026-10-23", text, })); row.sources = sources; try { const x = await api("extract", { sources, syntheticOnly: true, useModel: true, freshModel: true, }); row.extractionStatus = x.status; if (!x.ok) throw new Error(x.data.error); const d = x.data; row.draft = d; check( "Fresh live model used", d.mode === "model" && d.strategy === "narrative" && d.metrics?.providerCalls > 0 && !d.metrics?.cacheHit ); const usable = d.facts.filter((f) => !f.problem); const missing = item.required.filter( (g) => !usable.some((f) => match(f, g)) ); const prohibited = (item.withheld ?? []).filter((g) => usable.some((f) => match(f, g)) ); const allowed = [...item.required, ...(item.optional ?? [])]; const unexpected = usable.filter((f) => !allowed.some((g) => match(f, g))); check("Required facts recovered", missing.length === 0, missing); check("Prohibited facts not accepted", prohibited.length === 0, prohibited); check("No unexpected usable facts", unexpected.length === 0, unexpected); check( "Every quote matches original source", d.facts.every( (f) => sources .find((s) => s.id === f.sourceId) ?.text.split(/\r?\n/) [f.line - 1]?.trim() === f.quote.trim() ) ); const review = Object.fromEntries( d.facts.map((f) => [f.id, f.problem ? "exclude" : "include"]) ); const payload = { sources, syntheticOnly: true, draftId: d.id, review, fingerprint: d.fingerprint, acknowledgeUnparsed: true, }; const unreviewed = await api("assess", { ...payload, review: {} }); check("Missing review blocked", !unreviewed.ok); const unacknowledged = await api("assess", { ...payload, acknowledgeUnparsed: false, }); check( "Original-note acknowledgement required", !unacknowledged.ok && /original notes/i.test(unacknowledged.data.error) ); let a = await api("assess", payload); row.initialAssessment = a; if (item.blocked) { check( "Conflicting evidence blocked", !a.ok && a.data.error.includes(item.blocked) ); } if (item.identityProbe) { const identityReview = { ...review }; for (const f of d.facts) if (f.sourceId === sources[1].id && f.kind === "patient") identityReview[f.id] = "exclude"; const probe = await api("assess", { ...payload, review: identityReview }); row.identityProbe = probe; check( "Removing only wrong identity remains blocked", !probe.ok && /different patient/.test(probe.data.error) ); } if (item.exclude) { for (const f of d.facts) if ( item.exclude.some( (e) => f.sourceId === sources[e.source].id && (!e.kind || e.kind === f.kind) && (!e.detailIncludes || f.detail.includes(e.detailIncludes)) ) ) review[f.id] = "exclude"; a = await api("assess", payload); } row.finalAssessment = a; check("Assessment completed after explicit review", a.ok); if (a.ok) { const rules = a.data.contextual.findings.map((f) => f.ruleId); check( "Expected engine result", (!item.rule || rules.includes(item.rule)) && (item.rules || []).every((id) => rules.includes(id)) && (!item.absentRule || !rules.includes(item.absentRule)), rules ); const exported = await api("reports/" + a.data.reportId); row.export = exported; check( "Export preserves reviewed evidence", exported.ok && exported.data.extraction.id === d.id && JSON.stringify(exported.data.sources) === JSON.stringify(sources) && JSON.stringify(exported.data.review) === JSON.stringify(review) && exported.data.assessment.contextual.rulesReviewed === false ); } const stale = await api("assess", { ...payload, sources: sources.map((s, i) => i ? s : { ...s, text: s.text + "\nEdited after review." } ), }); check( "Edited sources invalidate old draft", !stale.ok && /changed/i.test(stale.data.error) ); } catch (e) { row.error = e.message; check("Flow completed", false, e.message); } row.totalMs = Math.round(performance.now() - start); row.passed = row.checks.every((c) => c.passed); report.cases.push(row); console.log( JSON.stringify({ id: row.id, passed: row.passed, ms: row.totalMs, providerCalls: row.draft?.metrics?.providerCalls, failures: row.checks.filter((c) => !c.passed), error: row.error, }) ); } report.passed = report.cases.filter((c) => c.passed).length; report.total = report.cases.length; report.providerCalls = report.cases.reduce( (s, c) => s + (c.draft?.metrics?.providerCalls ?? 0), 0 ); const timings = report.cases .filter((c) => c.draft) .map((c) => c.draft.durationMs) .sort((a, b) => a - b); report.extractionMedianMs = timings[Math.floor(timings.length / 2)]; report.extractionP95Ms = timings[Math.ceil(timings.length * 0.95) - 1]; await fs.writeFile( process.argv[2] || "validation/messy-flow-2026-10-23.json", JSON.stringify(report, null, 2) + "\n" ); console.log( JSON.stringify({ passed: report.passed, total: report.total, providerCalls: report.providerCalls, medianMs: report.extractionMedianMs, p95Ms: report.extractionP95Ms, }) ); process.exitCode = report.passed === report.total ? 0 : 1;