# eval/aggregate_eval_run.py import json, argparse from pathlib import Path from collections import defaultdict import csv def load_jsonl(path): with open(path, "r", encoding="utf-8") as f: for line in f: if line.strip(): yield json.loads(line) def safe_mean(xs): xs = [x for x in xs if x is not None] return sum(xs) / len(xs) if xs else None def main(): ap = argparse.ArgumentParser() ap.add_argument("--raw_dir", required=True, help="Path to eval/runs//raw") ap.add_argument("--out_dir", help="Defaults to parent/summary") # NEW: filtering flags ap.add_argument("--include_run_id", nargs="*", help="Only include these run IDs (e.g., 7e30eb 42af10)") ap.add_argument("--exclude_run_id", nargs="*", help="Exclude these run IDs") args = ap.parse_args() raw_dir = Path(args.raw_dir) out_dir = Path(args.out_dir) if args.out_dir else raw_dir.parent / "summary" out_dir.mkdir(parents=True, exist_ok=True) by_scn = defaultdict(lambda: { "latencies": [], "refusals": [], "persona_hits": [], "rag_hits": [], "memory_recall_hits": [], "continuity_breaks": [], "turns": 0, "errors": 0, }) # Collect detailed recall events for the memory summary report memory_recall_events = [] meta = {"eval_id": None, "git_sha": None, "model_ids": set()} def run_id_from_filename(p: Path) -> str: # expects ..._.jsonl stem = p.stem return stem.rsplit("_", 1)[-1] if "_" in stem else "unknown" for p in raw_dir.glob("*.jsonl"): rid = run_id_from_filename(p) # Apply filters if args.include_run_id and rid not in args.include_run_id: continue if args.exclude_run_id and rid in args.exclude_run_id: continue for row in load_jsonl(p): scn = row.get("scenario_id", "unknown") by_scn[scn]["turns"] += 1 by_scn[scn]["latencies"].append(row.get("latency_ms")) m = row.get("metrics", {}) by_scn[scn]["refusals"].append(m.get("refusal")) by_scn[scn]["persona_hits"].append(m.get("persona_hit")) by_scn[scn]["rag_hits"].append(m.get("rag_hit")) # Memory recall metrics (T3.6 Part 2) if m.get("memory_is_recall_turn"): by_scn[scn]["memory_recall_hits"].append(m.get("memory_recall_hit")) memory_recall_events.append({ "scenario_id": scn, "turn": row.get("turn"), "expected": m.get("memory_expected"), "recall_hit": m.get("memory_recall_hit"), "qa_check_type": m.get("memory_qa_check_type"), "user_input": row.get("user_input", "")[:200], "agent_response": row.get("agent_response", "")[:300], "run_id": row.get("run_id"), "repeat": row.get("repeat"), }) if m.get("continuity_break") is not None: by_scn[scn]["continuity_breaks"].append(m.get("continuity_break")) if row.get("status") != "ok": by_scn[scn]["errors"] += 1 if meta["eval_id"] is None: meta["eval_id"] = row.get("eval_id") meta["git_sha"] = row.get("git_sha") if row.get("model_id"): meta["model_ids"].add(row["model_id"]) # write per-scenario CSV csv_path = out_dir / "metrics_by_scenario.csv" with open(csv_path, "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["scenario_id", "turns", "mean_latency_ms", "refusal_rate", "persona_hit_rate", "rag_hit_rate", "memory_recall_rate", "continuity_break_rate", "errors"]) for scn, d in sorted(by_scn.items()): mean_lat = safe_mean(d["latencies"]) refusal_rate = safe_mean(d["refusals"]) persona_rate = safe_mean(d["persona_hits"]) rag_rate = safe_mean(d["rag_hits"]) mem_rate = safe_mean(d["memory_recall_hits"]) cont_rate = safe_mean(d["continuity_breaks"]) w.writerow([scn, d["turns"], round(mean_lat, 1) if mean_lat is not None else None, round(refusal_rate, 3) if refusal_rate is not None else None, round(persona_rate, 3) if persona_rate is not None else None, round(rag_rate, 3) if rag_rate is not None else None, round(mem_rate, 3) if mem_rate is not None else None, round(cont_rate, 3) if cont_rate is not None else None, d["errors"]]) # write overall summary all_lat = [x for d in by_scn.values() for x in d["latencies"] if x is not None] all_ref = [x for d in by_scn.values() for x in d["refusals"] if x is not None] all_per = [x for d in by_scn.values() for x in d["persona_hits"] if x is not None] all_rag = [x for d in by_scn.values() for x in d["rag_hits"] if x is not None] all_mem = [x for d in by_scn.values() for x in d["memory_recall_hits"] if x is not None] all_cont = [x for d in by_scn.values() for x in d["continuity_breaks"] if x is not None] overall = { "eval_id": meta["eval_id"], "git_sha": meta["git_sha"], "model_ids": sorted(meta["model_ids"]), "scenarios": len(by_scn), "turns": sum(d["turns"] for d in by_scn.values()), "mean_latency_ms": round(safe_mean(all_lat), 1) if all_lat else None, "refusal_rate": round(safe_mean(all_ref), 3) if all_ref else None, "persona_hit_rate": round(safe_mean(all_per), 3) if all_per else None, "rag_hit_rate": round(safe_mean(all_rag), 3) if all_rag else None, "memory_recall_rate": round(safe_mean(all_mem), 3) if all_mem else None, "memory_recall_events": len(all_mem), "continuity_break_rate": round(safe_mean(all_cont), 3) if all_cont else None, "total_errors": sum(d["errors"] for d in by_scn.values()), } (out_dir / "overall.json").write_text(json.dumps(overall, indent=2), encoding="utf-8") print(f"✔ Wrote {csv_path}") print(f"✔ Wrote {out_dir / 'overall.json'}") # write memory recall detail report (T3.6 Part 2) if memory_recall_events: mem_csv_path = out_dir / "memory_recall_summary.csv" with open(mem_csv_path, "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["scenario_id", "run_id", "repeat", "turn", "expected", "recall_hit", "qa_check_type", "user_input", "agent_response"]) for evt in memory_recall_events: w.writerow([ evt["scenario_id"], evt["run_id"], evt["repeat"], evt["turn"], evt["expected"], evt["recall_hit"], evt["qa_check_type"], evt["user_input"], evt["agent_response"], ]) print(f"✔ Wrote {mem_csv_path} ({len(memory_recall_events)} recall events)") if __name__ == "__main__": main()