{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import path_setup  # noqa: F401 — adds project root to sys.path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ingest import load_faq_data\n",
    "\n",
    "documents = load_faq_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aeada09a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pydantic import BaseModel\n",
    "\n",
    "class Questions(BaseModel):\n",
    "    questions: list[str]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57504dfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_gen_instructions = \"\"\"\n",
    "    You are an expert medical software tester creating evaluation datasets.\n",
    "    You are given one Questions and Answers Knowledge Base (QA KB).\n",
    "\n",
    "    Generate 2 different realistic search queries a panicked, non-medical user might type into a search bar if they were experiencing this emergency right now.\n",
    "\n",
    "    Rules:\n",
    "    1. Do NOT use the exact medical titles or phrasing from the official question (e.g., avoid using the words 'pit viper' or 'coral snake' if possible).\n",
    "    2. Use descriptive, frantic symptoms, slang, or layperson descriptions instead (e.g., 'a snake with red and yellow bands bit me' or 'got bit by a rattlesnake lookalike').\n",
    "    3. Return your response strictly as a valid JSON list of strings.\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9612443f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\nfrom config import make_llm_client, MODEL_NAME\n\nload_dotenv()\n\n# 1. Initialize your client object with a model list\nlitellm_client = make_llm_client()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f49b0915",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from evaluation.evaluation_utils import map_progress\n",
    "from evaluation.evaluation import generate_ground_truth\n",
    "\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=6) as pool:\n",
    "    results = map_progress(\n",
    "        pool,\n",
    "        documents,\n",
    "        generate_ground_truth,\n",
    "        litellm_client,\n",
    "        data_gen_instructions,\n",
    "        Questions\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e979cd13",
   "metadata": {},
   "outputs": [],
   "source": [
    "clinical_eval_set = []\n",
    "for res in results:\n",
    "    clinical_eval_set.extend(res)\n",
    "\n",
    "len(clinical_eval_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28e691b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "clinical_eval_set[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9045b9c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df_ground_truth = pd.DataFrame(clinical_eval_set)\n",
    "\n",
    "df_ground_truth.to_csv(\"data/clinical_eval_set.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "first-aid-qa-rag-assistant (3.12.7.final.0)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}