{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import path_setup  # noqa: F401 — adds project root to sys.path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6042cdd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1b6ab789",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df_ground_truth = pd.read_csv(\"data/ground_truth.csv\")\n",
    "ground_truth = df_ground_truth.to_dict(orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "13697931",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ingest import load_faq_data, build_index, build_vector_index\n",
    "documents = load_faq_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2ba2a75a",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_idx = {}\n",
    "\n",
    "for doc in documents:\n",
    "    doc_idx[doc[\"id\"]] = doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b474a27d",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_index = build_index(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d753534c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating Embeddings in Batches...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28ea3a7725d94aa19553113c94c91b8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/56 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from embedder import Embedder\n",
    "\n",
    "embedder = Embedder()\n",
    "index = build_vector_index(embedder, documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "10501e03",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\nfrom config import make_llm_client, MODEL_NAME\n\nload_dotenv()\n\n# 1. Initialize your client object with a model list\nlitellm_client = make_llm_client()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cfb781ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation.evaluation_utils import RAGWithUsage\n",
    "\n",
    "assistant = RAGWithUsage(\n",
    "    index=index,\n",
    "    llm_client=litellm_client,\n",
    "    text_index=text_index,\n",
    "    vector_index=index,\n",
    "    embedder=embedder,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "606082dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation.evaluation import generate_rag_answer\n",
    "\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from evaluation.evaluation_utils import map_progress_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3646fbe8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'Should I move a person if the scene is unsafe but no immediate threats like fire or traffic?',\n",
       " 'answer_llm': 'No, you should **not** move the person if there is no immediate danger (such as fire, oncoming traffic, or toxic fumes). The best course of action is to stay with them, administer first aid on the spot, and wait for professional medical help to arrive.',\n",
       " 'answer_orig': \"You should only move an injured person if there is immediate danger such as a fire, oncoming traffic, or toxic fumes. Otherwise, it's best to leave them where they are, administer first aid on the spot, and wait for professional medical help to arrive.\",\n",
       " 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer_record = generate_rag_answer(ground_truth[1], doc_idx, assistant)\n",
    "answer_record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d1fac026",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "baf91eed677c4a15a2a2ac6d33f0f5e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with ThreadPoolExecutor(max_workers=6) as pool:\n",
    "    results = map_progress_answers(pool, ground_truth, generate_rag_answer, doc_idx, assistant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "05102cae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13874"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4b4a90b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "answers = []\n",
    "\n",
    "for answer_record in results:\n",
    "    answers.append(answer_record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "58dc6cb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3239971000000008"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "assistant.total_cost()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "445ae805",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_answers = pd.DataFrame(answers)\n",
    "#df_answers.to_csv(\"data/rag-answers.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bea3a55c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df_answers = pd.read_csv(\"data/rag-answers.csv\")\n",
    "answers = df_answers.to_dict(orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "73be079b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pydantic import BaseModel, Field\n",
    "from typing import Literal\n",
    "\n",
    "class AnswerEvaluation(BaseModel):\n",
    "    reasoning: str = Field(\n",
    "        description=\"Reasoning about the quality of the answer.\"\n",
    "    )\n",
    "    score: Literal[\"good\", \"bad\"] = Field(\n",
    "        description=\"'good' if the answer is correct and complete, 'bad' otherwise.\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "90a36025",
   "metadata": {},
   "outputs": [],
   "source": [
    "aqa_judge_instructions = \"\"\"\n",
    "You are an expert evaluator. You will be given:\n",
    "1. A question from a person at accident site\n",
    "2. The original answer from the FAQ (ground truth)\n",
    "3. An answer generated by an AI assistant\n",
    "\n",
    "Your task is to decide if the AI answer is semantically equivalent to\n",
    "the original answer.\n",
    "\n",
    "Rules:\n",
    "- The AI answer does NOT need to be word-for-word identical\n",
    "- It should convey the same key information\n",
    "- Extra detail is fine as long as the core answer is correct\n",
    "- Mark 'bad' only if the AI answer is wrong or misses the key point\n",
    "\n",
    "Be fair and focus on correctness, not style.\n",
    "\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d10a29a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "aqa_judge_prompt = \"\"\"\n",
    "Question:\n",
    "{question}\n",
    "\n",
    "Original Answer (ground truth):\n",
    "{answer_orig}\n",
    "\n",
    "AI Answer:\n",
    "{answer_llm}\n",
    "\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "38eea4a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "rec = answers[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9ce4c4fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AnswerEvaluation(reasoning=\"The AI answer correctly conveys the core principle of the original FAQ: **do not move an injured person if there is no immediate danger** (e.g., fire, traffic, toxic fumes). It retains the key points—leaving them in place, administering first aid on-site, and waiting for professional help—while adding clarity with an example ('no fire, no traffic, no toxic fumes'). The phrasing is slightly more concise but still accurate. The only minor deviation is the inclusion of the phrase *'unless absolutely necessary'* (which is not explicitly stated in the original), but this does not undermine the correctness of the core advice. The AI answer is semantically equivalent and well-structured.\", score='good')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from evaluation.evaluation_utils import calc_price, calc_total_price, llm_structured_retry, map_progress\n",
    "from evaluation.evaluation import evaluate_aqa\n",
    "\n",
    "eval_result, usage = evaluate_aqa(\n",
    "    rec[\"question\"],\n",
    "    rec[\"answer_orig\"],\n",
    "    rec[\"answer_llm\"],\n",
    "    litellm_client,\n",
    "    aqa_judge_instructions,\n",
    "    aqa_judge_prompt,\n",
    "    AnswerEvaluation,\n",
    ")\n",
    "\n",
    "eval_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6537e127",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e91f5472c7864f399bded365f95b7c59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[92m22:49:00 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:49:20 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:49:26 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:50:05 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:50:57 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:54:40 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:55:55 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:56:50 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:57:19 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:57:56 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:58:20 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:58:23 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:58:44 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m22:59:14 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:00:02 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:00:22 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:00:27 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:01:48 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:01:57 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:03:17 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:04:52 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:10:06 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:11:20 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:11:24 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:13:34 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:14:03 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:14:14 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:16:01 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:16:32 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:16:43 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:19:19 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:20:01 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:23:00 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:23:36 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n",
      "\u001b[92m23:23:40 - LiteLLM:WARNING\u001b[0m: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'\n"
     ]
    }
   ],
   "source": [
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from evaluation.evaluation_utils import map_progress_judge\n",
    "from evaluation.evaluation import judge_record\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=6) as pool:\n",
    "    results, usage = map_progress_judge(\n",
    "        pool,\n",
    "        answers,\n",
    "        judge_record,\n",
    "        litellm_client,\n",
    "        aqa_judge_instructions,\n",
    "        aqa_judge_prompt,\n",
    "        AnswerEvaluation,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "ae027c5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluations = []\n",
    "\n",
    "for evaluation in results:\n",
    "    evaluations.append(evaluation)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f76084d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval = pd.DataFrame(evaluations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "330bf178",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13874"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(evaluations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "916a551f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good: 13234/13874 = 95.39%\n"
     ]
    }
   ],
   "source": [
    "good_count = (df_eval[\"score\"] == \"good\").sum()\n",
    "total_count = len(df_eval)\n",
    "print(f\"Good: {good_count}/{total_count} = {good_count/total_count:.2%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "af792ded",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>document</th>\n",
       "      <th>score</th>\n",
       "      <th>reasoning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>What’s the safest height to move them at?</td>\n",
       "      <td>144f2d0d-2a4f-4163-824b-a858b14f6354</td>\n",
       "      <td>bad</td>\n",
       "      <td>The AI answer is incorrect and does not provid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Is the human crutch technique best for people ...</td>\n",
       "      <td>0eb1534a-9340-4571-ba52-f4b190a2bc06</td>\n",
       "      <td>bad</td>\n",
       "      <td>The AI answer incorrectly states uncertainty (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>What’s the first thing to do if someone’s leg ...</td>\n",
       "      <td>757b0832-8ad4-4a72-9660-141cdecafa90</td>\n",
       "      <td>bad</td>\n",
       "      <td>The AI answer expands on the original ground t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Should I wrap my arm around a casualty’s waist...</td>\n",
       "      <td>c3fd2aaf-bb6c-44b4-a434-ee56aca06728</td>\n",
       "      <td>bad</td>\n",
       "      <td>The AI answer is **not semantically equivalent...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>Should you move a casualty immediately when fu...</td>\n",
       "      <td>0eb6039b-d6db-4640-9bf0-6447fec59964</td>\n",
       "      <td>bad</td>\n",
       "      <td>The AI answer introduces a nuanced but incorre...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             question  \\\n",
       "19          What’s the safest height to move them at?   \n",
       "28  Is the human crutch technique best for people ...   \n",
       "38  What’s the first thing to do if someone’s leg ...   \n",
       "42  Should I wrap my arm around a casualty’s waist...   \n",
       "72  Should you move a casualty immediately when fu...   \n",
       "\n",
       "                                document score  \\\n",
       "19  144f2d0d-2a4f-4163-824b-a858b14f6354   bad   \n",
       "28  0eb1534a-9340-4571-ba52-f4b190a2bc06   bad   \n",
       "38  757b0832-8ad4-4a72-9660-141cdecafa90   bad   \n",
       "42  c3fd2aaf-bb6c-44b4-a434-ee56aca06728   bad   \n",
       "72  0eb6039b-d6db-4640-9bf0-6447fec59964   bad   \n",
       "\n",
       "                                            reasoning  \n",
       "19  The AI answer is incorrect and does not provid...  \n",
       "28  The AI answer incorrectly states uncertainty (...  \n",
       "38  The AI answer expands on the original ground t...  \n",
       "42  The AI answer is **not semantically equivalent...  \n",
       "72  The AI answer introduces a nuanced but incorre...  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_eval[df_eval[\"score\"] == \"bad\"].head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "first-aid-qa-assistant-onnx",
   "language": "python",
   "name": "first-aid-qa-assistant-onnx"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}