{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import path_setup  # noqa: F401 — adds project root to sys.path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e71f84d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0f65f2f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df_ground_truth = pd.read_csv(\"data/ground_truth.csv\")\n",
    "ground_truth = df_ground_truth.to_dict(orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f861eb4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ingest import load_faq_data, build_index\n",
    "\n",
    "documents = load_faq_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "70f8fdb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2775"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e695b878",
   "metadata": {},
   "outputs": [],
   "source": [
    "index = build_index(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c3765cfb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=0.5, answer_boost=1.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "059366cef9d945beaa734bad1a1b3845",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=0.5, answer_boost=2.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8926f60dc3a542deb1a5d0af86935ffc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=0.5, answer_boost=4.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da47760219754857958ea9ba806d0d27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=0.5, answer_boost=10.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8b4e99cbb2b40fd88bace1d5c10a32f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=1.0, answer_boost=1.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f9a76032ac4480e9d3ee0f762359ea7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=1.0, answer_boost=2.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e4e090cf4ee41f092cd670756e5541b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=1.0, answer_boost=4.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0f41644dc434c618bd42113e258b3c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=1.0, answer_boost=10.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac711218ba8e4f9081619bf96218aaf3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=1.5, answer_boost=1.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50896ffa76ba48cdabde68f9053d1573",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=1.5, answer_boost=2.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9965fd8dc926411581afa2a56cb75baa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=1.5, answer_boost=4.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba4cf4e6db9341db980265ea892c2c1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating question_boost=1.5, answer_boost=10.0,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0be689445f9c4bb6bc16bfcc1bd5725d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from evaluation.evaluation import evaluate, search_boosts\n",
    "\n",
    "results = []\n",
    "\n",
    "for question_boost in [0.5, 1.0, 1.5]:\n",
    "    for answer_boost in [1.0, 2.0, 4.0, 10.0]:\n",
    "            print(\n",
    "                f\"Evaluating question_boost={question_boost},\"\n",
    "                f\" answer_boost={answer_boost},\"\n",
    "            )\n",
    "            result = evaluate(\n",
    "                ground_truth,\n",
    "                lambda query, question_boost=question_boost, answer_boost=answer_boost: search_boosts(\n",
    "                    query,\n",
    "                    question_boost,\n",
    "                    answer_boost,\n",
    "                    index\n",
    "                )\n",
    "            )\n",
    "\n",
    "            results.append({\n",
    "                \"question\": question_boost,\n",
    "                \"answer\": answer_boost,\n",
    "                \"hit_rate\": result[\"hit_rate\"],\n",
    "                \"mrr\": result[\"mrr\"],\n",
    "            })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bac39327",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>hit_rate</th>\n",
       "      <th>mrr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.5</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.427274</td>\n",
       "      <td>0.28798</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   question  answer  hit_rate      mrr\n",
       "0       0.5     1.0  0.427274  0.28798\n",
       "1       0.5     2.0  0.427274  0.28798\n",
       "2       0.5     4.0  0.427274  0.28798\n",
       "3       0.5    10.0  0.427274  0.28798\n",
       "4       1.0     1.0  0.427274  0.28798\n",
       "5       1.0     2.0  0.427274  0.28798\n",
       "6       1.0     4.0  0.427274  0.28798\n",
       "7       1.0    10.0  0.427274  0.28798\n",
       "8       1.5     1.0  0.427274  0.28798\n",
       "9       1.5     2.0  0.427274  0.28798"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results = pd.DataFrame(results)\n",
    "df_results.sort_values(\"mrr\", ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "915fa33d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63a37b6c754b4a1a844ed8973549db6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'hit_rate': 0.4272740377684878, 'mrr': 0.2879799144683107}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from evaluation.evaluation import text_search\n",
    "\n",
    "evaluate(\n",
    "    ground_truth,\n",
    "    lambda query='', index=index: text_search(query, index)\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "first-aid-qa-rag-assistant (3.12.7)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}