{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import path_setup  # noqa: F401 — adds project root to sys.path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5906d422",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "19c9c1d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df_ground_truth = pd.read_csv(\"data/ground_truth.csv\")\n",
    "ground_truth = df_ground_truth.to_dict(orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e3c28735",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ingest import load_faq_data, build_index, build_vector_index\n",
    "\n",
    "documents = load_faq_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9d0057f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_index = build_index(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a7734953",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating Embeddings in Batches...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea0481f32b484757a63da43fa53f1fb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/56 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from embedder import Embedder\n",
    "\n",
    "embedder = Embedder()\n",
    "index = build_vector_index(embedder, documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "30dfcb65",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation.evaluation import text_search, vector_search, rrf, evaluate\n",
    "\n",
    "def hybrid_search(query,  k=60):\n",
    "    text_results = text_search(query, text_index, num_results=10)\n",
    "    vector_results = vector_search(query, embedder, index, num_results=10)\n",
    "\n",
    "    return rrf([text_results, vector_results], k=k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c8372e37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fb984d9e1c44d8d9cf031207dcd9365",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'hit_rate': 0.6015568689635289, 'mrr': 0.4079116813223703}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(ground_truth, hybrid_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cfb258e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating Hybrid Search k=1 ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ba41ca3859e4de695bae3794be078d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating Hybrid Search k=50 ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72c5aa6a6dca4d9a8e8eb64141f8c28f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating Hybrid Search k=100 ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1723a80a83e845ffbc4d07e9e43204f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating Hybrid Search k=200 ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "408a427ba89f4ab3b2fbdbe9046528bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_tune = []\n",
    "for k in [1, 50, 100, 200]:\n",
    "    print(f\"Evaluating Hybrid Search k={k} ...\")\n",
    "    result = evaluate(\n",
    "        ground_truth,\n",
    "        lambda query, k=k: hybrid_search(\n",
    "            query,\n",
    "            k\n",
    "        )\n",
    "    )\n",
    "\n",
    "    results_tune.append({\n",
    "        \"k\": k,\n",
    "        \"hit_rate\": result[\"hit_rate\"],\n",
    "        \"mrr\": result[\"mrr\"],\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4aa21520",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>hit_rate</th>\n",
       "      <th>mrr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>0.601557</td>\n",
       "      <td>0.407912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100</td>\n",
       "      <td>0.601557</td>\n",
       "      <td>0.407912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>200</td>\n",
       "      <td>0.601557</td>\n",
       "      <td>0.407912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.608765</td>\n",
       "      <td>0.404864</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     k  hit_rate       mrr\n",
       "1   50  0.601557  0.407912\n",
       "2  100  0.601557  0.407912\n",
       "3  200  0.601557  0.407912\n",
       "0    1  0.608765  0.404864"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results = pd.DataFrame(results_tune)\n",
    "df_results.sort_values(\"mrr\", ascending=False).head(4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "first-aid-qa-assistant-onnx",
   "language": "python",
   "name": "first-aid-qa-assistant-onnx"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}