{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import path_setup  # noqa: F401 — adds project root to sys.path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3f54447f",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cde936ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df_ground_truth = pd.read_csv(\"data/ground_truth.csv\")\n",
    "ground_truth = df_ground_truth.to_dict(orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d0711bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ingest import load_faq_data, build_index\n",
    "\n",
    "documents = load_faq_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f53f54d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ground_truth = pd.read_csv(\"data/clinical_eval_set.csv\")\n",
    "clinical_eval_set = df_ground_truth.to_dict(orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "00e48a66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--- Loading and evaluating mode: lsh ---\n",
      "Generating Embeddings in Batches...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38d0156e128845638b9a0c1378446553",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/56 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding Generation Complete! Time taken: 98.93 seconds.\n",
      "Building HNSW Index in SQLite...\n",
      "Database Indexing Complete! Time taken: 1.08 seconds.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6c267e863a44e6c95b7ed72301d8ce4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5526 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--- Loading and evaluating mode: ivf ---\n",
      "Generating Embeddings in Batches...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70e3d70a0f764aa38c01059864db0b88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/56 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding Generation Complete! Time taken: 90.34 seconds.\n",
      "Building HNSW Index in SQLite...\n",
      "Database Indexing Complete! Time taken: 1.47 seconds.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de9f0a6f66e64b44b11c14451417bc1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5526 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--- Loading and evaluating mode: hnsw ---\n",
      "Generating Embeddings in Batches...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bd65c7b33474f1fbb82a301c33dd235",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/56 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding Generation Complete! Time taken: 86.77 seconds.\n",
      "Building HNSW Index in SQLite...\n",
      "Database Indexing Complete! Time taken: 9.84 seconds.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a27ba7f744b48e08dea13b55847d4fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5526 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================================================================\n",
      "                  FIRST AID EMBEDDING MODEL TOURNAMENT MATRIX                  \n",
      "================================================================================\n",
      "Mode Hit Rate @ 1 (Accuracy) Avg Search Latency (ms) SQLite DB Size (KB)\n",
      " lsh                   12.3%                 31.02ms           8120.0 KB\n",
      " ivf                   15.6%                 28.02ms           6244.0 KB\n",
      "hnsw                   15.5%                 27.12ms           6356.0 KB\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "import pandas as pd\n",
    "from sqlitesearch import VectorSearchIndex\n",
    "from embedder import Embedder\n",
    "from tqdm.auto import tqdm\n",
    "import numpy as np\n",
    "\n",
    "# 1. DEFINE THE 4 TOURNAMENT CANDIDATES\n",
    "model_info = {\n",
    "        \"name\": \"all-MiniLM-L6-v2\",\n",
    "        \"path\": \"models/Xenova/all-MiniLM-L6-v2\",\n",
    "        \"dim\": 384,\n",
    "        \"notes\": \"Lightweight Baseline\"\n",
    "    }\n",
    "\n",
    "sqlite_vector_modes = ['lsh', 'ivf', 'hnsw']\n",
    "\n",
    "comparison_results = []\n",
    "\n",
    "# 3. EXECUTE THE BENCHMARK LOOP\n",
    "for mode in sqlite_vector_modes:\n",
    "    print(f\"\\n--- Loading and evaluating mode: {mode} ---\")\n",
    "\n",
    "    # Initialize the model locally via sentence-transformers (easy local alternative for embedding generation)\n",
    "    model = Embedder(model_info[\"path\"])\n",
    "\n",
    "    # Prepare database storage path\n",
    "    db_path = f\"sqlite/vector/evals/embedding-models/{model_info['name']}/eval_db.db\"\n",
    "    if os.path.exists(db_path):\n",
    "        os.remove(db_path)\n",
    "\n",
    "    # Text strings to index\n",
    "    faq_texts = [f\"Q: {item['question']} A: {item['answer']}\" for item in documents]\n",
    "\n",
    "    # Track indexing speed\n",
    "    print(\"Generating Embeddings in Batches...\")\n",
    "    start_gen_time = time.time()\n",
    "\n",
    "    batch_size = 50\n",
    "    db_vectors = []\n",
    "\n",
    "    for i in tqdm(range(0, len(faq_texts), batch_size)):\n",
    "        batch = faq_texts[i:i + batch_size]\n",
    "        batch_vectors = model.encode_batch(batch)\n",
    "        db_vectors.extend(batch_vectors)\n",
    "\n",
    "    db_vectors = np.array(db_vectors)\n",
    "\n",
    "    gen_duration = time.time() - start_gen_time\n",
    "    print(f\"Embedding Generation Complete! Time taken: {gen_duration:.2f} seconds.\")\n",
    "\n",
    "    # Insert vectors into sqlitesearch DB\n",
    "    print(f\"Building {mode} Index in SQLite...\")\n",
    "\n",
    "    start_fit_time = time.time()\n",
    "    vector_index = VectorSearchIndex(mode=mode, db_path=db_path)\n",
    "    vector_index.fit(db_vectors, documents)\n",
    "    fit_duration = time.time() - start_fit_time\n",
    "\n",
    "    print(f\"Database Indexing Complete! Time taken: {fit_duration:.2f} seconds.\")\n",
    "\n",
    "    # Run the query test evaluation\n",
    "    hits_at_1 = 0\n",
    "    total_queries = len(clinical_eval_set)\n",
    "    total_latency_ms = 0\n",
    "\n",
    "    for i in tqdm(range(0, len(clinical_eval_set))):\n",
    "        test = clinical_eval_set[i]\n",
    "        # Measure latency per retrieval\n",
    "        start_query_time = time.time()\n",
    "\n",
    "        # Generate query vector\n",
    "        q_vector = model.encode(test[\"question\"])\n",
    "\n",
    "        # Query local sqlite vector database (limit=1 forces strict top rank scoring)\n",
    "        results = vector_index.search(q_vector, num_results=1)\n",
    "\n",
    "        query_duration = (time.time() - start_query_time) * 1000\n",
    "        total_latency_ms += query_duration\n",
    "\n",
    "        # Check if the top result is the correct clinical match\n",
    "        if results and results[0].get(\"id\") == test[\"document\"]:\n",
    "            hits_at_1 += 1\n",
    "\n",
    "    # Calculate performance matrix stats\n",
    "    accuracy_rate = (hits_at_1 / total_queries) * 100\n",
    "    avg_latency = total_latency_ms / total_queries\n",
    "    db_file_size_kb = os.path.getsize(db_path) / 1024\n",
    "\n",
    "    # Append analytics payloads\n",
    "    comparison_results.append({\n",
    "        \"Mode\": mode,\n",
    "        \"Hit Rate @ 1 (Accuracy)\": f\"{accuracy_rate:.1f}%\",\n",
    "        \"Avg Search Latency (ms)\": f\"{avg_latency:.2f}ms\",\n",
    "        \"SQLite DB Size (KB)\": f\"{db_file_size_kb:.1f} KB\"\n",
    "    })\n",
    "\n",
    "    # Clear the temporary DB file footprint from memory/disk\n",
    "    if os.path.exists(db_path):\n",
    "        vector_index.close()\n",
    "        os.remove(db_path)\n",
    "\n",
    "# 4. PRINT PERFORMANCE MATRIX TABLE\n",
    "df_comparison = pd.DataFrame(comparison_results)\n",
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"                  FIRST AID EMBEDDING MODEL TOURNAMENT MATRIX                  \")\n",
    "print(\"=\"*80)\n",
    "print(df_comparison.to_string(index=False))\n",
    "print(\"=\"*80)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3f019548",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_comparison.to_csv(\"data/sqlite-vector-modes-comparison.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "first-aid-qa-assistant-onnx",
   "language": "python",
   "name": "first-aid-qa-assistant-onnx"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}