{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import path_setup # noqa: F401 — adds project root to sys.path\n" ] }, { "cell_type": "code", "execution_count": 1, "id": "5c2c49fd", "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "id": "1d881c8a", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "df_ground_truth = pd.read_csv(\"data/ground_truth.csv\")\n", "ground_truth = df_ground_truth.to_dict(orient=\"records\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "ac3f4db2", "metadata": {}, "outputs": [], "source": [ "from ingest import load_faq_data\n", "\n", "documents = load_faq_data()" ] }, { "cell_type": "code", "execution_count": 5, "id": "c82e098b", "metadata": {}, "outputs": [], "source": [ "df_ground_truth = pd.read_csv(\"data/clinical_eval_set.csv\")\n", "clinical_eval_set = df_ground_truth.to_dict(orient=\"records\")" ] }, { "cell_type": "code", "execution_count": null, "id": "74297d35", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Loading and evaluating model: all-MiniLM-L6-v2 ---\n", "Generating Embeddings in Batches...\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f64e39992dbe433a954bb288e1381f6e", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/56 [00:00 \u001b[39m\u001b[32m44\u001b[39m model = Embedder(model_info[\u001b[33m\"path\"\u001b[39m])\n\u001b[32m 45\u001b[39m \n\u001b[32m 46\u001b[39m \u001b[38;5;66;03m# Prepare database storage path\u001b[39;00m\n\u001b[32m 47\u001b[39m db_path = f\"sqlite/vector/evals/embedding-models/{model_info[\u001b[33m'name'\u001b[39m]}/eval_db.db\"\n", "\u001b[36mFile \u001b[39m\u001b[32m~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/embedder.py:10\u001b[39m, in \u001b[36mEmbedder.__init__\u001b[39m\u001b[34m(self, path)\u001b[39m\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, path=\u001b[33m\"\u001b[39m\u001b[33mmodels/Xenova/all-MiniLM-L6-v2\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m 9\u001b[39m path = Path(path)\n\u001b[32m---> \u001b[39m\u001b[32m10\u001b[39m \u001b[38;5;28mself\u001b[39m.tokenizer = \u001b[30;43mTokenizer\u001b[39;49m\u001b[30;43m.\u001b[39;49m\u001b[30;43mfrom_file\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mstr\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mpath\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43m/\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43m\"\u001b[39;49m\u001b[30;43mtokenizer.json\u001b[39;49m\u001b[30;43m\"\u001b[39;49m\u001b[30;43m)\u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m 11\u001b[39m \u001b[38;5;28mself\u001b[39m.session = ort.InferenceSession(\n\u001b[32m 12\u001b[39m \u001b[38;5;28mstr\u001b[39m(path / \u001b[33m\"\u001b[39m\u001b[33mmodel.onnx\u001b[39m\u001b[33m\"\u001b[39m), providers=[\u001b[33m\"\u001b[39m\u001b[33mCPUExecutionProvider\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m 13\u001b[39m )\n\u001b[32m 14\u001b[39m \u001b[38;5;28mself\u001b[39m.input_names = {inp.name \u001b[38;5;28;01mfor\u001b[39;00m inp \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.session.get_inputs()}\n", "\u001b[31mException\u001b[39m: No such file or directory (os error 2)" ] } ], "source": [ "import os\n", "import time\n", "import pandas as pd\n", "from sqlitesearch import VectorSearchIndex\n", "from embedder import Embedder\n", "from tqdm.auto import tqdm\n", "import numpy as np\n", "\n", "# 1. DEFINE THE 4 TOURNAMENT CANDIDATES\n", "candidate_models = [\n", " {\n", " \"name\": \"all-MiniLM-L6-v2\",\n", " \"path\": \"models/Xenova/all-MiniLM-L6-v2\",\n", " \"dim\": 384,\n", " \"notes\": \"Lightweight Baseline\"\n", " },\n", " {\n", " \"name\": \"all-MiniLM-L12-v2\",\n", " \"path\": \"models/Xenova/all-MiniLM-L12-v2\",\n", " \"dim\": 384,\n", " \"notes\": \"Mid-Weight Quality Jump\"\n", " },\n", " {\n", " \"name\": \"nomic-embed-text-v1\",\n", " \"path\": \"models/Xenova/nomic-embed-text-v1\",\n", " \"dim\": 384,\n", " \"notes\": \"Mid-Weight Quality Jump\"\n", " },\n", " {\n", " \"name\": \"bge-base-en-v1.5\",\n", " \"path\": \"models/Xenova/bge-base-en-v1.5\",\n", " \"dim\": 768,\n", " \"notes\": \"Retrieval Specialist\"\n", " }\n", "]\n", "\n", "comparison_results = []\n", "\n", "# 3. EXECUTE THE BENCHMARK LOOP\n", "for model_info in candidate_models:\n", " print(f\"\\n--- Loading and evaluating model: {model_info['name']} ---\")\n", "\n", " # Initialize the model locally via sentence-transformers (easy local alternative for embedding generation)\n", " model = Embedder(model_info[\"path\"])\n", "\n", " # Prepare database storage path\n", " db_path = f\"sqlite/vector/evals/embedding-models/{model_info['name']}/eval_db.db\"\n", " if os.path.exists(db_path):\n", " os.remove(db_path)\n", "\n", " # Text strings to index\n", " faq_texts = [f\"Q: {item['question']} A: {item['answer']}\" for item in documents]\n", "\n", " # Track indexing speed\n", " print(\"Generating Embeddings in Batches...\")\n", " start_gen_time = time.time()\n", "\n", " batch_size = 50\n", " db_vectors = []\n", "\n", " for i in tqdm(range(0, len(faq_texts), batch_size)):\n", " batch = faq_texts[i:i + batch_size]\n", " batch_vectors = model.encode_batch(batch)\n", " db_vectors.extend(batch_vectors)\n", "\n", " db_vectors = np.array(db_vectors)\n", "\n", " gen_duration = time.time() - start_gen_time\n", " print(f\"Embedding Generation Complete! Time taken: {gen_duration:.2f} seconds.\")\n", "\n", " # Insert vectors into sqlitesearch DB\n", " print(\"Building HNSW Index in SQLite...\")\n", "\n", " start_fit_time = time.time()\n", " vector_index = VectorSearchIndex(mode=\"hnsw\", db_path=db_path)\n", " vector_index.fit(db_vectors, documents)\n", " fit_duration = time.time() - start_fit_time\n", "\n", " print(f\"Database Indexing Complete! Time taken: {fit_duration:.2f} seconds.\")\n", "\n", " # Run the query test evaluation\n", " hits_at_1 = 0\n", " total_queries = len(clinical_eval_set)\n", " total_latency_ms = 0\n", "\n", " for i in tqdm(range(0, len(clinical_eval_set))):\n", " test = clinical_eval_set[i]\n", " # Measure latency per retrieval\n", " start_query_time = time.time()\n", "\n", " # Generate query vector\n", " q_vector = model.encode(test[\"question\"])\n", "\n", " # Query local sqlite vector database (limit=1 forces strict top rank scoring)\n", " results = vector_index.search(q_vector, num_results=1)\n", "\n", " query_duration = (time.time() - start_query_time) * 1000\n", " total_latency_ms += query_duration\n", "\n", " # Check if the top result is the correct clinical match\n", " if results and results[0].get(\"id\") == test[\"document\"]:\n", " hits_at_1 += 1\n", "\n", " # Calculate performance matrix stats\n", " accuracy_rate = (hits_at_1 / total_queries) * 100\n", " avg_latency = total_latency_ms / total_queries\n", " db_file_size_kb = os.path.getsize(db_path) / 1024\n", "\n", " # Append analytics payloads\n", " comparison_results.append({\n", " \"Model Name\": model_info[\"name\"],\n", " \"Vector Dimensions\": model_info[\"dim\"],\n", " \"Hit Rate @ 1 (Accuracy)\": f\"{accuracy_rate:.1f}%\",\n", " \"Avg Search Latency (ms)\": f\"{avg_latency:.2f}ms\",\n", " \"SQLite DB Size (KB)\": f\"{db_file_size_kb:.1f} KB\",\n", " \"Notes\": model_info[\"notes\"]\n", " })\n", "\n", " # Clear the temporary DB file footprint from memory/disk\n", " if os.path.exists(db_path):\n", " vector_index.close()\n", " os.remove(db_path)\n", "\n", "# 4. PRINT PERFORMANCE MATRIX TABLE\n", "df_comparison = pd.DataFrame(comparison_results)\n", "print(\"\\n\" + \"=\"*80)\n", "print(\" FIRST AID EMBEDDING MODEL TOURNAMENT MATRIX \")\n", "print(\"=\"*80)\n", "print(df_comparison.to_string(index=False))\n", "print(\"=\"*80)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "97b7f853", "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'df_comparison' is not defined", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m df_comparison.to_csv(\u001b[33m\"data/embedding-models-comparison.csv\"\u001b[39m, index=\u001b[38;5;28;01mFalse\u001b[39;00m)\n", "\u001b[31mNameError\u001b[39m: name 'df_comparison' is not defined" ] } ], "source": [ "df_comparison.to_csv(\"data/embedding-models-comparison.csv\", index=False)" ] } ], "metadata": { "kernelspec": { "display_name": "first-aid-qa-assistant-onnx", "language": "python", "name": "first-aid-qa-assistant-onnx" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" } }, "nbformat": 4, "nbformat_minor": 5 }