import path_setup # noqa: F401 — adds project root to sys.path
%load_ext autoreload
%autoreload 2
import pandas as pd
df_ground_truth = pd.read_csv("data/ground_truth.csv")
ground_truth = df_ground_truth.to_dict(orient="records")
from ingest import load_faq_data
documents = load_faq_data()
df_ground_truth = pd.read_csv("data/clinical_eval_set.csv")
clinical_eval_set = df_ground_truth.to_dict(orient="records")
import os
import time
import pandas as pd
from sqlitesearch import VectorSearchIndex
from embedder import Embedder
from tqdm.auto import tqdm
import numpy as np
# 1. DEFINE THE 4 TOURNAMENT CANDIDATES
candidate_models = [
{
"name": "all-MiniLM-L6-v2",
"path": "models/Xenova/all-MiniLM-L6-v2",
"dim": 384,
"notes": "Lightweight Baseline"
},
{
"name": "all-MiniLM-L12-v2",
"path": "models/Xenova/all-MiniLM-L12-v2",
"dim": 384,
"notes": "Mid-Weight Quality Jump"
},
{
"name": "nomic-embed-text-v1",
"path": "models/Xenova/nomic-embed-text-v1",
"dim": 384,
"notes": "Mid-Weight Quality Jump"
},
{
"name": "bge-base-en-v1.5",
"path": "models/Xenova/bge-base-en-v1.5",
"dim": 768,
"notes": "Retrieval Specialist"
}
]
comparison_results = []
# 3. EXECUTE THE BENCHMARK LOOP
for model_info in candidate_models:
print(f"\n--- Loading and evaluating model: {model_info['name']} ---")
# Initialize the model locally via sentence-transformers (easy local alternative for embedding generation)
model = Embedder(model_info["path"])
# Prepare database storage path
db_path = f"sqlite/vector/evals/embedding-models/{model_info['name']}/eval_db.db"
if os.path.exists(db_path):
os.remove(db_path)
# Text strings to index
faq_texts = [f"Q: {item['question']} A: {item['answer']}" for item in documents]
# Track indexing speed
print("Generating Embeddings in Batches...")
start_gen_time = time.time()
batch_size = 50
db_vectors = []
for i in tqdm(range(0, len(faq_texts), batch_size)):
batch = faq_texts[i:i + batch_size]
batch_vectors = model.encode_batch(batch)
db_vectors.extend(batch_vectors)
db_vectors = np.array(db_vectors)
gen_duration = time.time() - start_gen_time
print(f"Embedding Generation Complete! Time taken: {gen_duration:.2f} seconds.")
# Insert vectors into sqlitesearch DB
print("Building HNSW Index in SQLite...")
start_fit_time = time.time()
vector_index = VectorSearchIndex(mode="hnsw", db_path=db_path)
vector_index.fit(db_vectors, documents)
fit_duration = time.time() - start_fit_time
print(f"Database Indexing Complete! Time taken: {fit_duration:.2f} seconds.")
# Run the query test evaluation
hits_at_1 = 0
total_queries = len(clinical_eval_set)
total_latency_ms = 0
for i in tqdm(range(0, len(clinical_eval_set))):
test = clinical_eval_set[i]
# Measure latency per retrieval
start_query_time = time.time()
# Generate query vector
q_vector = model.encode(test["question"])
# Query local sqlite vector database (limit=1 forces strict top rank scoring)
results = vector_index.search(q_vector, num_results=1)
query_duration = (time.time() - start_query_time) * 1000
total_latency_ms += query_duration
# Check if the top result is the correct clinical match
if results and results[0].get("id") == test["document"]:
hits_at_1 += 1
# Calculate performance matrix stats
accuracy_rate = (hits_at_1 / total_queries) * 100
avg_latency = total_latency_ms / total_queries
db_file_size_kb = os.path.getsize(db_path) / 1024
# Append analytics payloads
comparison_results.append({
"Model Name": model_info["name"],
"Vector Dimensions": model_info["dim"],
"Hit Rate @ 1 (Accuracy)": f"{accuracy_rate:.1f}%",
"Avg Search Latency (ms)": f"{avg_latency:.2f}ms",
"SQLite DB Size (KB)": f"{db_file_size_kb:.1f} KB",
"Notes": model_info["notes"]
})
# Clear the temporary DB file footprint from memory/disk
if os.path.exists(db_path):
vector_index.close()
os.remove(db_path)
# 4. PRINT PERFORMANCE MATRIX TABLE
df_comparison = pd.DataFrame(comparison_results)
print("\n" + "="*80)
print(" FIRST AID EMBEDDING MODEL TOURNAMENT MATRIX ")
print("="*80)
print(df_comparison.to_string(index=False))
print("="*80)
--- Loading and evaluating model: all-MiniLM-L6-v2 ---
Generating Embeddings in Batches...
0%| | 0/56 [00:00<?, ?it/s]
Embedding Generation Complete! Time taken: 214.93 seconds.
Building HNSW Index in SQLite...
Database Indexing Complete! Time taken: 112.02 seconds.
0%| | 0/5526 [00:00<?, ?it/s]
--- Loading and evaluating model: all-MiniLM-L12-v2 ---
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
Cell In[6], line 44
40 for model_info in candidate_models:
41 print(f"\n--- Loading and evaluating model: {model_info['name']} ---")
42
43 # Initialize the model locally via sentence-transformers (easy local alternative for embedding generation)
---> 44 model = Embedder(model_info["path"])
45
46 # Prepare database storage path
47 db_path = f"sqlite/vector/evals/embedding-models/{model_info['name']}/eval_db.db"
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/embedder.py:10, in Embedder.__init__(self, path)
8 def __init__(self, path="models/Xenova/all-MiniLM-L6-v2"):
9 path = Path(path)
---> 10 self.tokenizer = Tokenizer.from_file(str(path / "tokenizer.json"))
11 self.session = ort.InferenceSession(
12 str(path / "model.onnx"), providers=["CPUExecutionProvider"]
13 )
14 self.input_names = {inp.name for inp in self.session.get_inputs()}
Exception: No such file or directory (os error 2)
df_comparison.to_csv("data/embedding-models-comparison.csv", index=False)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[1], line 1
----> 1 df_comparison.to_csv("data/embedding-models-comparison.csv", index=False)
NameError: name 'df_comparison' is not defined