import path_setup  # noqa: F401 — adds project root to sys.path

%load_ext autoreload
%autoreload 2

import pandas as pd

df_ground_truth = pd.read_csv("data/ground_truth.csv")
ground_truth = df_ground_truth.to_dict(orient="records")

from ingest import load_faq_data, build_index, build_vector_index

documents = load_faq_data()

text_index = build_index(documents)

from embedder import Embedder

embedder = Embedder()
index = build_vector_index(embedder, documents)

Generating Embeddings in Batches...



  0%|          | 0/56 [00:00<?, ?it/s]

from evaluation.evaluation import text_search, vector_search, rrf, evaluate

def hybrid_search(query,  k=60):
    text_results = text_search(query, text_index, num_results=10)
    vector_results = vector_search(query, embedder, index, num_results=10)

    return rrf([text_results, vector_results], k=k)

evaluate(ground_truth, hybrid_search)

  0%|          | 0/13874 [00:00<?, ?it/s]





{'hit_rate': 0.6015568689635289, 'mrr': 0.4079116813223703}

results_tune = []
for k in [1, 50, 100, 200]:
    print(f"Evaluating Hybrid Search k={k} ...")
    result = evaluate(
        ground_truth,
        lambda query, k=k: hybrid_search(
            query,
            k
        )
    )

    results_tune.append({
        "k": k,
        "hit_rate": result["hit_rate"],
        "mrr": result["mrr"],
    })

Evaluating Hybrid Search k=1 ...



  0%|          | 0/13874 [00:00<?, ?it/s]


Evaluating Hybrid Search k=50 ...



  0%|          | 0/13874 [00:00<?, ?it/s]


Evaluating Hybrid Search k=100 ...



  0%|          | 0/13874 [00:00<?, ?it/s]


Evaluating Hybrid Search k=200 ...



  0%|          | 0/13874 [00:00<?, ?it/s]

df_results = pd.DataFrame(results_tune)
df_results.sort_values("mrr", ascending=False).head(4)