import path_setup  # noqa: F401 — adds project root to sys.path

%load_ext autoreload
%autoreload 2

The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload

import pandas as pd

df_ground_truth = pd.read_csv("data/ground_truth.csv")
ground_truth = df_ground_truth.to_dict(orient="records")

from ingest import load_faq_data, build_vector_index

documents = load_faq_data()

len(documents)

2775

from embedder import Embedder

embedder = Embedder()
index = build_vector_index(embedder, documents)

Generating Embeddings in Batches...



  0%|          | 0/56 [00:00<?, ?it/s]

from evaluation.evaluation import evaluate, vector_search

results = []

for num_of_results in [3, 5, 10]:
    print(
        f"Evaluating vector search num_of_results={num_of_results},"
    )
    result = evaluate(
        ground_truth,
        lambda query, embedder=embedder, num_results=num_of_results: vector_search(
            query,
            embedder,
            index,
            num_results
        )
    )

    results.append({
        "num_of_results": num_of_results,
        "hit_rate": result["hit_rate"],
        "mrr": result["mrr"],
    })

Evaluating question_boost=3,



  0%|          | 0/13874 [00:00<?, ?it/s]


Evaluating question_boost=5,



  0%|          | 0/13874 [00:00<?, ?it/s]


Evaluating question_boost=10,



  0%|          | 0/13874 [00:00<?, ?it/s]

df_results = pd.DataFrame(results)
df_results.sort_values("mrr", ascending=False).head(10)

from evaluation.evaluation import vector_search

evaluate(
    ground_truth,
    lambda query='', embedder=embedder, index=index: vector_search(query, embedder, index, 10)
)

  0%|          | 0/13874 [00:00<?, ?it/s]





{'hit_rate': 0.7320167219259046, 'mrr': 0.46667842212565647}