import path_setup # noqa: F401 — adds project root to sys.path
%load_ext autoreload
%autoreload 2
import pandas as pd
df_ground_truth = pd.read_csv("data/ground_truth.csv")
ground_truth = df_ground_truth.to_dict(orient="records")
from ingest import load_faq_data, build_index
documents = load_faq_data()
len(documents)
2775
index = build_index(documents)
from evaluation.evaluation import evaluate, search_boosts
results = []
for question_boost in [0.5, 1.0, 1.5]:
for answer_boost in [1.0, 2.0, 4.0, 10.0]:
print(
f"Evaluating question_boost={question_boost},"
f" answer_boost={answer_boost},"
)
result = evaluate(
ground_truth,
lambda query, question_boost=question_boost, answer_boost=answer_boost: search_boosts(
query,
question_boost,
answer_boost,
index
)
)
results.append({
"question": question_boost,
"answer": answer_boost,
"hit_rate": result["hit_rate"],
"mrr": result["mrr"],
})
Evaluating question_boost=0.5, answer_boost=1.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=0.5, answer_boost=2.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=0.5, answer_boost=4.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=0.5, answer_boost=10.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=1.0, answer_boost=1.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=1.0, answer_boost=2.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=1.0, answer_boost=4.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=1.0, answer_boost=10.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=1.5, answer_boost=1.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=1.5, answer_boost=2.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=1.5, answer_boost=4.0,
0%| | 0/13874 [00:00<?, ?it/s]
Evaluating question_boost=1.5, answer_boost=10.0,
0%| | 0/13874 [00:00<?, ?it/s]
df_results = pd.DataFrame(results)
df_results.sort_values("mrr", ascending=False).head(10)
|
question |
answer |
hit_rate |
mrr |
| 0 |
0.5 |
1.0 |
0.427274 |
0.28798 |
| 1 |
0.5 |
2.0 |
0.427274 |
0.28798 |
| 2 |
0.5 |
4.0 |
0.427274 |
0.28798 |
| 3 |
0.5 |
10.0 |
0.427274 |
0.28798 |
| 4 |
1.0 |
1.0 |
0.427274 |
0.28798 |
| 5 |
1.0 |
2.0 |
0.427274 |
0.28798 |
| 6 |
1.0 |
4.0 |
0.427274 |
0.28798 |
| 7 |
1.0 |
10.0 |
0.427274 |
0.28798 |
| 8 |
1.5 |
1.0 |
0.427274 |
0.28798 |
| 9 |
1.5 |
2.0 |
0.427274 |
0.28798 |
from evaluation.evaluation import text_search
evaluate(
ground_truth,
lambda query='', index=index: text_search(query, index)
)
0%| | 0/13874 [00:00<?, ?it/s]
{'hit_rate': 0.4272740377684878, 'mrr': 0.2879799144683107}