import path_setup  # noqa: F401 — adds project root to sys.path

%load_ext autoreload
%autoreload 2

import pandas as pd

df_ground_truth = pd.read_csv("data/ground_truth.csv")
ground_truth = df_ground_truth.to_dict(orient="records")

from ingest import load_faq_data, build_index, build_vector_index
documents = load_faq_data()

doc_idx = {}

for doc in documents:
    doc_idx[doc["id"]] = doc

text_index = build_index(documents)

from embedder import Embedder

embedder = Embedder()
index = build_vector_index(embedder, documents)

Generating Embeddings in Batches...



  0%|          | 0/56 [00:00<?, ?it/s]

from dotenv import load_dotenv
from config import make_llm_client, MODEL_NAME

load_dotenv()

# 1. Initialize your client object with a model list
litellm_client = make_llm_client()

from evaluation.evaluation_utils import RAGWithUsage

assistant = RAGWithUsage(
    index=index,
    llm_client=litellm_client,
    text_index=text_index,
    vector_index=index,
    embedder=embedder,
)

from evaluation.evaluation import generate_rag_answer

from concurrent.futures import ThreadPoolExecutor
from evaluation.evaluation_utils import map_progress_answers

answer_record = generate_rag_answer(ground_truth[1], doc_idx, assistant)
answer_record

{'question': 'Should I move a person if the scene is unsafe but no immediate threats like fire or traffic?',
 'answer_llm': 'No, you should **not** move the person if there is no immediate danger (such as fire, oncoming traffic, or toxic fumes). The best course of action is to stay with them, administer first aid on the spot, and wait for professional medical help to arrive.',
 'answer_orig': "You should only move an injured person if there is immediate danger such as a fire, oncoming traffic, or toxic fumes. Otherwise, it's best to leave them where they are, administer first aid on the spot, and wait for professional medical help to arrive.",
 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}

with ThreadPoolExecutor(max_workers=6) as pool:
    results = map_progress_answers(pool, ground_truth, generate_rag_answer, doc_idx, assistant)

  0%|          | 0/13874 [00:00<?, ?it/s]

len(results)

13874

answers = []

for answer_record in results:
    answers.append(answer_record)

assistant.total_cost()

1.3239971000000008

#df_answers = pd.DataFrame(answers)
#df_answers.to_csv("data/rag-answers.csv", index=False)

import pandas as pd

df_answers = pd.read_csv("data/rag-answers.csv")
answers = df_answers.to_dict(orient="records")

from pydantic import BaseModel, Field
from typing import Literal

class AnswerEvaluation(BaseModel):
    reasoning: str = Field(
        description="Reasoning about the quality of the answer."
    )
    score: Literal["good", "bad"] = Field(
        description="'good' if the answer is correct and complete, 'bad' otherwise."
    )

aqa_judge_instructions = """
You are an expert evaluator. You will be given:
1. A question from a person at accident site
2. The original answer from the FAQ (ground truth)
3. An answer generated by an AI assistant

Your task is to decide if the AI answer is semantically equivalent to
the original answer.

Rules:
- The AI answer does NOT need to be word-for-word identical
- It should convey the same key information
- Extra detail is fine as long as the core answer is correct
- Mark 'bad' only if the AI answer is wrong or misses the key point

Be fair and focus on correctness, not style.
""".strip()

aqa_judge_prompt = """
Question:
{question}

Original Answer (ground truth):
{answer_orig}

AI Answer:
{answer_llm}
""".strip()

rec = answers[0]

from evaluation.evaluation_utils import calc_price, calc_total_price, llm_structured_retry, map_progress
from evaluation.evaluation import evaluate_aqa

eval_result, usage = evaluate_aqa(
    rec["question"],
    rec["answer_orig"],
    rec["answer_llm"],
    litellm_client,
    aqa_judge_instructions,
    aqa_judge_prompt,
    AnswerEvaluation,
)

eval_result

AnswerEvaluation(reasoning="The AI answer correctly conveys the core principle of the original FAQ: **do not move an injured person if there is no immediate danger** (e.g., fire, traffic, toxic fumes). It retains the key points—leaving them in place, administering first aid on-site, and waiting for professional help—while adding clarity with an example ('no fire, no traffic, no toxic fumes'). The phrasing is slightly more concise but still accurate. The only minor deviation is the inclusion of the phrase *'unless absolutely necessary'* (which is not explicitly stated in the original), but this does not undermine the correctness of the core advice. The AI answer is semantically equivalent and well-structured.", score='good')

from concurrent.futures import ThreadPoolExecutor
from evaluation.evaluation_utils import map_progress_judge
from evaluation.evaluation import judge_record

with ThreadPoolExecutor(max_workers=6) as pool:
    results, usage = map_progress_judge(
        pool,
        answers,
        judge_record,
        litellm_client,
        aqa_judge_instructions,
        aqa_judge_prompt,
        AnswerEvaluation,
    )

  0%|          | 0/13874 [00:00<?, ?it/s]


22:49:00 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:49:20 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:49:26 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:50:05 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:50:57 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:54:40 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:55:55 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:56:50 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:57:19 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:57:56 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:58:20 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:58:23 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:58:44 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
22:59:14 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:00:02 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:00:22 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:00:27 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:01:48 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:01:57 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:03:17 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:04:52 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:10:06 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:11:20 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:11:24 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:13:34 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:14:03 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:14:14 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:16:01 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:16:32 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:16:43 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:19:19 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:20:01 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:23:00 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:23:36 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'
23:23:40 - LiteLLM:WARNING: core_helpers.py:110 - Unmapped finish_reason 'error', defaulting to 'stop'

evaluations = []

for evaluation in results:
    evaluations.append(evaluation)

df_eval = pd.DataFrame(evaluations)

len(evaluations)

13874

good_count = (df_eval["score"] == "good").sum()
total_count = len(df_eval)
print(f"Good: {good_count}/{total_count} = {good_count/total_count:.2%}")

Good: 13234/13874 = 95.39%

df_eval[df_eval["score"] == "bad"].head()