import path_setup # noqa: F401 — adds project root to sys.path
%load_ext autoreload
%autoreload 2
from ingest import load_faq_data
documents = load_faq_data()
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
Cell In[3], line 2
1 from ingest import load_faq_data
----> 2 documents = load_faq_data()
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/src/ingest.py:11, in load_faq_data()
8 def load_faq_data():
9 documents = []
---> 11 with open("data/firstaidqa_v1-first-half.json", "r") as file:
12 documents = json.load(file)
14 return documents
FileNotFoundError: [Errno 2] No such file or directory: 'data/firstaidqa_v1-first-half.json'
len(documents)
2775
documents[0]
{'question': 'When should you move an injured person at an accident site?',
'answer': "You should only move an injured person if there is immediate danger such as a fire, oncoming traffic, or toxic fumes. Otherwise, it's best to leave them where they are, administer first aid on the spot, and wait for professional medical help to arrive.",
'id': 'a220806f-b207-480c-9a25-2787503e7efe'}
doc = documents[0]
print(doc["id"])
print(doc["question"])
print(doc["answer"])
a220806f-b207-480c-9a25-2787503e7efe
When should you move an injured person at an accident site?
You should only move an injured person if there is immediate danger such as a fire, oncoming traffic, or toxic fumes. Otherwise, it's best to leave them where they are, administer first aid on the spot, and wait for professional medical help to arrive.
from pydantic import BaseModel
class Questions(BaseModel):
questions: list[str]
data_gen_instructions = """
You emulate a person at the accident site.
You are given one Questions and Answers Knowledge Base (QA KB).
Formulate 5 questions this person might ask that are answered using QA KB.
Rules:
- The QA KB should contain the answer to each question.
- Make the questions complete and not too short.
- Use as few words as possible from the QA KB; don't copy its phrasing.
- The questions should resemble how people actually ask things online:
not too formal, not too short, not too long.
- Ask about the content of the QA KB, not about its formatting or structure.
""".strip()
from dotenv import load_dotenv
from config import make_llm_client, MODEL_NAME
load_dotenv()
# 1. Initialize your client object with a model list
litellm_client = make_llm_client()
[92m16:21:47 - LiteLLM:WARNING[0m: utils.py:2730 - register_model: model=5fde7336d29cebcd30558ae0085b7916413c5817d51a8443bbd6001c998e0990 not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info
[92m16:21:47 - LiteLLM:WARNING[0m: utils.py:2730 - register_model: model=mistral/ministral-3b-2512 not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info
import json
user_prompt = json.dumps(doc)
messages = [
{"role": "developer", "content": data_gen_instructions},
{"role": "user", "content": user_prompt}
]
response = litellm_client.completion(
model=MODEL_NAME,
messages=messages,
response_format=Questions
)
structured_data = Questions.model_validate_json(response.choices[0].message.content)
structured_data
Questions(questions=['Should I move someone who’s hurt in a car crash if there’s no immediate risk?', 'When’s it safe to leave an injured person at the scene until help arrives?', 'What’s the rule about moving someone if they’re trapped but not in danger?', 'Do I risk moving them if there’s no fire, traffic, or gas leaks?', 'When’s it okay to leave them and just help them stay still?'])
structured_data.questions
['Should I move someone who’s hurt in a car crash if there’s no immediate risk?',
'When’s it safe to leave an injured person at the scene until help arrives?',
'What’s the rule about moving someone if they’re trapped but not in danger?',
'Do I risk moving them if there’s no fire, traffic, or gas leaks?',
'When’s it okay to leave them and just help them stay still?']
from evaluation.evaluation_utils import llm_structured
result, usage = llm_structured(
litellm_client,
data_gen_instructions,
user_prompt,
Questions
)
print(result.questions)
['Should I move someone who’s hurt in a car crash if there’s a risk of fire or traffic coming my way?', 'When’s it okay to shift an injured person out of the way of a dangerous situation?', 'What’s the rule about moving someone if they’re trapped but no immediate threat?', 'Do I risk moving them if they’re hurt but there’s no fire or traffic danger?', 'What’s the best way to handle an injured person if the scene isn’t unsafe?']
usage.prompt_tokens, usage.completion_tokens
(245, 113)
from evaluation.evaluation_utils import calc_price
cost = calc_price(usage)
cost
{'input_cost': Decimal('0.0000245'),
'output_cost': Decimal('0.0000113'),
'total_cost': Decimal('0.0000358')}
records = []
for q in result.questions:
records.append({
"question": q,
"document": doc["id"]
})
records
[{'question': 'Should I move someone who’s hurt in a car crash if there’s a risk of fire or traffic coming my way?',
'document': 'a220806f-b207-480c-9a25-2787503e7efe'},
{'question': 'When’s it okay to shift an injured person out of the way of a dangerous situation?',
'document': 'a220806f-b207-480c-9a25-2787503e7efe'},
{'question': 'What’s the rule about moving someone if they’re trapped but no immediate threat?',
'document': 'a220806f-b207-480c-9a25-2787503e7efe'},
{'question': 'Do I risk moving them if they’re hurt but there’s no fire or traffic danger?',
'document': 'a220806f-b207-480c-9a25-2787503e7efe'},
{'question': 'What’s the best way to handle an injured person if the scene isn’t unsafe?',
'document': 'a220806f-b207-480c-9a25-2787503e7efe'}]
from evaluation.evaluation import generate_ground_truth
out, usage = generate_ground_truth(
doc,
litellm_client,
data_gen_instructions,
Questions
)
print(out, usage)
[{'question': 'Should I move someone who’s hurt in a car crash if there’s no immediate risk like fire or traffic?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}, {'question': 'When’s it okay to shift an injured person out of the way, and when should I just stay put?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}, {'question': 'What’s the rule about moving someone if they’re trapped but not in danger from flames or cars?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}, {'question': 'Am I safe leaving an injured person behind if there’s no fire or traffic hazard?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}, {'question': 'How do I know if I should move someone or just wait for help?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}] Usage(completion_tokens=116, prompt_tokens=245, total_tokens=361, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cache_write_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None, video_tokens=None))
from tqdm.auto import tqdm
ground_truth = []
usages = []
for doc in tqdm(documents[:5]):
records, usage = generate_ground_truth(
doc,
litellm_client,
data_gen_instructions,
Questions
)
ground_truth.extend(records)
usages.append(usage)
0%| | 0/5 [00:00<?, ?it/s]
from concurrent.futures import ThreadPoolExecutor
from evaluation.evaluation_utils import map_progress
with ThreadPoolExecutor(max_workers=6) as pool:
results = map_progress(
pool,
documents,
generate_ground_truth,
litellm_client,
data_gen_instructions,
Questions
)
0%| | 0/2775 [00:00<?, ?it/s]
ground_truth = []
usages = []
for records, usage in results:
ground_truth.extend(records)
usages.append(usage)
len(ground_truth)
13874
from evaluation.evaluation_utils import calc_total_price
calc_total_price(usages)
0.09661060000000017
import pandas as pd
df_ground_truth = pd.DataFrame(ground_truth)
df_ground_truth.to_csv("data/ground_truth.csv", index=False)