{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "8213bc5f", "metadata": {}, "outputs": [], "source": [ "import path_setup # noqa: F401 — adds project root to sys.path\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "dfd8c461", "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": null, "id": "9f7965b5", "metadata": {}, "outputs": [ { "ename": "FileNotFoundError", "evalue": "[Errno 2] No such file or directory: 'data/firstaidqa_v1-first-half.json'", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", "\u001b[31mFileNotFoundError\u001b[39m Traceback (most recent call last)", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m ingest \u001b[38;5;28;01mimport\u001b[39;00m load_faq_data\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m documents = load_faq_data()\n", "\u001b[36mFile \u001b[39m\u001b[32m~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/src/ingest.py:11\u001b[39m, in \u001b[36mload_faq_data\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mload_faq_data\u001b[39m():\n\u001b[32m 9\u001b[39m documents = []\n\u001b[32m---> \u001b[39m\u001b[32m11\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[30;43mopen\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43m\"\u001b[39;49m\u001b[30;43mdata/firstaidqa_v1-first-half.json\u001b[39;49m\u001b[30;43m\"\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43m\"\u001b[39;49m\u001b[30;43mr\u001b[39;49m\u001b[30;43m\"\u001b[39;49m\u001b[30;43m)\u001b[39;49m \u001b[38;5;28;01mas\u001b[39;00m file:\n\u001b[32m 12\u001b[39m documents = json.load(file)\n\u001b[32m 14\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m documents\n", "\u001b[31mFileNotFoundError\u001b[39m: [Errno 2] No such file or directory: 'data/firstaidqa_v1-first-half.json'" ] } ], "source": [ "from ingest import load_faq_data\n", "documents = load_faq_data()" ] }, { "cell_type": "code", "execution_count": 3, "id": "d4ea4819", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2775" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(documents)" ] }, { "cell_type": "code", "execution_count": 5, "id": "68bda7fc", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'question': 'When should you move an injured person at an accident site?',\n", " 'answer': \"You should only move an injured person if there is immediate danger such as a fire, oncoming traffic, or toxic fumes. Otherwise, it's best to leave them where they are, administer first aid on the spot, and wait for professional medical help to arrive.\",\n", " 'id': 'a220806f-b207-480c-9a25-2787503e7efe'}" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "documents[0]" ] }, { "cell_type": "code", "execution_count": 3, "id": "013e5d4a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "a220806f-b207-480c-9a25-2787503e7efe\n", "When should you move an injured person at an accident site?\n", "You should only move an injured person if there is immediate danger such as a fire, oncoming traffic, or toxic fumes. Otherwise, it's best to leave them where they are, administer first aid on the spot, and wait for professional medical help to arrive.\n" ] } ], "source": [ "doc = documents[0]\n", "print(doc[\"id\"])\n", "print(doc[\"question\"])\n", "print(doc[\"answer\"])" ] }, { "cell_type": "code", "execution_count": 4, "id": "9b09e856", "metadata": {}, "outputs": [], "source": [ "from pydantic import BaseModel\n", "\n", "class Questions(BaseModel):\n", " questions: list[str]" ] }, { "cell_type": "code", "execution_count": 5, "id": "fdf29590", "metadata": {}, "outputs": [], "source": [ "data_gen_instructions = \"\"\"\n", "You emulate a person at the accident site.\n", "You are given one Questions and Answers Knowledge Base (QA KB).\n", "Formulate 5 questions this person might ask that are answered using QA KB.\n", "\n", "Rules:\n", "- The QA KB should contain the answer to each question.\n", "- Make the questions complete and not too short.\n", "- Use as few words as possible from the QA KB; don't copy its phrasing.\n", "- The questions should resemble how people actually ask things online:\n", " not too formal, not too short, not too long.\n", "- Ask about the content of the QA KB, not about its formatting or structure.\n", "\"\"\".strip()" ] }, { "cell_type": "code", "execution_count": 6, "id": "2998368d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[92m16:21:47 - LiteLLM:WARNING\u001b[0m: utils.py:2730 - register_model: model=5fde7336d29cebcd30558ae0085b7916413c5817d51a8443bbd6001c998e0990 not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info\n", "\u001b[92m16:21:47 - LiteLLM:WARNING\u001b[0m: utils.py:2730 - register_model: model=mistral/ministral-3b-2512 not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info\n" ] } ], "source": [ "from dotenv import load_dotenv\n", "from config import make_llm_client, MODEL_NAME\n", "\n", "load_dotenv()\n", "\n", "# 1. Initialize your client object with a model list\n", "litellm_client = make_llm_client()\n" ] }, { "cell_type": "code", "execution_count": 7, "id": "6720f65c", "metadata": {}, "outputs": [], "source": [ "import json\n", "\n", "user_prompt = json.dumps(doc)" ] }, { "cell_type": "code", "execution_count": 8, "id": "58e4495c", "metadata": {}, "outputs": [], "source": [ "messages = [\n", " {\"role\": \"developer\", \"content\": data_gen_instructions},\n", " {\"role\": \"user\", \"content\": user_prompt}\n", "]" ] }, { "cell_type": "code", "execution_count": 9, "id": "8e99599b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Questions(questions=['Should I move someone who’s hurt in a car crash if there’s no immediate risk?', 'When’s it safe to leave an injured person at the scene until help arrives?', 'What’s the rule about moving someone if they’re trapped but not in danger?', 'Do I risk moving them if there’s no fire, traffic, or gas leaks?', 'When’s it okay to leave them and just help them stay still?'])" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "response = litellm_client.completion(\n", " model=MODEL_NAME,\n", " messages=messages,\n", " response_format=Questions\n", ")\n", "\n", "structured_data = Questions.model_validate_json(response.choices[0].message.content)\n", "structured_data" ] }, { "cell_type": "code", "execution_count": 10, "id": "44d82801", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Should I move someone who’s hurt in a car crash if there’s no immediate risk?',\n", " 'When’s it safe to leave an injured person at the scene until help arrives?',\n", " 'What’s the rule about moving someone if they’re trapped but not in danger?',\n", " 'Do I risk moving them if there’s no fire, traffic, or gas leaks?',\n", " 'When’s it okay to leave them and just help them stay still?']" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "structured_data.questions" ] }, { "cell_type": "code", "execution_count": 11, "id": "8be66efa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['Should I move someone who’s hurt in a car crash if there’s a risk of fire or traffic coming my way?', 'When’s it okay to shift an injured person out of the way of a dangerous situation?', 'What’s the rule about moving someone if they’re trapped but no immediate threat?', 'Do I risk moving them if they’re hurt but there’s no fire or traffic danger?', 'What’s the best way to handle an injured person if the scene isn’t unsafe?']\n" ] } ], "source": [ "from evaluation.evaluation_utils import llm_structured\n", "\n", "result, usage = llm_structured(\n", " litellm_client,\n", " data_gen_instructions,\n", " user_prompt,\n", " Questions\n", ")\n", "\n", "print(result.questions)" ] }, { "cell_type": "code", "execution_count": 13, "id": "94e759c4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(245, 113)" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "usage.prompt_tokens, usage.completion_tokens" ] }, { "cell_type": "code", "execution_count": 14, "id": "6598605e", "metadata": {}, "outputs": [], "source": [ "from evaluation.evaluation_utils import calc_price" ] }, { "cell_type": "code", "execution_count": 18, "id": "0fffe29d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'input_cost': Decimal('0.0000245'),\n", " 'output_cost': Decimal('0.0000113'),\n", " 'total_cost': Decimal('0.0000358')}" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cost = calc_price(usage)\n", "\n", "cost" ] }, { "cell_type": "code", "execution_count": 19, "id": "0607df3f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'question': 'Should I move someone who’s hurt in a car crash if there’s a risk of fire or traffic coming my way?',\n", " 'document': 'a220806f-b207-480c-9a25-2787503e7efe'},\n", " {'question': 'When’s it okay to shift an injured person out of the way of a dangerous situation?',\n", " 'document': 'a220806f-b207-480c-9a25-2787503e7efe'},\n", " {'question': 'What’s the rule about moving someone if they’re trapped but no immediate threat?',\n", " 'document': 'a220806f-b207-480c-9a25-2787503e7efe'},\n", " {'question': 'Do I risk moving them if they’re hurt but there’s no fire or traffic danger?',\n", " 'document': 'a220806f-b207-480c-9a25-2787503e7efe'},\n", " {'question': 'What’s the best way to handle an injured person if the scene isn’t unsafe?',\n", " 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}]" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "records = []\n", "\n", "for q in result.questions:\n", " records.append({\n", " \"question\": q,\n", " \"document\": doc[\"id\"]\n", " })\n", "\n", "records" ] }, { "cell_type": "code", "execution_count": 22, "id": "39e6bf5b", "metadata": {}, "outputs": [], "source": [ "from evaluation.evaluation import generate_ground_truth\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b4e47ab1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[{'question': 'Should I move someone who’s hurt in a car crash if there’s no immediate risk like fire or traffic?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}, {'question': 'When’s it okay to shift an injured person out of the way, and when should I just stay put?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}, {'question': 'What’s the rule about moving someone if they’re trapped but not in danger from flames or cars?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}, {'question': 'Am I safe leaving an injured person behind if there’s no fire or traffic hazard?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}, {'question': 'How do I know if I should move someone or just wait for help?', 'document': 'a220806f-b207-480c-9a25-2787503e7efe'}] Usage(completion_tokens=116, prompt_tokens=245, total_tokens=361, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cache_write_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None, video_tokens=None))\n" ] } ], "source": [ "out, usage = generate_ground_truth(\n", " doc,\n", " litellm_client,\n", " data_gen_instructions,\n", " Questions\n", " )\n", "\n", "print(out, usage)" ] }, { "cell_type": "code", "execution_count": null, "id": "ccb9916c", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f5b200d21e2041cbad5dfae870d82f72", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/5 [00:00