import path_setup # noqa: F401 — adds project root to sys.path
%load_ext autoreload
%autoreload 2
import pandas as pd
df_ground_truth = pd.read_csv("data/ground_truth.csv")
ground_truth = df_ground_truth.to_dict(orient="records")
from ingest import load_faq_data, build_index, build_vector_index
documents = load_faq_data()
doc_idx = {}
for doc in documents:
doc_idx[doc["id"]] = doc
text_index = build_index(documents)
from embedder import Embedder
embedder = Embedder()
index = build_vector_index(embedder, documents)
from dotenv import load_dotenv
load_dotenv()
True
from litellm import completion
from config import make_llm_client, MODEL_NAME
# 1. Initialize your client object with a model list
litellm_router = make_llm_client()
from copy import deepcopy
from toyaikit.llm import OpenAIChatCompletionsClient
# 2. Build a custom wrapper class to satisfy ToyAIKit's send_request contract
class LiteLLMToToyAIClient(OpenAIChatCompletionsClient):
def __init__(self, router, model_name="mistral-large"):
self.router = router
self.model_name = model_name
@staticmethod
def _clean_tool_schema(tools):
if tools is None:
return None
cleaned_tools = []
for tool in tools:
if isinstance(tool, dict):
tool = deepcopy(tool)
params = tool.get("parameters")
if isinstance(params, dict):
properties = params.get("properties")
if isinstance(properties, dict) and "self" in properties:
properties.pop("self", None)
required = params.get("required")
if isinstance(required, list):
params["required"] = [p for p in required if p != "self"]
cleaned_tools.append(tool)
else:
cleaned_tools.append(tool)
return cleaned_tools
def send_request(self, chat_messages, tools=None, output_format=None, **kwargs):
"""ToyAIKit runners call this method internally."""
if tools is None:
litellm_tools = None
elif hasattr(tools, "get_tools"):
litellm_tools = self._clean_tool_schema(tools.get_tools())
else:
litellm_tools = self._clean_tool_schema(tools)
response = self.router.completion(
model=self.model_name,
messages=chat_messages,
tools=litellm_tools,
_skip_mcp_handler=True,
**kwargs
)
return response
# 3. Instantiate your custom client wrapper
llm_client = LiteLLMToToyAIClient(router=litellm_router, model_name=MODEL_NAME)
from typing import Any
from toyaikit.tools import Tools
from toyaikit.chat.runners import OpenAIResponsesRunner
from evaluation.evaluation_utils import RAGWithUsage
agent_tools = Tools()
agent_tools.add_tool(RAGWithUsage.search)
instructions = """
You're a First Aid QA Assistant. Answer person at accident site questions based on
the FAQ search results. Use the search tool before answering.
""".strip()
runner = OpenAIResponsesRunner(
tools=agent_tools,
developer_prompt=instructions,
llm_client=llm_client, # type: ignore[arg-type]
)
rec = ground_truth[0]
result = runner.loop(prompt=rec["question"])
---------------------------------------------------------------------------
MaskedHTTPStatusError Traceback (most recent call last)
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/llms/custom_httpx/llm_http_handler.py:321, in BaseLLMHTTPHandler._make_common_sync_call(self, sync_httpx_client, provider_config, api_base, headers, data, timeout, litellm_params, logging_obj, stream, signed_json_body)
319 continue
320 else:
--> 321 raise self._handle_error(e=e, provider_config=provider_config)
322 except Exception as e:
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/llms/custom_httpx/http_handler.py:1195, in HTTPHandler.post(self, url, data, json, params, headers, stream, timeout, files, content, logging_obj)
1194 except httpx.HTTPStatusError as e:
-> 1195 _raise_masked_sync_error(e, stream)
1196 except Exception as e:
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/llms/custom_httpx/http_handler.py:434, in _raise_masked_sync_error(e, stream)
433 _text = mask_sensitive_info(_safe_get_response_text(e.response))
--> 434 raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
MaskedHTTPStatusError: Client error '422 Unprocessable Entity' for url 'https://api.mistral.ai/v1/chat/completions'
For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422
During handling of the above exception, another exception occurred:
OpenAIError Traceback (most recent call last)
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/main.py:5481, in completion(model, messages, timeout, temperature, top_p, n, stream, stream_options, stop, max_completion_tokens, max_tokens, modalities, prediction, audio, presence_penalty, frequency_penalty, logit_bias, user, reasoning_effort, verbosity, response_format, seed, tools, tool_choice, logprobs, top_logprobs, parallel_tool_calls, web_search_options, include_server_side_tool_invocations, deployment_id, extra_headers, safety_identifier, service_tier, functions, function_call, base_url, api_version, api_key, model_list, thinking, shared_session, enable_json_schema_validation, **kwargs)
5480 elif custom_llm_provider == "mistral":
-> 5481 response = _complete_mistral(_dispatch_ctx)
5482 elif "replicate" in model or custom_llm_provider == "replicate" or model in litellm.replicate_models:
5483 # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN")
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/main.py:2443, in _complete_mistral(ctx)
2441 api_base = api_base or litellm.api_base or get_secret("MISTRAL_API_BASE") or "https://api.mistral.ai/v1"
-> 2443 return base_llm_http_handler.completion(
2444 model=model,
2445 messages=messages,
2446 api_base=api_base,
2447 custom_llm_provider=custom_llm_provider,
2448 model_response=model_response,
2449 encoding=_get_encoding(),
2450 logging_obj=logging,
2451 optional_params=optional_params,
2452 timeout=timeout,
2453 litellm_params=litellm_params,
2454 shared_session=shared_session,
2455 acompletion=acompletion,
2456 stream=stream,
2457 api_key=api_key,
2458 headers=headers,
2459 client=client,
2460 provider_config=provider_config,
2461 )
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/llms/custom_httpx/llm_http_handler.py:599, in BaseLLMHTTPHandler.completion(self, model, messages, api_base, custom_llm_provider, model_response, encoding, logging_obj, optional_params, timeout, litellm_params, acompletion, stream, fake_stream, api_key, headers, client, provider_config, shared_session)
597 sync_httpx_client = client
598
--> 599 response = self._make_common_sync_call(
600 sync_httpx_client=sync_httpx_client,
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/llms/custom_httpx/llm_http_handler.py:321, in BaseLLMHTTPHandler._make_common_sync_call(self, sync_httpx_client, provider_config, api_base, headers, data, timeout, litellm_params, logging_obj, stream, signed_json_body)
319 continue
320 else:
--> 321 raise self._handle_error(e=e, provider_config=provider_config)
322 except Exception as e:
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/llms/custom_httpx/llm_http_handler.py:5546, in BaseLLMHTTPHandler._handle_error(self, e, provider_config)
5544 )
5545
-> 5546 raise provider_config.get_error_class(
5547 error_message=error_text,
OpenAIError: {"detail":[{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchTool","type"],"msg":"Input should be <ConversationToolTypes.Websearch: 'web_search'>","input":"function","ctx":{"expected":"<ConversationToolTypes.Websearch: 'web_search'>"}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchPremiumTool","type"],"msg":"Input should be <ConversationToolTypes.WebsearchPremium: 'web_search_premium'>","input":"function","ctx":{"expected":"<ConversationToolTypes.WebsearchPremium: 'web_search_premium'>"}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchPremiumTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchPremiumTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchPremiumTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CodeInterpreterTool","type"],"msg":"Input should be <ConversationToolTypes.CodeInterpreter: 'code_interpreter'>","input":"function","ctx":{"expected":"<ConversationToolTypes.CodeInterpreter: 'code_interpreter'>"}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CodeInterpreterTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CodeInterpreterTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CodeInterpreterTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"ImageGenerationTool","type"],"msg":"Input should be <ConversationToolTypes.ImageGeneration: 'image_generation'>","input":"function","ctx":{"expected":"<ConversationToolTypes.ImageGeneration: 'image_generation'>"}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"ImageGenerationTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"ImageGenerationTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"ImageGenerationTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","type"],"msg":"Input should be <ConversationToolTypes.DocumentLibrary: 'document_library'>","input":"function","ctx":{"expected":"<ConversationToolTypes.DocumentLibrary: 'document_library'>"}},{"type":"missing","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","library_ids"],"msg":"Field required","input":{"type":"function","name":"search","description":"No description provided.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","type"],"msg":"Input should be <ConversationToolTypes.Connector: 'connector'>","input":"function","ctx":{"expected":"<ConversationToolTypes.Connector: 'connector'>"}},{"type":"missing","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","connector_id"],"msg":"Field required","input":{"type":"function","name":"search","description":"No description provided.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"missing","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"Tool","function"],"msg":"Field required","input":{"type":"function","name":"search","description":"No description provided.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"Tool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"Tool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"Tool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"string_type","loc":["body","tools","list[str]",0],"msg":"Input should be a valid string","input":{"type":"function","name":"search","description":"No description provided.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}}}]}
During handling of the above exception, another exception occurred:
BadRequestError Traceback (most recent call last)
Cell In[28], line 3
1 rec = ground_truth[0]
2
----> 3 result = runner.loop(prompt=rec["question"])
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/toyaikit/chat/runners.py:252, in OpenAIResponsesRunner.loop(self, prompt, previous_messages, callback, output_format)
249 total_output_tokens = 0
251 while True:
--> 252 response = self.llm_client.send_request(
253 chat_messages=chat_messages,
254 tools=self.tools,
255 output_format=output_format,
256 )
257 if callback:
258 callback.on_response(response)
Cell In[25], line 40, in LiteLLMToToyAIClient.send_request(self, chat_messages, tools, output_format, **kwargs)
36 litellm_tools = self._clean_tool_schema(tools.get_tools())
37 else:
38 litellm_tools = self._clean_tool_schema(tools)
39
---> 40 response = self.router.completion(
41 model=self.model_name,
42 messages=chat_messages,
43 tools=litellm_tools,
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:1612, in Router.completion(self, model, messages, **kwargs)
1608
1609 response = self.function_with_fallbacks(**kwargs)
1610 return response
1611 except Exception as e:
-> 1612 raise e
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:1612, in Router.completion(self, model, messages, **kwargs)
1608
1609 response = self.function_with_fallbacks(**kwargs)
1610 return response
1611 except Exception as e:
-> 1612 raise e
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:6605, in Router.function_with_fallbacks(self, *args, **kwargs)
6601 Sync wrapper for async_function_with_fallbacks
6602
6603 Wrapped to reduce code duplication and prevent bugs.
6604 """
-> 6605 return run_async_function(self.async_function_with_fallbacks, *args, **kwargs)
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/asyncify.py:110, in run_async_function(async_function, *args, **kwargs)
108 with ThreadPoolExecutor(max_workers=1) as executor:
109 future = executor.submit(run_in_new_loop)
--> 110 return future.result()
112 except RuntimeError:
113 # No running event loop, we can safely run in this thread
114 return run_in_new_loop()
File ~/anaconda3/lib/python3.12/concurrent/futures/_base.py:456, in Future.result(self, timeout)
454 raise CancelledError()
455 elif self._state == FINISHED:
--> 456 return self.__get_result()
457 else:
458 raise TimeoutError()
File ~/anaconda3/lib/python3.12/concurrent/futures/_base.py:401, in Future.__get_result(self)
399 if self._exception:
400 try:
--> 401 raise self._exception
402 finally:
403 # Break a reference cycle with the exception in self._exception
404 self = None
File ~/anaconda3/lib/python3.12/concurrent/futures/thread.py:58, in _WorkItem.run(self)
55 return
57 try:
---> 58 result = self.fn(*self.args, **self.kwargs)
59 except BaseException as exc:
60 self.future.set_exception(exc)
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/asyncify.py:98, in run_async_function.<locals>.run_in_new_loop()
96 try:
97 asyncio.set_event_loop(new_loop)
---> 98 return new_loop.run_until_complete(async_function(*args, **kwargs))
99 finally:
100 new_loop.close()
File ~/anaconda3/lib/python3.12/asyncio/base_events.py:687, in BaseEventLoop.run_until_complete(self, future)
684 if not future.done():
685 raise RuntimeError('Event loop stopped before Future completed.')
--> 687 return future.result()
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:6257, in Router.async_function_with_fallbacks(self, *args, **kwargs)
6253 attempted_fallbacks=0,
6254 )
6255 return response
6256 except Exception as e:
-> 6257 return await self.async_function_with_fallbacks_common_utils(
6258 e,
6259 disable_fallbacks,
6260 fallbacks,
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:6219, in Router.async_function_with_fallbacks_common_utils(self, e, disable_fallbacks, fallbacks, context_window_fallbacks, content_policy_fallbacks, model_group, args, kwargs, include_fallback_errors)
6215 original_exception.message += ( # type: ignore
6216 "\nError doing the fallback: {}".format(fallback_failure_exception_str)
6217 )
6218
-> 6219 raise original_exception
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:6257, in Router.async_function_with_fallbacks(self, *args, **kwargs)
6253 attempted_fallbacks=0,
6254 )
6255 return response
6256 except Exception as e:
-> 6257 return await self.async_function_with_fallbacks_common_utils(
6258 e,
6259 disable_fallbacks,
6260 fallbacks,
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:6392, in Router.async_function_with_retries(self, *args, **kwargs)
6388
6389 # raises an exception if this error should not be retries
6390 # Skip this check if retry policy applies (retry policy takes precedence)
6391 if not _retry_policy_applies:
-> 6392 self.should_retry_this_error(
6393 error=e,
6394 healthy_deployments=_healthy_deployments,
6395 all_deployments=_all_deployments,
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:6571, in Router.should_retry_this_error(self, error, healthy_deployments, all_deployments, context_window_fallbacks, content_policy_fallbacks, regular_fallbacks)
6567 status_code = getattr(error, "status_code", None)
6568 if status_code is not None and not litellm._should_retry(status_code):
6569 # 401/403 are special cases - allow retry if multiple deployments exist (handled below)
6570 if status_code not in (401, 403):
-> 6571 raise error
6572
6573 if isinstance(error, litellm.NotFoundError):
6574 raise error
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:6392, in Router.async_function_with_retries(self, *args, **kwargs)
6388
6389 # raises an exception if this error should not be retries
6390 # Skip this check if retry policy applies (retry policy takes precedence)
6391 if not _retry_policy_applies:
-> 6392 self.should_retry_this_error(
6393 error=e,
6394 healthy_deployments=_healthy_deployments,
6395 all_deployments=_all_deployments,
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:6501, in Router.make_call(self, original_function, *args, **kwargs)
6497 """
6498 Handler for making a call to the .completion()/.embeddings()/etc. functions.
6499 """
6500 model_group = kwargs.get("model")
-> 6501 response = original_function(*args, **kwargs)
6502 if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
6503 response = await response
6504 ## PROCESS RESPONSE HEADERS
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:1706, in Router._completion(self, model, messages, **kwargs)
1702 # Set per-deployment num_retries on exception for retry logic
1703 if deployment is not None:
1704 self._set_deployment_num_retries_on_exception(e, deployment)
1705 self._set_failed_deployment_id_on_exception(e, deployment)
-> 1706 raise e
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/router.py:1706, in Router._completion(self, model, messages, **kwargs)
1702 # Set per-deployment num_retries on exception for retry logic
1703 if deployment is not None:
1704 self._set_deployment_num_retries_on_exception(e, deployment)
1705 self._set_failed_deployment_id_on_exception(e, deployment)
-> 1706 raise e
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/utils.py:1577, in client.<locals>.wrapper(*args, **kwargs)
1573 if logging_obj:
1574 logging_obj.failure_handler(
1575 e, traceback_exception, start_time, end_time
1576 ) # DO NOT MAKE THREADED - router retry fallback relies on this!
-> 1577 raise e
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/utils.py:1425, in client.<locals>.wrapper(*args, **kwargs)
1423 print_verbose(f"Error while checking max token limit: {str(e)}")
1424 # MODEL CALL
-> 1425 result = original_function(*args, **kwargs)
1426 end_time = datetime.datetime.now()
1427 if _is_streaming_request(
1428 kwargs=kwargs,
1429 call_type=call_type,
1430 ):
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/main.py:5607, in completion(model, messages, timeout, temperature, top_p, n, stream, stream_options, stop, max_completion_tokens, max_tokens, modalities, prediction, audio, presence_penalty, frequency_penalty, logit_bias, user, reasoning_effort, verbosity, response_format, seed, tools, tool_choice, logprobs, top_logprobs, parallel_tool_calls, web_search_options, include_server_side_tool_invocations, deployment_id, extra_headers, safety_identifier, service_tier, functions, function_call, base_url, api_version, api_key, model_list, thinking, shared_session, enable_json_schema_validation, **kwargs)
5604 return response
5605 except Exception as e:
5606 ## Map to OpenAI Exception
-> 5607 raise exception_type(
5608 model=model,
5609 custom_llm_provider=custom_llm_provider,
5610 original_exception=e,
5611 completion_kwargs=args,
5612 extra_kwargs=kwargs,
5613 )
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py:2499, in exception_type(model, original_exception, custom_llm_provider, completion_kwargs, extra_kwargs)
2497 if isinstance(e, error_type):
2498 setattr(e, "litellm_response_headers", litellm_response_headers)
-> 2499 raise e # it's already mapped
2500 raised_exc = APIConnectionError(
2501 message="{}\n{}".format(
2502 original_exception,
(...) 2506 model="",
2507 )
2508 setattr(raised_exc, "litellm_response_headers", litellm_response_headers)
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py:2265, in exception_type(model, original_exception, custom_llm_provider, completion_kwargs, extra_kwargs)
2252 extract_and_raise_litellm_exception(
2253 response=getattr(original_exception, "response", None),
2254 error_str=error_str,
2255 model=model,
2256 custom_llm_provider=custom_llm_provider,
2257 )
2258 if (
2259 custom_llm_provider == "openai"
2260 or custom_llm_provider == "text-completion-openai"
(...) 2263 or custom_llm_provider == "mistral"
2264 ):
-> 2265 _map_openai_exception(
2266 model=model,
2267 original_exception=mappable_exception,
2268 custom_llm_provider=custom_llm_provider,
2269 error_str=error_str,
2270 exception_type=exception_type,
2271 exception_provider=exception_provider,
2272 extra_information=extra_information,
2273 )
2274 elif custom_llm_provider == "anthropic" or custom_llm_provider == "anthropic_text": # one of the anthropics
2275 _map_anthropic_exception(
2276 model=model,
2277 original_exception=mappable_exception,
(...) 2282 extra_information=extra_information,
2283 )
File ~/AI-Practice/DataTalks/llm-zoomcamp-2026/first-aid-qa-rag-assistant/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py:422, in _map_openai_exception(model, original_exception, custom_llm_provider, error_str, exception_type, exception_provider, extra_information)
415 raise Timeout(
416 message=f"Timeout Error: {exception_provider} - {message}",
417 model=model,
418 llm_provider=custom_llm_provider,
419 litellm_debug_info=extra_information,
420 )
421 elif original_exception.status_code == 422:
--> 422 raise BadRequestError(
423 message=f"{exception_provider} - {message}",
424 model=model,
425 llm_provider=custom_llm_provider,
426 response=getattr(original_exception, "response", None),
427 litellm_debug_info=extra_information,
428 body=getattr(original_exception, "body", None),
429 )
430 elif original_exception.status_code == 429:
431 raise RateLimitError(
432 message=f"RateLimitError: {exception_provider} - {message}",
433 model=model,
(...) 436 litellm_debug_info=extra_information,
437 )
BadRequestError: litellm.BadRequestError: MistralException - {"detail":[{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchTool","type"],"msg":"Input should be <ConversationToolTypes.Websearch: 'web_search'>","input":"function","ctx":{"expected":"<ConversationToolTypes.Websearch: 'web_search'>"}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchPremiumTool","type"],"msg":"Input should be <ConversationToolTypes.WebsearchPremium: 'web_search_premium'>","input":"function","ctx":{"expected":"<ConversationToolTypes.WebsearchPremium: 'web_search_premium'>"}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchPremiumTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchPremiumTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"WebSearchPremiumTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CodeInterpreterTool","type"],"msg":"Input should be <ConversationToolTypes.CodeInterpreter: 'code_interpreter'>","input":"function","ctx":{"expected":"<ConversationToolTypes.CodeInterpreter: 'code_interpreter'>"}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CodeInterpreterTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CodeInterpreterTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CodeInterpreterTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"ImageGenerationTool","type"],"msg":"Input should be <ConversationToolTypes.ImageGeneration: 'image_generation'>","input":"function","ctx":{"expected":"<ConversationToolTypes.ImageGeneration: 'image_generation'>"}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"ImageGenerationTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"ImageGenerationTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"ImageGenerationTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","type"],"msg":"Input should be <ConversationToolTypes.DocumentLibrary: 'document_library'>","input":"function","ctx":{"expected":"<ConversationToolTypes.DocumentLibrary: 'document_library'>"}},{"type":"missing","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","library_ids"],"msg":"Field required","input":{"type":"function","name":"search","description":"No description provided.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"DocumentLibraryTool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"literal_error","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","type"],"msg":"Input should be <ConversationToolTypes.Connector: 'connector'>","input":"function","ctx":{"expected":"<ConversationToolTypes.Connector: 'connector'>"}},{"type":"missing","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","connector_id"],"msg":"Field required","input":{"type":"function","name":"search","description":"No description provided.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"CustomConnector","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"missing","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"Tool","function"],"msg":"Field required","input":{"type":"function","name":"search","description":"No description provided.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}}},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"Tool","name"],"msg":"Extra inputs are not permitted","input":"search"},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"Tool","description"],"msg":"Extra inputs are not permitted","input":"No description provided."},{"type":"extra_forbidden","loc":["body","tools","list[union[WebSearchTool,WebSearchPremiumTool,CodeInterpreterTool,ImageGenerationTool,DocumentLibraryTool,CustomConnector,Tool]]",0,"Tool","parameters"],"msg":"Extra inputs are not permitted","input":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}},{"type":"string_type","loc":["body","tools","list[str]",0],"msg":"Input should be a valid string","input":{"type":"function","name":"search","description":"No description provided.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"query parameter"},"num_results":{"type":"string","description":"num_results parameter"}},"required":["query"],"additionalProperties":false}}}]}. Received Model Group=ministral-3b-2512
Available Model Group Fallbacks=None