from gitsource import GithubRepositoryDataReader

reader = GithubRepositoryDataReader(
    repo_owner="DataTalksClub",
    repo_name="llm-zoomcamp",
    commit_id="8c1834d",
    allowed_extensions={"md"},
    filename_filter=lambda path: "/lessons/" in path,
)

documents = [file.parse() for file in reader.read()]

from embedder import Embedder

embed = Embedder()

2026-07-01 18:36:15.271981698 [W:onnxruntime:Default, device_discovery.cc:133 GetPciBusId] Skipping pci_bus_id for PCI path at "/sys/devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A03:00/device:07/VMBUS:01/5620e0c7-8062-4dce-aeb7-520c7ef76171" because filename "5620e0c7-8062-4dce-aeb7-520c7ef76171" did not match expected pattern of [0-9a-f]+:[0-9a-f]+:[0-9a-f]+[.][0-9a-f]+

documents[0]

{'content': '# Introduction\n\nVideo: [Watch this lesson](https://www.youtube.com/watch?v=rQYyFxf1FWw&list=PL3MmuxUbc_hLZFNgSad56pDBKK8KO0XIv)\n\nIn this module, we\'ll build a working Retrieval-Augmented\nGeneration (RAG) system from scratch, step by step.\n\nWe write everything in plain Python. We build a small search index by\nhand and call the LLM ourselves. I want you to see every piece first.\nThat way you know what a framework does for you before you reach for\none.\n\nPlaces where you can find me:\n\n- [My substack](https://alexeyondata.substack.com/)\n- [LinkedIn](https://www.linkedin.com/in/agrigorev/)\n- [X](https://x.com/Al_Grigor)\n\n## LLMs\n\nAn LLM (Large Language Model) is a neural network trained on massive\namounts of text. Given a prompt, it generates a continuation - a\nplausible next piece of text.\n\nThink of your phone. When you type "how are" in WhatsApp, it suggests\n"you" as the next word. "How are you" is the most common continuation.\nYour phone uses a simple language model for that. It predicts the next\nword based on what you typed so far.\n\nA large language model does the same thing, but at a much larger scale.\nIt has billions of parameters and is trained on most of the text on the\ninternet. When it predicts the next word, it feels like you\'re talking\nto an intelligent being. It understands what you ask and gives\nmeaningful answers.\n\nIn this course, we treat LLMs as black boxes. We won\'t look inside or\ncover the theory, and we won\'t host a model ourselves. We use an LLM\nprovider and call it over an API. For us, an LLM is a box: text goes in,\ntext comes out.\n\nBut LLMs have limitations:\n\n- Knowledge cutoff: they only know what was in their training data.\n  If you ask about something that happened after training, they won\'t\n  know - or worse, they\'ll make something up.\n- No access to your data: they can\'t see your documents, databases,\n  or internal systems unless you provide that information.\n- Hallucinations: they sometimes produce confident-sounding answers\n  that are simply wrong.\n\n## The project\n\nRAG solves these problems by giving the LLM relevant documents at\nquestion time. We don\'t hope the model memorized the answer. We\nretrieve the right information and hand it to the LLM, and the model\ngenerates a grounded response. This lets us inject knowledge the model\nnever saw during training. That\'s why RAG is still the most common way\npeople use LLMs in the industry.\n\nTo make this concrete, we build a FAQ agent for our course. A student\nasks something like "when does the course start?" and the agent answers\nfrom the FAQ data we prepared.\n\nThis module has two parts.\n\nIn Part 1 (the next 9 lessons) we will:\n\n- Understand what RAG is and how it works\n- Build a search engine over a real FAQ dataset\n- Write a prompt that combines the user\'s question with search results\n- Wire it all together into a working RAG pipeline\n- Split ingestion and query into separate processes\n\nIn Part 2, we make the pipeline agentic. The LLM decides when and\nwhat to search, instead of running the same fixed flow every time.\n\nThe final code from this module is available in the\n[code/](../code/) directory.\n\n[← Back to module](../) | [Environment →](02-environment.md)',
 'filename': '01-agentic-rag/lessons/01-intro.md'}

first_3_pages = [
    '01-agentic-rag/lessons/01-intro.md',
    '01-agentic-rag/lessons/02-environment.md',
    '01-agentic-rag/lessons/03-rag.md'
]
lst_first_3_pages = list(filter(lambda x: x['filename'] in first_3_pages, documents))
lst_first_3_pages

[{'content': '# Introduction\n\nVideo: [Watch this lesson](https://www.youtube.com/watch?v=rQYyFxf1FWw&list=PL3MmuxUbc_hLZFNgSad56pDBKK8KO0XIv)\n\nIn this module, we\'ll build a working Retrieval-Augmented\nGeneration (RAG) system from scratch, step by step.\n\nWe write everything in plain Python. We build a small search index by\nhand and call the LLM ourselves. I want you to see every piece first.\nThat way you know what a framework does for you before you reach for\none.\n\nPlaces where you can find me:\n\n- [My substack](https://alexeyondata.substack.com/)\n- [LinkedIn](https://www.linkedin.com/in/agrigorev/)\n- [X](https://x.com/Al_Grigor)\n\n## LLMs\n\nAn LLM (Large Language Model) is a neural network trained on massive\namounts of text. Given a prompt, it generates a continuation - a\nplausible next piece of text.\n\nThink of your phone. When you type "how are" in WhatsApp, it suggests\n"you" as the next word. "How are you" is the most common continuation.\nYour phone uses a simple language model for that. It predicts the next\nword based on what you typed so far.\n\nA large language model does the same thing, but at a much larger scale.\nIt has billions of parameters and is trained on most of the text on the\ninternet. When it predicts the next word, it feels like you\'re talking\nto an intelligent being. It understands what you ask and gives\nmeaningful answers.\n\nIn this course, we treat LLMs as black boxes. We won\'t look inside or\ncover the theory, and we won\'t host a model ourselves. We use an LLM\nprovider and call it over an API. For us, an LLM is a box: text goes in,\ntext comes out.\n\nBut LLMs have limitations:\n\n- Knowledge cutoff: they only know what was in their training data.\n  If you ask about something that happened after training, they won\'t\n  know - or worse, they\'ll make something up.\n- No access to your data: they can\'t see your documents, databases,\n  or internal systems unless you provide that information.\n- Hallucinations: they sometimes produce confident-sounding answers\n  that are simply wrong.\n\n## The project\n\nRAG solves these problems by giving the LLM relevant documents at\nquestion time. We don\'t hope the model memorized the answer. We\nretrieve the right information and hand it to the LLM, and the model\ngenerates a grounded response. This lets us inject knowledge the model\nnever saw during training. That\'s why RAG is still the most common way\npeople use LLMs in the industry.\n\nTo make this concrete, we build a FAQ agent for our course. A student\nasks something like "when does the course start?" and the agent answers\nfrom the FAQ data we prepared.\n\nThis module has two parts.\n\nIn Part 1 (the next 9 lessons) we will:\n\n- Understand what RAG is and how it works\n- Build a search engine over a real FAQ dataset\n- Write a prompt that combines the user\'s question with search results\n- Wire it all together into a working RAG pipeline\n- Split ingestion and query into separate processes\n\nIn Part 2, we make the pipeline agentic. The LLM decides when and\nwhat to search, instead of running the same fixed flow every time.\n\nThe final code from this module is available in the\n[code/](../code/) directory.\n\n[← Back to module](../) | [Environment →](02-environment.md)',
  'filename': '01-agentic-rag/lessons/01-intro.md'},
 {'content': '# Environment\n\nVideo: [Watch this lesson](https://www.youtube.com/watch?v=3U4gBrmkZyM&list=PL3MmuxUbc_hLZFNgSad56pDBKK8KO0XIv)\n\nFor this module, all you need is Python with Jupyter.\n\n## Prerequisites\n\nYou need the following:\n\n- Python (3.14 or later)\n- An [OpenAI account](https://openai.com/) (or an OpenAI-compatible\n  provider like Groq, Gemini, or Ollama)\n- Basic familiarity with Python and the command line\n\n## Creating the project\n\nWe\'ll start from scratch - no cloning needed. You\'ll create the\nproject yourself, step by step.\n\nFirst, install uv. It\'s a Python package manager, and I switched all my\nprojects to it because it\'s fast and convenient. Once I started using\nit, I never wanted to go back.\n\nOn Mac or Linux:\n\n```bash\ncurl -LsSf https://astral.sh/uv/install.sh | sh\n```\n\nOn Windows:\n\n```powershell\npowershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"\n```\n\n(You can also use `pip install uv` if you prefer.)\n\nCreate an empty folder for the project and initialize it:\n\n```bash\nmkdir llm-zoomcamp-code\ncd llm-zoomcamp-code\nuv init\n```\n\nThis creates a `pyproject.toml` and a basic project structure.\n\nNow add the dependencies we\'ll need:\n\n```bash\nuv add requests minsearch openai jupyter python-dotenv\n```\n\nThis installs:\n\n- `requests` - to fetch the FAQ dataset from the internet\n- `minsearch` - a simple in-memory search engine for indexing and\n  searching text\n- `openai` - the OpenAI API client for calling the LLM\n- `jupyter` - the notebook environment where we\'ll write and run code\n- `python-dotenv` - to load API keys from a `.env` file\n\n## Setting up API keys\n\nWe need an API key to talk to the LLM. If you\'re using OpenAI, you\'ll\nneed to deposit some money first. The minimum is $5 (as of June 2026).\nThis lesson costs well under 10 cents to run, so that $5 goes a long\nway.\n\nI also recommend creating a separate OpenAI project for the course.\nThen you can open the usage page and see exactly how much you spent\nhere, apart from your other work.\n\nThe safest way to store the key is in a `.env` file that never gets\ncommitted to git.\n\nCreate a `.env` file in your project folder and put your API key in\nit:\n\n```bash\nOPENAI_API_KEY=sk-YOUR_KEY_HERE\n```\n\nNow add `.env` to `.gitignore` to make sure you never accidentally\ncommit your key:\n\n```bash\n.env\n```\n\nNever commit `.env` to git. Treat the API key like a password. If it\nleaks, someone else can run up charges on your account.\n\n## Starting Jupyter\n\nStart Jupyter:\n\n```bash\nuv run jupyter notebook\n```\n\nCreate a new notebook. Throughout the course, you\'ll copy code from\nthe section notes into notebook cells.\n\nCheck that the OpenAI client works:\n\n```python\nfrom dotenv import load_dotenv\nload_dotenv()\n\nfrom openai import OpenAI\nopenai_client = OpenAI()\n```\n\nIf you see an error, make sure the key in your `.env` file is\ncorrect.\n\nFor Groq or other OpenAI-compatible providers, add the key to\n`.env`:\n\n```bash\nGROQ_API_KEY=your_key_here\n```\n\nAnd configure the client:\n\n```python\nfrom openai import OpenAI\nimport os\n\nopenai_client = OpenAI(\n    api_key=os.getenv("GROQ_API_KEY"),\n    base_url="https://api.groq.com/openai/v1"\n)\n```\n\n## (Optional) Auto-loading .env with dirdotenv\n\nIf you don\'t want to call `load_dotenv()` in every notebook, use\n[dirdotenv](https://github.com/alexeygrigorev/dirdotenv).\n\nIt loads `.env` files automatically when you `cd` into a directory:\n\n```bash\nuv tool install dirdotenv\necho \'eval "$(dirdotenv hook bash)"\' >> ~/.bashrc\n```\n\nRestart your terminal, and now whenever you enter the project\ndirectory, the variables from `.env` are loaded automatically. No\n`load_dotenv()` needed.\n\n[← Introduction](01-intro.md) | [What is RAG →](03-rag.md)',
  'filename': '01-agentic-rag/lessons/02-environment.md'},
 {'content': '# RAG\n\nVideo: [Watch this lesson](https://www.youtube.com/watch?v=JktYwBIDErk&list=PL3MmuxUbc_hLZFNgSad56pDBKK8KO0XIv)\n\nWe run free Zoomcamp courses at DataTalks.Club on data engineering,\nmachine learning, MLOps, and other topics. Each course has its own\nFAQ document with common questions and answers.\n\nSome of these documents have over 300 questions. Students ask us\nthings in Slack like "Can I still join after the course started?" or\n"How do I get a certificate?" Finding those answers in the FAQ is\ntedious.\n\nWe want a bot that takes all this knowledge and answers student\nquestions in natural language.\n\nIn this module, we\'ll build that system. But first, let\'s see why we\ncan\'t send the question straight to an LLM and call it a day.\n\n## Plain LLMs lack our data\n\nFirst, let\'s define a function to talk to the LLM:\n\n```python\ndef llm(prompt):\n    response = openai_client.responses.create(\n        model="gpt-5.4-mini",\n        input=prompt\n    )\n    return response.output_text\n```\n\nThis is our black box - text goes in, text comes out.\n\nLet\'s test it:\n\n```python\nllm("Hey, what\'s up?")\n```\n\nIt replies with something. The LLM works.\n\nAsk it a course-specific\nquestion:\n\n```python\nquestion = "I just discovered the course. Can I join now?"\nanswer = llm(question)\nprint(answer)\n```\n\nThe LLM gives a generic answer. It might say "you can usually join" or\n"check the course website." It doesn\'t know about our specific Zoomcamp\ncourses, their enrollment policies, or their schedules. It tries to be\nhelpful, but has no idea about actual enrollment status or policies.\n\nThis is different from a question like "how do I cook salmon?" - the\nLLM knows the answer because cooking salmon is common knowledge. But\nour courses are not in the training data.\n\n## Adding context manually\n\nMore context can fix this. The FAQ website has questions and answers\nabout our courses.\n\nCopy some of that content into the prompt:\n\n```python\ncontext = """\nI just discovered the course. Can I still join?\nYes, but if you want to receive a certificate, you need to submit your project while we\'re still accepting submissions.\n\nCourse: I have registered for the LLM Zoomcamp. When can I expect to receive the confirmation email?\nYou don\'t need it. You\'re accepted. You can also just start learning and submitting homework (while the form is open) without registering. It is not checked against any registered list. Registration is just to gauge interest before the start date.\n\nWhat is the video/zoom link to the stream for the "Office Hours" or live/workshop sessions?\nThe zoom link is only published to instructors/presenters/TAs. Students participate via YouTube Live and submit questions to Slido.\n\nCloud alternatives with GPU\nCheck the quota and reset cycle carefully. Potential options include Google Colab, Kaggle, Databricks.\n"""\n```\n\nNotice the prompt doesn\'t end with `Answer:`. With older models like\nGPT-3 we added that to nudge the model into completing the sentence.\nModern models don\'t need the hint, so we drop it.\n\nBuild a prompt that includes both the question and the context:\n\n```python\nprompt = f"""\nYour task is to answer questions from the course participants\nbased on the provided context.\n\nUse the context to find relevant information and provide accurate\nanswers. If the answer is not found in the context,\nrespond with "I don\'t know."\n\nQuestion:\n{question}\n\nContext:\n{context}\n"""\n```\n\nInstead of sending the raw question to the LLM, we send this prompt:\n\n```python\nanswer = llm(prompt)\nprint(answer)\n```\n\nAfter that, the answer is correct: "Yes, you can still join. If you want to\nreceive a certificate, you need to submit your project while\nsubmissions are still open."\n\nThis is the answer we actually want to give to our students. What we\njust did is nothing but RAG.\n\n## Retrieval plus generation\n\nRAG stands for Retrieval-Augmented Generation. Generation is the LLM\nproducing text, and retrieval is search. We retrieve relevant documents\nfrom our knowledge base and use them to augment what the LLM generates.\nThat search step is what gives the LLM the context it needs to answer\ncorrectly.\n\nWhat we just did was naive. I knew in advance which FAQ entry held the\nanswer and pasted it in by hand. What we want instead is to perform\nsearch automatically. We take the student\'s question, find the most\nrelevant documents, and send those to the LLM.\n\nIn code, it looks like this:\n\n```python\ndef rag(question):\n    search_results = search(question)\n    user_prompt = build_prompt(question, search_results)\n    return llm(user_prompt)\n```\n\nThat\'s the entire architecture. It comes down to three components.\n\nThe pieces are search, the prompt, and the LLM:\n\n- search\n- prompt\n- LLM\n\n\n```mermaid\nflowchart TD\n    U([User])\n\n    APP[Application]\n\n    DB[(DB)]\n    DOCS[[D1 ... D5]]\n\n    PROMPT[Build Prompt<br/>Question + Context]\n\n    LLM[LLM]\n\n    ANSWER([Answer])\n\n    U -->|Question| APP\n\n    APP -->|Query| DB\n    DB -->|Retrieved Data| DOCS\n    DOCS --> APP\n\n    APP --> PROMPT\n    PROMPT --> LLM\n\n    LLM --> ANSWER\n    ANSWER --> U\n```\n\nThe LLM only sees the documents we hand it, so its answers are grounded\nin our data. If the right document is retrieved, the answer is good. If\nit\'s not, the LLM gets the wrong context and the answer is wrong. Your\nmodel is only as good as your retrieval, so search quality matters a\nlot for RAG.\n\nThe database and the LLM can be anything. In this course we use\nminsearch and then sqlitesearch for search, and OpenAI for the LLM. But\nyou can swap any component for another and see what works better.\n\nBecause each piece is independent, RAG stays flexible. To use Anthropic\ninstead of OpenAI, you swap the LLM call. To use Elasticsearch instead\nof minsearch, you swap the search call. Nothing else changes.\n\nIn the next section, we\'ll look at the dataset we\'ll use for our FAQ\nknowledge base.\n\n[← Environment](02-environment.md) | [The Course FAQ Dataset →](04-dataset.md)',
  'filename': '01-agentic-rag/lessons/03-rag.md'}]

from pydantic import BaseModel

class Questions(BaseModel):
    questions: list[str]

data_gen_instructions = """
You emulate a student who is taking our LLM course.
You are given one lesson page from the course.
Formulate 5 questions this student might ask that are answered by this page.

Rules:
- The page should contain the answer to each question.
- Make the questions complete and not too short.
- Use as few words as possible from the page; don't copy its phrasing.
- The questions should resemble how people actually ask things online:
  not too formal, not too short, not too long.
- Ask about the content of the lesson, not about its formatting or filename.
""".strip()

from dotenv import load_dotenv
from openai import OpenAI

load_dotenv()
openai_client = OpenAI()

from evaluation_utils import llm_structured
import json

total_input_tokens = 0
for page in lst_first_3_pages:
    user_prompt = json.dumps(page)
    messages = [
        {"role": "developer", "content": data_gen_instructions},
        {"role": "user", "content": user_prompt}
    ]

    result, usage = llm_structured(
        openai_client,
        data_gen_instructions,
        user_prompt,
        Questions
    )

    total_input_tokens += usage.input_tokens

int(total_input_tokens/len(lst_first_3_pages))

1353

import pandas as pd

df_ground_truth = pd.read_csv("data/ground-truth.csv")
ground_truth = df_ground_truth.to_dict(orient="records")

pd.DataFrame(ground_truth)

from gitsource import chunk_documents
chunks = chunk_documents(documents, size=2000, step=1000)
len(chunks)

295

from minsearch import Index

min_index = Index(
    keyword_fields = ['filename'],
    text_fields=['content']
)

min_index.fit(chunks)

<minsearch.minsearch.Index at 0x7f145fa34740>

def text_search(query, num_results=5):
    results = min_index.search(query, num_results=num_results)

    return results

query = q = ground_truth[0]["question"]
filename = ground_truth[0]["filename"]
results_text = text_search(query, num_results=5)
results_text[0]

{'start': 3000,
 'content': 'we drop it.\n\nBuild a prompt that includes both the question and the context:\n\n```python\nprompt = f"""\nYour task is to answer questions from the course participants\nbased on the provided context.\n\nUse the context to find relevant information and provide accurate\nanswers. If the answer is not found in the context,\nrespond with "I don\'t know."\n\nQuestion:\n{question}\n\nContext:\n{context}\n"""\n```\n\nInstead of sending the raw question to the LLM, we send this prompt:\n\n```python\nanswer = llm(prompt)\nprint(answer)\n```\n\nAfter that, the answer is correct: "Yes, you can still join. If you want to\nreceive a certificate, you need to submit your project while\nsubmissions are still open."\n\nThis is the answer we actually want to give to our students. What we\njust did is nothing but RAG.\n\n## Retrieval plus generation\n\nRAG stands for Retrieval-Augmented Generation. Generation is the LLM\nproducing text, and retrieval is search. We retrieve relevant documents\nfrom our knowledge base and use them to augment what the LLM generates.\nThat search step is what gives the LLM the context it needs to answer\ncorrectly.\n\nWhat we just did was naive. I knew in advance which FAQ entry held the\nanswer and pasted it in by hand. What we want instead is to perform\nsearch automatically. We take the student\'s question, find the most\nrelevant documents, and send those to the LLM.\n\nIn code, it looks like this:\n\n```python\ndef rag(question):\n    search_results = search(question)\n    user_prompt = build_prompt(question, search_results)\n    return llm(user_prompt)\n```\n\nThat\'s the entire architecture. It comes down to three components.\n\nThe pieces are search, the prompt, and the LLM:\n\n- search\n- prompt\n- LLM\n\n\n```mermaid\nflowchart TD\n    U([User])\n\n    APP[Application]\n\n    DB[(DB)]\n    DOCS[[D1 ... D5]]\n\n    PROMPT[Build Prompt<br/>Question + Context]\n\n    LLM[LLM]\n\n    ANSWER([Answer])\n\n    U -->|Question| APP\n\n    APP -->|Query| DB\n    DB -->|Retrieved Data| DOCS\n    DOCS --> APP\n\n    APP --> PROMPT\n    PROMPT -',
 'filename': '01-agentic-rag/lessons/03-rag.md'}

texts = [doc['content'] for doc in chunks]

texts[0]

'# Introduction\n\nVideo: [Watch this lesson](https://www.youtube.com/watch?v=rQYyFxf1FWw&list=PL3MmuxUbc_hLZFNgSad56pDBKK8KO0XIv)\n\nIn this module, we\'ll build a working Retrieval-Augmented\nGeneration (RAG) system from scratch, step by step.\n\nWe write everything in plain Python. We build a small search index by\nhand and call the LLM ourselves. I want you to see every piece first.\nThat way you know what a framework does for you before you reach for\none.\n\nPlaces where you can find me:\n\n- [My substack](https://alexeyondata.substack.com/)\n- [LinkedIn](https://www.linkedin.com/in/agrigorev/)\n- [X](https://x.com/Al_Grigor)\n\n## LLMs\n\nAn LLM (Large Language Model) is a neural network trained on massive\namounts of text. Given a prompt, it generates a continuation - a\nplausible next piece of text.\n\nThink of your phone. When you type "how are" in WhatsApp, it suggests\n"you" as the next word. "How are you" is the most common continuation.\nYour phone uses a simple language model for that. It predicts the next\nword based on what you typed so far.\n\nA large language model does the same thing, but at a much larger scale.\nIt has billions of parameters and is trained on most of the text on the\ninternet. When it predicts the next word, it feels like you\'re talking\nto an intelligent being. It understands what you ask and gives\nmeaningful answers.\n\nIn this course, we treat LLMs as black boxes. We won\'t look inside or\ncover the theory, and we won\'t host a model ourselves. We use an LLM\nprovider and call it over an API. For us, an LLM is a box: text goes in,\ntext comes out.\n\nBut LLMs have limitations:\n\n- Knowledge cutoff: they only know what was in their training data.\n  If you ask about something that happened after training, they won\'t\n  know - or worse, they\'ll make something up.\n- No access to your data: they can\'t see your documents, databases,\n  or internal systems unless you provide that information.\n- Hallucinations: they sometimes produce confident-sounding answers\n  that are simply '

from tqdm.auto import tqdm
import numpy as np

batch_size = 50
X = []

for i in tqdm(range(0, len(texts), batch_size)):
    batch = texts[i:i + batch_size]
    batch_vectors = embed.encode_batch(batch)
    X.extend(batch_vectors)

X = np.array(X)

  0%|          | 0/6 [00:00<?, ?it/s]

from minsearch import VectorSearch

min_vector_index = VectorSearch(
        keyword_fields=['filename']
    )

min_vector_index.fit(X, chunks)

<minsearch.vector.VectorSearch at 0x7f145fe6a4b0>

def vector_search(query, num_results=5):
    query_vector = embed.encode(query)
    results = min_vector_index.search(query_vector, num_results=num_results)

    return results

results_vector = vector_search(query, num_results=10)

results_vector[0]

{'start': 0,
 'content': '# Introduction\n\nVideo: [Watch this lesson](https://www.youtube.com/watch?v=rQYyFxf1FWw&list=PL3MmuxUbc_hLZFNgSad56pDBKK8KO0XIv)\n\nIn this module, we\'ll build a working Retrieval-Augmented\nGeneration (RAG) system from scratch, step by step.\n\nWe write everything in plain Python. We build a small search index by\nhand and call the LLM ourselves. I want you to see every piece first.\nThat way you know what a framework does for you before you reach for\none.\n\nPlaces where you can find me:\n\n- [My substack](https://alexeyondata.substack.com/)\n- [LinkedIn](https://www.linkedin.com/in/agrigorev/)\n- [X](https://x.com/Al_Grigor)\n\n## LLMs\n\nAn LLM (Large Language Model) is a neural network trained on massive\namounts of text. Given a prompt, it generates a continuation - a\nplausible next piece of text.\n\nThink of your phone. When you type "how are" in WhatsApp, it suggests\n"you" as the next word. "How are you" is the most common continuation.\nYour phone uses a simple language model for that. It predicts the next\nword based on what you typed so far.\n\nA large language model does the same thing, but at a much larger scale.\nIt has billions of parameters and is trained on most of the text on the\ninternet. When it predicts the next word, it feels like you\'re talking\nto an intelligent being. It understands what you ask and gives\nmeaningful answers.\n\nIn this course, we treat LLMs as black boxes. We won\'t look inside or\ncover the theory, and we won\'t host a model ourselves. We use an LLM\nprovider and call it over an API. For us, an LLM is a box: text goes in,\ntext comes out.\n\nBut LLMs have limitations:\n\n- Knowledge cutoff: they only know what was in their training data.\n  If you ask about something that happened after training, they won\'t\n  know - or worse, they\'ll make something up.\n- No access to your data: they can\'t see your documents, databases,\n  or internal systems unless you provide that information.\n- Hallucinations: they sometimes produce confident-sounding answers\n  that are simply ',
 'filename': '01-agentic-rag/lessons/01-intro.md'}

def rrf(result_lists, k=60, num_results=5):
    scores = {}
    docs = {}

    for results in result_lists:
        for rank, doc in enumerate(results):
            key = (doc["filename"], doc["start"])
            scores[key] = scores.get(key, 0) + 1 / (k + rank)
            docs[key] = doc

    ranked = sorted(scores, key=scores.get, reverse=True)
    return [docs[key] for key in ranked[:num_results]]

def hybrid_search(query, k=60):
    text_results = text_search(query, num_results=10)
    vector_results = vector_search(query, num_results=10)
    return rrf([text_results, vector_results], k=k)

results_hybrid = hybrid_search(query)
results_hybrid[0]

{'start': 0,
 'content': '# Introduction\n\nVideo: [Watch this lesson](https://www.youtube.com/watch?v=rQYyFxf1FWw&list=PL3MmuxUbc_hLZFNgSad56pDBKK8KO0XIv)\n\nIn this module, we\'ll build a working Retrieval-Augmented\nGeneration (RAG) system from scratch, step by step.\n\nWe write everything in plain Python. We build a small search index by\nhand and call the LLM ourselves. I want you to see every piece first.\nThat way you know what a framework does for you before you reach for\none.\n\nPlaces where you can find me:\n\n- [My substack](https://alexeyondata.substack.com/)\n- [LinkedIn](https://www.linkedin.com/in/agrigorev/)\n- [X](https://x.com/Al_Grigor)\n\n## LLMs\n\nAn LLM (Large Language Model) is a neural network trained on massive\namounts of text. Given a prompt, it generates a continuation - a\nplausible next piece of text.\n\nThink of your phone. When you type "how are" in WhatsApp, it suggests\n"you" as the next word. "How are you" is the most common continuation.\nYour phone uses a simple language model for that. It predicts the next\nword based on what you typed so far.\n\nA large language model does the same thing, but at a much larger scale.\nIt has billions of parameters and is trained on most of the text on the\ninternet. When it predicts the next word, it feels like you\'re talking\nto an intelligent being. It understands what you ask and gives\nmeaningful answers.\n\nIn this course, we treat LLMs as black boxes. We won\'t look inside or\ncover the theory, and we won\'t host a model ourselves. We use an LLM\nprovider and call it over an API. For us, an LLM is a box: text goes in,\ntext comes out.\n\nBut LLMs have limitations:\n\n- Knowledge cutoff: they only know what was in their training data.\n  If you ask about something that happened after training, they won\'t\n  know - or worse, they\'ll make something up.\n- No access to your data: they can\'t see your documents, databases,\n  or internal systems unless you provide that information.\n- Hallucinations: they sometimes produce confident-sounding answers\n  that are simply ',
 'filename': '01-agentic-rag/lessons/01-intro.md'}

def compute_relevance(q, search_function):
    file_name = q['filename']
    results = search_function(query=q['question'])

    relevance = []
    for d in results:
        relevance.append(int(d['filename'] == file_name))

    return relevance

q = ground_truth[0]
print(q['question'])
compute_relevance(q, text_search)

What exactly is a retrieval-augmented generation system, and why does it help with answers that the model wouldn't know on its own?





[0, 0, 0, 0, 1]

from tqdm.auto import tqdm

def compute_relevance_total(ground_truth, search_function):
    relevance_total = []

    for q in tqdm(ground_truth):
        relevance = compute_relevance(q, search_function)
        relevance_total.append(relevance)

    return relevance_total

relevance_text = compute_relevance_total(ground_truth, text_search)

  0%|          | 0/360 [00:00<?, ?it/s]

relevance_text[:15]

[[0, 0, 0, 0, 1],
 [1, 0, 1, 0, 0],
 [1, 1, 0, 0, 1],
 [1, 0, 1, 0, 0],
 [0, 0, 0, 0, 0],
 [0, 0, 0, 0, 0],
 [0, 1, 0, 0, 0],
 [0, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 1, 1, 0, 0],
 [1, 1, 0, 0, 0],
 [0, 1, 0, 0, 1],
 [0, 0, 0, 0, 0],
 [0, 1, 1, 0, 0],
 [0, 0, 0, 0, 0]]

def hit_rate(relevance):
    cnt = 0

    for line in relevance:
        if 1 in line:
            cnt = cnt + 1

    return cnt / len(relevance)

hit_rate(relevance_text)

0.7583333333333333

relevance_vector = compute_relevance_total(ground_truth, vector_search)

  0%|          | 0/360 [00:00<?, ?it/s]

def mrr(relevance):
    total_score = 0.0

    for line in relevance:
        rank = line.index(1) if 1 in line else -1
        if rank > -1:
            score = 1 / (rank + 1)
            total_score = total_score + score

    return total_score / len(relevance)

mrr(relevance_vector)

0.5486111111111112

def evaluate(ground_truth, search_function):
    relevance_total = compute_relevance_total(ground_truth, search_function)

    return {
        "hit_rate": hit_rate(relevance_total),
        "mrr": mrr(relevance_total),
    }

evaluate(ground_truth, text_search)

  0%|          | 0/360 [00:00<?, ?it/s]





{'hit_rate': 0.7583333333333333, 'mrr': 0.5942592592592594}

evaluate(ground_truth, vector_search)

  0%|          | 0/360 [00:00<?, ?it/s]





{'hit_rate': 0.725, 'mrr': 0.5486111111111112}

evaluate(ground_truth, hybrid_search)

  0%|          | 0/360 [00:00<?, ?it/s]





{'hit_rate': 0.8361111111111111, 'mrr': 0.637916666666667}

results_tune = []
for k in [1, 50, 100, 200]:
    print(f"Evaluating Hybrid Search k={k} ...")
    result = evaluate(
        ground_truth,
        lambda query, k=k: hybrid_search(
            query,
            k
        )
    )

    results_tune.append({
        "k": k,
        "hit_rate": result["hit_rate"],
        "mrr": result["mrr"],
    })

Evaluating Hybrid Search k=1 ...



  0%|          | 0/360 [00:00<?, ?it/s]


Evaluating Hybrid Search k=50 ...



  0%|          | 0/360 [00:00<?, ?it/s]


Evaluating Hybrid Search k=100 ...



  0%|          | 0/360 [00:00<?, ?it/s]


Evaluating Hybrid Search k=200 ...



  0%|          | 0/360 [00:00<?, ?it/s]

df_results = pd.DataFrame(results_tune)
df_results.sort_values("mrr", ascending=False).head(4)