first-aid-rag-assistant / src / evaluation / evaluation_utils.py
evaluation_utils.py
Raw
import logging
import time
from decimal import Decimal

from tqdm.auto import tqdm

# RAGWithUsage lives in rag_usage (production code); re-exported here so that
# existing notebooks that do `from evaluation.evaluation_utils import RAGWithUsage`
# continue to work without changes.
from rag.rag_usage import RAGWithUsage  # noqa: F401  (re-export)
from config import MODEL_NAME

logger = logging.getLogger(__name__)


def calc_price(usage):
    input_price_per_million = 0.1
    output_price_per_million = 0.1

    input_cost = (usage.prompt_tokens / 1_000_000) * input_price_per_million
    output_cost = (usage.completion_tokens / 1_000_000) * output_price_per_million
    total_cost = input_cost + output_cost

    return {
        "input_cost": round(Decimal(input_cost), 7),
        "output_cost": round(Decimal(output_cost), 7),
        "total_cost": round(Decimal(total_cost), 7),
    }


def calc_total_price(usages):
    total_cost = 0.0

    for usage in usages:
        cost = calc_price(usage)
        total_cost = total_cost + float(cost["total_cost"])

    return total_cost


def llm_structured(
    client, instructions, user_prompt, output_type, model=MODEL_NAME
):

    messages = [
        {"role": "developer", "content": instructions},
        {"role": "user", "content": user_prompt},
    ]

    response = client.completion(
        model=model, messages=messages, response_format=output_type
    )

    output_parsed = output_type.model_validate_json(response.choices[0].message.content)

    return output_parsed, response.usage


def llm_structured_retry(
    client,
    instructions,
    user_prompt,
    output_type,
    model=MODEL_NAME,
    max_retries=3,
):
    for attempt in range(max_retries):
        try:
            return llm_structured(
                client,
                instructions,
                user_prompt,
                output_type,
                model=model,
            )
        except Exception as exc:
            if attempt == max_retries - 1:
                raise
            logger.warning(
                "llm_structured_retry attempt %d/%d failed: %s",
                attempt + 1,
                max_retries,
                exc,
            )
            time.sleep(2**attempt)


def map_progress(pool, seq, f, llm_client, data_gen_instructions, Questions):
    results = []

    with tqdm(total=len(seq)) as progress:
        futures = []

        for el in seq:
            future = pool.submit(f, el, llm_client, data_gen_instructions, Questions)
            future.add_done_callback(lambda p: progress.update())
            futures.append(future)

        for future in futures:
            result, usage = future.result()
            results.append(result)

    return results

def map_progress_answers(pool, seq, f, doc_idx, assistant):
    results = []

    with tqdm(total=len(seq)) as progress:
        futures = []

        for el in seq:
            future = pool.submit(f, el, doc_idx, assistant)
            future.add_done_callback(lambda p: progress.update())
            futures.append(future)

        for future in futures:
            result = future.result()
            results.append(result)

    return results

def map_progress_judge(
        pool,
        seq,
        f,
        litellm_client,
        aqa_judge_instructions,
        aqa_judge_prompt,
        AnswerEvaluation
    ):
    results = []
    usages = []

    with tqdm(total=len(seq)) as progress:
        futures = []

        for el in seq:
            future = pool.submit(
                f,
                el,
                litellm_client,
                aqa_judge_instructions,
                aqa_judge_prompt,
                AnswerEvaluation
            )
            future.add_done_callback(lambda p: progress.update())
            futures.append(future)

        for future in futures:
            result, usage = future.result()
            results.append(result)
            usages.append(usage)

    return results, usages