import logging import time from decimal import Decimal from tqdm.auto import tqdm # RAGWithUsage lives in rag_usage (production code); re-exported here so that # existing notebooks that do `from evaluation.evaluation_utils import RAGWithUsage` # continue to work without changes. from rag.rag_usage import RAGWithUsage # noqa: F401 (re-export) from config import MODEL_NAME logger = logging.getLogger(__name__) def calc_price(usage): input_price_per_million = 0.1 output_price_per_million = 0.1 input_cost = (usage.prompt_tokens / 1_000_000) * input_price_per_million output_cost = (usage.completion_tokens / 1_000_000) * output_price_per_million total_cost = input_cost + output_cost return { "input_cost": round(Decimal(input_cost), 7), "output_cost": round(Decimal(output_cost), 7), "total_cost": round(Decimal(total_cost), 7), } def calc_total_price(usages): total_cost = 0.0 for usage in usages: cost = calc_price(usage) total_cost = total_cost + float(cost["total_cost"]) return total_cost def llm_structured( client, instructions, user_prompt, output_type, model=MODEL_NAME ): messages = [ {"role": "developer", "content": instructions}, {"role": "user", "content": user_prompt}, ] response = client.completion( model=model, messages=messages, response_format=output_type ) output_parsed = output_type.model_validate_json(response.choices[0].message.content) return output_parsed, response.usage def llm_structured_retry( client, instructions, user_prompt, output_type, model=MODEL_NAME, max_retries=3, ): for attempt in range(max_retries): try: return llm_structured( client, instructions, user_prompt, output_type, model=model, ) except Exception as exc: if attempt == max_retries - 1: raise logger.warning( "llm_structured_retry attempt %d/%d failed: %s", attempt + 1, max_retries, exc, ) time.sleep(2**attempt) def map_progress(pool, seq, f, llm_client, data_gen_instructions, Questions): results = [] with tqdm(total=len(seq)) as progress: futures = [] for el in seq: future = pool.submit(f, el, llm_client, data_gen_instructions, Questions) future.add_done_callback(lambda p: progress.update()) futures.append(future) for future in futures: result, usage = future.result() results.append(result) return results def map_progress_answers(pool, seq, f, doc_idx, assistant): results = [] with tqdm(total=len(seq)) as progress: futures = [] for el in seq: future = pool.submit(f, el, doc_idx, assistant) future.add_done_callback(lambda p: progress.update()) futures.append(future) for future in futures: result = future.result() results.append(result) return results def map_progress_judge( pool, seq, f, litellm_client, aqa_judge_instructions, aqa_judge_prompt, AnswerEvaluation ): results = [] usages = [] with tqdm(total=len(seq)) as progress: futures = [] for el in seq: future = pool.submit( f, el, litellm_client, aqa_judge_instructions, aqa_judge_prompt, AnswerEvaluation ) future.add_done_callback(lambda p: progress.update()) futures.append(future) for future in futures: result, usage = future.result() results.append(result) usages.append(usage) return results, usages