import json
from pathlib import Path
import numpy as np
from sqlitesearch import TextSearchIndex, VectorSearchIndex
from tqdm.auto import tqdm
# Project root is two levels up from src/ingest.py (src/ → project root)
_ROOT = Path(__file__).parent.parent
def load_faq_data():
documents = []
with open(_ROOT / "data" / "firstaidqa_v1-first-half.json", "r") as file:
documents = json.load(file)
return documents
def build_index(documents):
db_path = _ROOT / "sqlite" / "text" / "text-search.db"
db_path.parent.mkdir(parents=True, exist_ok=True)
index = TextSearchIndex(
text_fields=["question", "answer"],
db_path=str(db_path),
)
if index:
return index
else:
index.fit(documents)
return index
def build_vector_index(embedder, documents):
db_path = _ROOT / "sqlite" / "vector" / "vector-search.db"
db_path.parent.mkdir(parents=True, exist_ok=True)
index = VectorSearchIndex(
mode="hnsw",
db_path=str(db_path),
)
if index:
return index
else:
faq_texts = [f"Q: {item['question']} A: {item['answer']}" for item in documents]
# Track indexing speed
print("Generating Embeddings in Batches...")
batch_size = 50
db_vectors = []
for i in tqdm(range(0, len(faq_texts), batch_size)):
batch = faq_texts[i : i + batch_size]
batch_vectors = embedder.encode_batch(batch)
db_vectors.extend(batch_vectors)
db_vectors = np.array(db_vectors)
index.fit(db_vectors, documents)
return index