codescraftman / ethanicbot_updater / utils.py
utils.py
Raw
# ethanicbot_updater/utils.py

from langchain_core.documents import Document
from langchain_community.embeddings import OpenAIEmbeddings
from langchain_community.vectorstores import FAISS


from django.conf import settings
import os

from machinelearning.models import RegressionModel, ClassificationModel, NLPModel, UnsupervisedModel

EMBEDDINGS = OpenAIEmbeddings()
INDEX_PATH = os.path.join(settings.BASE_DIR, "ethanicbot", "faiss_index")

def fetch_documents():
    docs = []

    def make_doc(model, title):
        for obj in model.objects.all():
            content = f"{title}: {obj.name}\n{obj.description}"
            docs.append(Document(page_content=content, metadata={"model": title}))

    make_doc(RegressionModel, "Regression")
    make_doc(ClassificationModel, "Classification")
    make_doc(NLPModel, "NLP")
    make_doc(UnsupervisedModel, "Unsupervised")

    return docs

def update_vectorstore():
    documents = fetch_documents()
    db = FAISS.from_documents(documents, EMBEDDINGS)
    db.save_local(INDEX_PATH)
    return f"Vector store updated with {len(documents)} documents."