import os
from django.apps import apps
from langchain.schema import Document
from langchain_community.vectorstores import FAISS
from langchain_community.embeddings import HuggingFaceEmbeddings
# Use a lightweight and effective embedding model
EMBEDDING_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
VECTORSTORE_PATH = "ethanicbot/langchain/faiss_index"
def load_all_model_data():
"""
Load data from all models in the Django project and create a list of Document objects.
Each document contains the model's field data and associated metadata.
"""
docs = []
for model in apps.get_models():
for instance in model.objects.all():
model_name = model.__name__
app_label = model._meta.app_label
metadata = {
"model": model_name,
"app": app_label,
"id": instance.pk,
}
fields = []
for field in model._meta.fields:
try:
value = getattr(instance, field.name)
fields.append(f"{field.name}: {value}")
except Exception as e:
print(f"⚠️ Error getting field {field.name} for instance {instance.pk}: {e}")
continue
content = f"Model: {model_name} | App: {app_label}\n" + "\n".join(fields)
docs.append(Document(page_content=content, metadata=metadata))
return docs
def create_vectorstore():
"""
Create a FAISS vector store from all model data, and save it to the specified path.
"""
try:
docs = load_all_model_data()
if not docs:
raise ValueError("No documents to index. Ensure models have data.")
embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL_NAME)
vectorstore = FAISS.from_documents(docs, embeddings)
vectorstore.save_local(VECTORSTORE_PATH)
print("✅ Vectorstore created and saved successfully.")
except Exception as e:
print(f"❌ Error creating vectorstore: {e}")
def load_vectorstore():
"""
Load the FAISS vector store from the saved location.
"""
try:
embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL_NAME)
vectorstore = FAISS.load_local(VECTORSTORE_PATH, embeddings, allow_dangerous_deserialization=True)
print("✅ Vectorstore loaded successfully.")
return vectorstore
except Exception as e:
print(f"❌ Error loading vectorstore: {e}")
return None