import numpy as np import onnxruntime as ort from tokenizers import Tokenizer from pathlib import Path # Default model location relative to this file (src/ → project root → models/) _DEFAULT_MODEL_PATH = Path(__file__).parent.parent / "models" / "Xenova" / "all-MiniLM-L6-v2" class Embedder: def __init__(self, path=None): path = Path(path) if path is not None else _DEFAULT_MODEL_PATH if not (path / "tokenizer.json").exists() or not (path / "model.onnx").exists(): import sys root_dir = Path(__file__).parent.parent if str(root_dir / "scripts") not in sys.path: sys.path.append(str(root_dir / "scripts")) from download import download download("Xenova/all-MiniLM-L6-v2", dest=root_dir / "models") self.tokenizer = Tokenizer.from_file(str(path / "tokenizer.json")) self.session = ort.InferenceSession( str(path / "model.onnx"), providers=["CPUExecutionProvider"] ) self.input_names = {inp.name for inp in self.session.get_inputs()} def encode(self, text, normalize=True): return self.encode_batch([text], normalize=normalize)[0] def encode_batch(self, texts, normalize=True): self.tokenizer.enable_padding() encoded = self.tokenizer.encode_batch(texts) feed = {} if "input_ids" in self.input_names: feed["input_ids"] = np.array([e.ids for e in encoded], dtype=np.int64) if "attention_mask" in self.input_names: feed["attention_mask"] = np.array( [e.attention_mask for e in encoded], dtype=np.int64 ) if "token_type_ids" in self.input_names: feed["token_type_ids"] = np.array( [e.type_ids for e in encoded], dtype=np.int64 ) hidden = self.session.run(None, feed)[0] mask = feed["attention_mask"][..., None] pooled = (hidden * mask).sum(axis=1) / mask.sum(axis=1) if normalize: pooled = pooled / np.linalg.norm(pooled, axis=1, keepdims=True) return pooled