memory/ingest.py

from memory.long_term import LongTermMemory
from memory.embedder import Embedder

class DatasetIngestor:
    def __init__(self):
        self.memory = LongTermMemory()
        self.embedder = Embedder()

    def ingest(self, samples, limit=None):
        for i, sample in enumerate(samples):
            if limit and i >= limit:
                break

            text = sample["text"]
            meta = sample.get("meta", {})

            embedding = self.embedder.encode(text)
            self.memory.add(text, embedding, meta)

        print(f"[INGEST] {i + 1} samples added to long-term memory")