File size: 1,276 Bytes
824c85b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
import os
from MemoryEngine import MemoryEngine
class MemoryBatchParser:
def __init__(self, segment_length=2000):
self.engine = MemoryEngine()
self.segment_length = segment_length
def _chunk_text(self, full_text):
# Split into 2000-character chunks
return [full_text[i:i+self.segment_length] for i in range(0, len(full_text), self.segment_length)]
def ingest_txt(self, filepath, default_structure="General > Ingested", agent_id="batch_parser"):
if not os.path.exists(filepath):
print("File not found.")
return
with open(filepath, "r", encoding="utf-8") as f:
content = f.read()
segments = self._chunk_text(content)
for i, seg in enumerate(segments):
self.engine.insert_segment(
raw_text=seg.strip(),
concepts="",
terms="",
structure=default_structure,
datapoints="",
comparisons="",
applications="",
agent_id=agent_id
)
print(f"[{i+1}/{len(segments)}] Segment processed.")
# Example usage
if __name__ == "__main__":
parser = MemoryBatchParser()
parser.ingest_txt("example_upload.txt")
|