corpus.py

import json
from encoder import TextEncoder  

encoder = TextEncoder(vocab_path="vocab.json", by_char=False)

with open("AgGPT10m.agmodel", "r") as f:
    encoded = json.load(f)

decoded_text = encoder.decode(encoded)

corpus = decoded_text