data.py

import json

with open('data.txt', 'r') as file:
    corpus = file.read()

lines = []
with open('data.jsonl', 'r') as file:
    for line in file:
        try:
            data = json.loads(line.strip())
            ai_response = data.get('text', '')
            url = data.get('url', '')
            lines.append(f"User: {url}\nAI: {ai_response}\n<|endoftext|>\n")
        except json.JSONDecodeError:
            pass

corpus += ''.join(lines)