Update train.py

train.py

@@ -0,0 +1,42 @@
+from fastai.text.all import *
+from pathlib import Path
+import pandas as pd
+import tiktoken
+
+enc = tiktoken.get_encoding("o200k_base")
+
+def tokenizer(s): 
+    ids = enc.encode(s)
+    tokens_list = [enc.decode([i]) for i in ids]
+    return tokens_list
+
+def main():
+    path = Path('data/chat_data.txt')
+    text = path.read_text(encoding='utf-8')
+
+    dls = TextDataLoaders.from_df(
+        pd.DataFrame({'text':[text]}), 
+        text_col='text',
+        is_lm=True,
+        tok_func=tokenizer,
+        seq_len=256
+    )
+
+    learn = language_model_learner(
+        dls, 
+        arch=AWD_LSTM,
+        metrics=[accuracy, Perplexity()],
+        pretrained=False
+    ).to_fp16()
+
+    learn.fit_one_cycle(5000, 1e-3)
+
+    # Export full learner (architecture + weights + vocab)
+    learn.export('model.pkl')
+
+    TEXT = """Hi!"""
+    generated = learn.predict(TEXT, 200, temperature=0.9)
+    print("\nGenerated text:\n", generated)
+
+if __name__ == "__main__":
+    main()