Upload 18 files

data/input.txt

@@ -0,0 +1,839 @@
+Q: What is the capital of France?
+A: Paris.
+
+Q: Who wrote Hamlet?
+A: William Shakespeare.
+
+Q: What is the largest planet?
+A: Jupiter.
+
+Q: What is the boiling point of water?
+A: 100 degrees Celsius.
+
+Q: What is the capital of Japan?
+A: Tokyo.
+
+Q: Who painted the Mona Lisa?
+A: Leonardo da Vinci.
+
+Q: What is the speed of light?
+A: 299,792 kilometers per second.
+
+Q: What is the capital of France?
+A: Paris.
+
+Q: What is the speed of light?
+A: 299,792 kilometers per second.
+
+Q: Who wrote Hamlet?
+A: William Shakespeare.
+
+Q: What is the largest planet?
+A: Jupiter.
+
+Q: What is the boiling point of water?
+A: 100 degrees Celsius.
+
+Q: What is the capital of Japan?
+A: Tokyo.
+
+Q: Who painted the Mona Lisa?
+A: Leonardo da Vinci.
+
+Q: What is the tallest mountain in the world?
+A: Mount Everest.
+
+Q: What is the chemical symbol for gold?
+A: Au.
+
+Q: What is the freezing point of water?
+A: 0 degrees Celsius.
+
+Q: Who invented the telephone?
+A: Alexander Graham Bell.
+
+Q: What is the largest ocean on Earth?
+A: The Pacific Ocean.
+
+Q: What is the process of photosynthesis?
+A: Photosynthesis is the process by which plants use sunlight to make food from carbon dioxide and water.
+
+Q: What is gravity?
+A: Gravity is the force that attracts objects toward the center of the Earth.
+
+Q: Who wrote Hamlet?
+A: William Shakespeare.
+
+Q: What is the capital of France?
+A: Paris.
+
+Q: What is the speed of light?
+A: 299,792 kilometers per second.
+
+Q: What is the largest ocean?
+A: The Pacific Ocean.
+
+Q: What is the boiling point of water?
+A: 100 degrees Celsius.
+
+Q: Who painted the Mona Lisa?
+A: Leonardo da Vinci.
+
+Q: What is the tallest mountain in the world?
+A: Mount Everest.
+
+Q: What is the chemical symbol for gold?
+A: Au.
+
+Q: What is the freezing point of water?
+A: 0 degrees Celsius.
+
+Q: Who invented the telephone?
+A: Alexander Graham Bell.
+
+Q: What is the largest planet?
+A: Jupiter.
+
+Q: What is gravity?
+A: Gravity is the force that attracts objects toward the center of the Earth.
+
+Q: What is the capital of France?
+A: Paris.
+
+Q: Who wrote Hamlet?
+A: William Shakespeare.
+
+Q: What is the speed of light?
+A: 299,792 kilometers per second.
+
+Q: What is the largest ocean?
+A: The Pacific Ocean.
+
+Q: What is the boiling point of water?
+A: 100 degrees Celsius.
+
+Q: Who painted the Mona Lisa?
+A: Leonardo da Vinci.
+
+Q: What is the tallest mountain in the world?
+A: Mount Everest.
+
+Q: What is the chemical symbol for gold?
+A: Au.
+
+Q: What is the freezing point of water?
+A: 0 degrees Celsius.
+
+Q: Who invented the telephone?
+A: Alexander Graham Bell.
+
+Q: What is the largest planet?
+A: Jupiter.
+
+Q: What is gravity?
+A: Gravity is the force that attracts objects toward the center of the Earth.
+
+Q: What is the main ingredient in bread?
+A: Flour.
+
+Q: What is the capital of Japan?
+A: Tokyo.
+
+Q: What is the largest continent?
+A: Asia.
+
+Q: What is the process of photosynthesis?
+A: Photosynthesis is the process by which plants use sunlight to make food from carbon dioxide and water.
+
+Q: What is the smallest prime number?
+A: 2.
+
+Q: What is the chemical symbol for water?
+A: H2O.
+
+Q: Who discovered penicillin?
+A: Alexander Fleming.
+
+Q: What is the tallest animal in the world?
+A: The giraffe.
+
+Q: What is the largest mammal?
+A: The blue whale.
+
+Q: What is the capital of Italy?
+A: Rome.
+
+Q: What is the largest desert in the world?
+A: The Sahara Desert.
+
+Q: What is the main gas in the air we breathe?
+A: Nitrogen.
+
+Q: What is the hardest natural substance?
+A: Diamond.
+
+Q: What is the largest country by area?
+A: Russia.
+
+Q: What is the longest river in the world?
+A: The Nile.
+
+Q: What is the main language spoken in Brazil?
+A: Portuguese.
+
+Q: What is the currency of the United States?
+A: The dollar.
+
+Q: What is the hottest planet in the solar system?
+A: Venus.
+
+Q: What is the largest island in the world?
+A: Greenland.
+
+Q: What is the capital of France?
+A: Paris.
+
+Q: Who wrote Hamlet?
+A: William Shakespeare.
+
+Q: What is the speed of light?
+A: 299,792 kilometers per second.
+
+Q: What is the largest ocean?
+A: The Pacific Ocean.
+
+Q: What is the boiling point of water?
+A: 100 degrees Celsius.
+
+Q: Who painted the Mona Lisa?
+A: Leonardo da Vinci.
+
+Q: What is the tallest mountain in the world?
+A: Mount Everest.
+
+Q: What is the chemical symbol for gold?
+A: Au.
+
+Q: What is the freezing point of water?
+A: 0 degrees Celsius.
+
+Q: Who invented the telephone?
+A: Alexander Graham Bell.
+
+Q: What is the largest planet?
+A: Jupiter.
+
+Q: What is gravity?
+A: Gravity is the force that attracts objects toward the center of the Earth.
+
+Q: What is the main ingredient in bread?
+A: Flour.
+
+Q: What is the capital of Japan?
+A: Tokyo.
+
+Q: What is the largest continent?
+A: Asia.
+
+Q: What is the process of photosynthesis?
+A: Photosynthesis is the process by which plants use sunlight to make food from carbon dioxide and water.
+
+Q: What is the smallest prime number?
+A: 2.
+
+Q: What is the chemical symbol for water?
+A: H2O.
+
+Q: Who discovered penicillin?
+A: Alexander Fleming.
+
+Q: What is the tallest animal in the world?
+A: The giraffe.
+
+Q: What is the largest mammal?
+A: The blue whale.
+
+Q: What is the capital of Italy?
+A: Rome.
+
+Q: What is the largest desert in the world?
+A: The Sahara Desert.
+
+Q: What is the main gas in the air we breathe?
+A: Nitrogen.
+
+Q: What is the hardest natural substance?
+A: Diamond.
+
+Q: What is the largest country by area?
+A: Russia.
+
+Q: What is the longest river in the world?
+A: The Nile.
+
+Q: What is the main language spoken in Brazil?
+A: Portuguese.
+
+Q: What is the currency of the United States?
+A: The dollar.
+
+Q: What is the hottest planet in the solar system?
+A: Venus.
+
+Q: What is the largest island in the world?
+A: Greenland.
+
+Q: What is the capital of Canada?
+A: Ottawa.
+
+Q: Who was the first President of the United States?
+A: George Washington.
+
+Q: What is the chemical symbol for iron?
+A: Fe.
+
+Q: What is the largest bone in the human body?
+A: The femur.
+
+Q: What is the main ingredient in glass?
+A: Silica.
+
+Q: What is the capital of Australia?
+A: Canberra.
+
+Q: What is the largest lake in the world?
+A: The Caspian Sea.
+
+Q: What is the main ingredient in chocolate?
+A: Cocoa.
+
+Q: What is the capital of Germany?
+A: Berlin.
+
+Q: What is the largest volcano in the world?
+A: Mauna Loa.
+
+Q: What is the main ingredient in pizza dough?
+A: Flour.
+
+Q: What is the capital of Russia?
+A: Moscow.
+
+Q: What is the largest bird in the world?
+A: The ostrich.
+
+Q: What is the main ingredient in sushi?
+A: Rice.
+
+Q: What is the capital of China?
+A: Beijing.
+
+Q: What is the largest fish in the world?
+A: The whale shark.
+
+Q: What is the main ingredient in guacamole?
+A: Avocado.
+
+Q: What is the capital of India?
+A: New Delhi.
+
+Q: What is the largest reptile in the world?
+A: The saltwater crocodile.
+
+Q: What is the main ingredient in hummus?
+A: Chickpeas.
+
+Q: What is the capital of Brazil?
+A: Brasília.
+
+Q: What is the largest flower in the world?
+A: Rafflesia arnoldii.
+
+Q: What is the main ingredient in bread?
+A: Flour.
+
+Q: What is the capital of France?
+A: Paris.
+
+Q: Who wrote Hamlet?
+A: William Shakespeare.
+
+Q: What is the speed of light?
+A: 299,792 kilometers per second.
+
+Q: What is the largest ocean?
+A: The Pacific Ocean.
+
+Q: What is the boiling point of water?
+A: 100 degrees Celsius.
+
+Q: Who painted the Mona Lisa?
+A: Leonardo da Vinci.
+
+Q: What is the tallest mountain in the world?
+A: Mount Everest.
+
+Q: What is the chemical symbol for gold?
+A: Au.
+
+Q: What is the freezing point of water?
+A: 0 degrees Celsius.
+
+Q: Who invented the telephone?
+A: Alexander Graham Bell.
+
+Q: What is the largest planet?
+A: Jupiter.
+
+Q: What is gravity?
+A: Gravity is the force that attracts objects toward the center of the Earth.
+
+Q: What is the main ingredient in bread?
+A: Flour.
+
+Q: What is the capital of Japan?
+A: Tokyo.
+
+Q: What is the largest continent?
+A: Asia.
+
+Q: What is the process of photosynthesis?
+A: Photosynthesis is the process by which plants use sunlight to make food from carbon dioxide and water.
+
+Q: What is the smallest prime number?
+A: 2.
+
+Q: What is the chemical symbol for water?
+A: H2O.
+
+Q: Who discovered penicillin?
+A: Alexander Fleming.
+
+Q: What is the tallest animal in the world?
+A: The giraffe.
+
+Q: What is the largest mammal?
+A: The blue whale.
+
+Q: What is the capital of Italy?
+A: Rome.
+
+Q: What is the largest desert in the world?
+A: The Sahara Desert.
+
+Q: What is the main gas in the air we breathe?
+A: Nitrogen.
+
+Q: What is the hardest natural substance?
+A: Diamond.
+
+Q: What is the largest country by area?
+A: Russia.
+
+Q: What is the longest river in the world?
+A: The Nile.
+
+Q: What is the main language spoken in Brazil?
+A: Portuguese.
+
+Q: What is the currency of the United States?
+A: The dollar.
+
+Q: What is the hottest planet in the solar system?
+A: Venus.
+
+Q: What is the largest island in the world?
+A: Greenland.
+
+Q: What is the capital of Canada?
+A: Ottawa.
+
+Q: Who was the first President of the United States?
+A: George Washington.
+
+Q: What is the chemical symbol for iron?
+A: Fe.
+
+Q: What is the largest bone in the human body?
+A: The femur.
+
+Q: What is the main ingredient in glass?
+A: Silica.
+
+Q: What is the capital of Australia?
+A: Canberra.
+
+Q: What is the largest lake in the world?
+A: The Caspian Sea.
+
+Q: What is the main ingredient in chocolate?
+A: Cocoa.
+
+Q: What is the capital of Germany?
+A: Berlin.
+
+Q: What is the largest volcano in the world?
+A: Mauna Loa.
+
+Q: What is the main ingredient in pizza dough?
+A: Flour.
+
+Q: What is the capital of Russia?
+A: Moscow.
+
+Q: What is the largest bird in the world?
+A: The ostrich.
+
+Q: What is the main ingredient in sushi?
+A: Rice.
+
+Q: What is the capital of China?
+A: Beijing.
+
+Q: What is the largest fish in the world?
+A: The whale shark.
+
+Q: What is the main ingredient in guacamole?
+A: Avocado.
+
+Q: What is the capital of India?
+A: New Delhi.
+
+Q: What is the largest reptile in the world?
+A: The saltwater crocodile.
+
+Q: What is the main ingredient in hummus?
+A: Chickpeas.
+
+Q: What is the capital of Brazil?
+A: Brasília.
+
+Q: What is the largest flower in the world?
+A: Rafflesia arnoldii.
+
+Q: What is the main ingredient in bread?
+A: Flour.
+
+Q: What is the capital of Spain?
+A: Madrid.
+
+Q: What is the largest peninsula in the world?
+A: The Arabian Peninsula.
+
+Q: What is the main ingredient in pasta?
+A: Wheat.
+
+Q: What is the capital of Egypt?
+A: Cairo.
+
+Q: What is the largest bay in the world?
+A: The Bay of Bengal.
+
+Q: What is the main ingredient in mayonnaise?
+A: Eggs.
+
+Q: What is the capital of South Africa?
+A: Pretoria.
+
+Q: What is the largest canyon in the world?
+A: The Grand Canyon.
+
+Q: What is the main ingredient in tofu?
+A: Soybeans.
+
+Q: What is the capital of Mexico?
+A: Mexico City.
+
+Q: What is the largest coral reef in the world?
+A: The Great Barrier Reef.
+
+Q: What is the main ingredient in ketchup?
+A: Tomatoes.
+
+Q: What is the capital of Argentina?
+A: Buenos Aires.
+
+Q: What is the largest waterfall in the world?
+A: Angel Falls.
+
+Q: What is the main ingredient in peanut butter?
+A: Peanuts.
+
+Q: What is the capital of Turkey?
+A: Ankara.
+
+Q: What is the largest stadium in the world?
+A: Rungrado 1st of May Stadium.
+
+Q: What is the main ingredient in miso soup?
+A: Miso paste.
+
+Q: What is the capital of Saudi Arabia?
+A: Riyadh.
+
+Q: What is the largest archipelago in the world?
+A: Indonesia.
+
+Q: What is the main ingredient in falafel?
+A: Chickpeas.
+
+Q: What is the capital of Thailand?
+A: Bangkok.
+
+Q: What is the largest cave in the world?
+A: Son Doong Cave.
+
+Q: What is the main ingredient in curry?
+A: Spices.
+
+Q: What is the capital of Sweden?
+A: Stockholm.
+
+Q: What is the largest glacier in the world?
+A: Lambert Glacier.
+
+Q: What is the main ingredient in risotto?
+A: Rice.
+
+Q: What is the capital of Norway?
+A: Oslo.
+
+Q: What is the largest hot desert in the world?
+A: The Sahara Desert.
+
+Q: What is the main ingredient in borscht?
+A: Beets.
+
+Q: What is the capital of Switzerland?
+A: Bern.
+
+Q: What is the largest freshwater lake in the world?
+A: Lake Superior.
+
+Q: What is the main ingredient in paella?
+A: Rice.
+
+Q: What is the capital of the Netherlands?
+A: Amsterdam.
+
+Q: What is the largest sea in the world?
+A: The Philippine Sea.
+
+Q: What is the main ingredient in pesto?
+A: Basil.
+
+Q: What is the capital of Greece?
+A: Athens.
+
+Q: What is the largest plateau in the world?
+A: The Tibetan Plateau.
+
+Q: What is the main ingredient in gazpacho?
+A: Tomatoes.
+
+Q: What is the capital of Portugal?
+A: Lisbon.
+
+Q: What is the largest delta in the world?
+A: The Ganges Delta.
+
+Q: What is the main ingredient in ramen?
+A: Noodles.
+
+Q: What is the capital of South Korea?
+A: Seoul.
+
+Q: What is the largest island country in the world?
+A: Indonesia.
+
+Q: What is the main ingredient in kimchi?
+A: Cabbage.
+
+Q: What is the capital of New Zealand?
+A: Wellington.
+
+Q: What is the largest peninsula in Europe?
+A: The Scandinavian Peninsula.
+
+Q: What is the main ingredient in gnocchi?
+A: Potatoes.
+
+Q: What is the capital of Denmark?
+A: Copenhagen.
+
+Q: What is the largest gulf in the world?
+A: The Gulf of Mexico.
+
+Q: What is the main ingredient in hummus?
+A: Chickpeas.
+
+Q: What is the capital of Finland?
+A: Helsinki.
+
+Q: What is the largest archipelago in Europe?
+A: The British Isles.
+
+Q: What is the main ingredient in moussaka?
+A: Eggplant.
+
+Q: What is the capital of Ireland?
+A: Dublin.
+
+Q: What is the largest lake in Africa?
+A: Lake Victoria.
+
+Q: What is the main ingredient in tabbouleh?
+A: Parsley.
+
+Q: What is the capital of Poland?
+A: Warsaw.
+
+Q: What is the largest river in Europe?
+A: The Volga River.
+
+Q: What is the main ingredient in sauerkraut?
+A: Cabbage.
+
+Q: What is the capital of Hungary?
+A: Budapest.
+
+Q: What is the largest city in the world by population?
+A: Tokyo.
+
+Q: What is the main ingredient in tempura?
+A: Batter.
+
+Q: What is the capital of Austria?
+A: Vienna.
+
+Q: What is the largest waterfall in Africa?
+A: Victoria Falls.
+
+Q: What is the main ingredient in baklava?
+A: Phyllo dough.
+
+Q: What is the capital of Belgium?
+A: Brussels.
+
+Q: What is the largest peninsula in North America?
+A: The Labrador Peninsula.
+
+Q: What is the main ingredient in couscous?
+A: Semolina.
+
+Q: What is the capital of Czech Republic?
+A: Prague.
+
+Q: What is the largest island in the Mediterranean Sea?
+A: Sicily.
+
+Q: What is the main ingredient in tzatziki?
+A: Yogurt.
+
+Q: What is the capital of Romania?
+A: Bucharest.
+
+Q: What is the largest city in Africa?
+A: Lagos.
+
+Q: What is the main ingredient in goulash?
+A: Beef.
+
+Q: What is the capital of Ukraine?
+A: Kyiv.
+
+Q: What is the largest city in South America?
+A: São Paulo.
+
+Q: What is the main ingredient in feijoada?
+A: Beans.
+
+Q: What is the capital of Chile?
+A: Santiago.
+
+Q: What is the largest city in Australia?
+A: Sydney.
+
+Q: What is the main ingredient in pavlova?
+A: Egg whites.
+
+Q: What is the capital of Peru?
+A: Lima.
+
+Q: What is the largest city in Canada?
+A: Toronto.
+
+Q: What is the main ingredient in poutine?
+A: French fries.
+
+Q: What is the capital of Colombia?
+A: Bogotá.
+
+Q: What is the largest city in India?
+A: Mumbai.
+
+Q: What is the main ingredient in biryani?
+A: Rice.
+
+Q: What is the capital of Pakistan?
+A: Islamabad.
+
+Q: What is the largest city in China?
+A: Shanghai.
+
+Q: What is the main ingredient in hot pot?
+A: Broth.
+
+Q: What is the capital of Indonesia?
+A: Jakarta.
+
+Q: What is the largest city in Russia?
+A: Moscow.
+
+Q: What is the main ingredient in borscht?
+A: Beets.
+
+Q: What is the capital of Saudi Arabia?
+A: Riyadh.
+
+Q: What is the largest city in Turkey?
+A: Istanbul.
+
+Q: What is the main ingredient in kebab?
+A: Meat.
+
+Q: What is the capital of Iran?
+A: Tehran.
+
+Q: What is the largest city in Egypt?
+A: Cairo.
+
+Q: What is the main ingredient in koshari?
+A: Rice.
+
+Q: What is the capital of Nigeria?
+A: Abuja.
+
+Q: What is the largest city in Nigeria?
+A: Lagos.
+
+Q: What is the main ingredient in jollof rice?
+A: Rice.
+
+Q: What is the capital of Kenya?
+A: Nairobi.
+
+Q: What is the largest city in Kenya?
+A: Nairobi.
+
+Q: What is the main ingredient in ugali?
+A: Maize flour.
+
+Q: What is the capital of Ethiopia?
+A: Addis Ababa.
+
+Q: What is the largest city in Ethiopia?
+A: Addis Ababa.
+
+Q: What is the main ingredient in injera?
+A: Teff flour.

finetune.py

@@ -0,0 +1,53 @@
+import torch
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer
+
+# Load your data
+dataset = load_dataset("json", data_files={"train": "qa_data.jsonl"})
+
+# Choose a model (GPT-2 small is easy to start)
+model_name = "gpt2"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
+
+# Add pad token if missing (GPT-2 doesn't have one by default)
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+
+# Tokenize
+def preprocess(example):
+    prompt = example["prompt"]
+    response = example["response"]
+    text = prompt + " " + response
+    tokens = tokenizer(
+        text,
+        truncation=True,
+        padding="max_length",
+        max_length=128,
+    )
+    tokens["labels"] = tokens["input_ids"].copy()
+    return tokens
+
+tokenized = dataset["train"].map(preprocess)
+
+# Training arguments
+args = TrainingArguments(
+    output_dir="gpt2-finetuned-qa",
+    per_device_train_batch_size=2,
+    num_train_epochs=5,
+    logging_steps=10,
+    save_steps=50,
+    fp16=True if torch.cuda.is_available() else False,
+    report_to="none",
+)
+
+# Trainer
+trainer = Trainer(
+    model=model,
+    args=args,
+    train_dataset=tokenized,
+)
+
+trainer.train()
+model.save_pretrained("gpt2-finetuned-qa")
+tokenizer.save_pretrained("gpt2-finetuned-qa")

generate.py

@@ -0,0 +1,14 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+model = AutoModelForCausalLM.from_pretrained("gpt2-finetuned-qa")
+tokenizer = AutoTokenizer.from_pretrained("gpt2-finetuned-qa")
+
+while True:
+    prompt = input("Q: ").strip()
+    if prompt.lower() in ["exit", "quit"]:
+        break
+    full_prompt = f"Q: {prompt}\nA:"
+    inputs = tokenizer(full_prompt, return_tensors="pt")
+    outputs = model.generate(**inputs, max_new_tokens=32, pad_token_id=tokenizer.eos_token_id)
+    print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+    print()

gpt2-finetuned-qa/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "gpt2",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 50257
+}

gpt2-finetuned-qa/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.1"
+}

gpt2-finetuned-qa/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d794b48566c887388ebae13afc50e77184fd881152e032b90b71997dbc350f6
+size 497774208

gpt2-finetuned-qa/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

gpt2-finetuned-qa/tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

model/transformer.py

@@ -0,0 +1,28 @@
+import torch
+import torch.nn as nn
+
+class SimpleGPT(nn.Module):
+    def __init__(self, vocab_size, block_size=8, n_embd=128, n_layer=4, n_head=4):
+        super().__init__()
+        self.token_emb = nn.Embedding(vocab_size, n_embd)
+        self.pos_emb = nn.Embedding(block_size, n_embd)
+        self.blocks = nn.ModuleList([
+            nn.TransformerEncoderLayer(d_model=n_embd, nhead=n_head, dropout=0.1)
+            for _ in range(n_layer)
+        ])
+        self.ln_f = nn.LayerNorm(n_embd)
+        self.head = nn.Linear(n_embd, vocab_size)
+        self.block_size = block_size
+
+    def forward(self, idx):
+        b, t = idx.size()
+        assert t <= self.block_size, "Sequence too long"
+        pos = torch.arange(0, t, dtype=torch.long, device=idx.device)
+        tok_emb = self.token_emb(idx)
+        pos_emb = self.pos_emb(pos)[None, :, :]
+        x = tok_emb + pos_emb
+        for block in self.blocks:
+            x = block(x)
+        x = self.ln_f(x)
+        logits = self.head(x)
+        return logits

qa_data.jsonl

@@ -0,0 +1,160 @@
+{"prompt": "Q: What is the capital of France?\nA:", "response": "Paris."}
+{"prompt": "Q: Who wrote Hamlet?\nA:", "response": "William Shakespeare."}
+{"prompt": "Q: What is the largest ocean?\nA:", "response": "The Pacific Ocean."}
+{"prompt": "Q: What is the boiling point of water?\nA:", "response": "100 degrees Celsius."}
+{"prompt": "Q: Who painted the Mona Lisa?\nA:", "response": "Leonardo da Vinci."}
+{"prompt": "Q: What is the tallest mountain in the world?\nA:", "response": "Mount Everest."}
+{"prompt": "Q: What is the chemical symbol for gold?\nA:", "response": "Au."}
+{"prompt": "Q: What is the freezing point of water?\nA:", "response": "0 degrees Celsius."}
+{"prompt": "Q: Who invented the telephone?\nA:", "response": "Alexander Graham Bell."}
+{"prompt": "Q: What is the largest planet?\nA:", "response": "Jupiter."}
+{"prompt": "Q: What is gravity?\nA:", "response": "Gravity is the force that attracts objects toward the center of the Earth."}
+{"prompt": "Q: What is the main ingredient in bread?\nA:", "response": "Flour."}
+{"prompt": "Q: What is the capital of Japan?\nA:", "response": "Tokyo."}
+{"prompt": "Q: What is the largest continent?\nA:", "response": "Asia."}
+{"prompt": "Q: What is the process of photosynthesis?\nA:", "response": "Photosynthesis is the process by which plants use sunlight to make food from carbon dioxide and water."}
+{"prompt": "Q: What is the smallest prime number?\nA:", "response": "2."}
+{"prompt": "Q: What is the chemical symbol for water?\nA:", "response": "H2O."}
+{"prompt": "Q: Who discovered penicillin?\nA:", "response": "Alexander Fleming."}
+{"prompt": "Q: What is the tallest animal in the world?\nA:", "response": "The giraffe."}
+{"prompt": "Q: What is the largest mammal?\nA:", "response": "The blue whale."}
+{"prompt": "Q: What is the capital of Italy?\nA:", "response": "Rome."}
+{"prompt": "Q: What is the largest desert in the world?\nA:", "response": "The Sahara Desert."}
+{"prompt": "Q: What is the main gas in the air we breathe?\nA:", "response": "Nitrogen."}
+{"prompt": "Q: What is the hardest natural substance?\nA:", "response": "Diamond."}
+{"prompt": "Q: What is the largest country by area?\nA:", "response": "Russia."}
+{"prompt": "Q: What is the longest river in the world?\nA:", "response": "The Nile."}
+{"prompt": "Q: What is the main language spoken in Brazil?\nA:", "response": "Portuguese."}
+{"prompt": "Q: What is the currency of the United States?\nA:", "response": "The dollar."}
+{"prompt": "Q: What is the hottest planet in the solar system?\nA:", "response": "Venus."}
+{"prompt": "Q: What is the largest island in the world?\nA:", "response": "Greenland."}
+{"prompt": "Q: What is the capital of Canada?\nA:", "response": "Ottawa."}
+{"prompt": "Q: Who was the first President of the United States?\nA:", "response": "George Washington."}
+{"prompt": "Q: What is the chemical symbol for iron?\nA:", "response": "Fe."}
+{"prompt": "Q: What is the largest bone in the human body?\nA:", "response": "The femur."}
+{"prompt": "Q: What is the main ingredient in glass?\nA:", "response": "Silica."}
+{"prompt": "Q: What is the capital of Australia?\nA:", "response": "Canberra."}
+{"prompt": "Q: What is the largest lake in the world?\nA:", "response": "The Caspian Sea."}
+{"prompt": "Q: What is the main ingredient in chocolate?\nA:", "response": "Cocoa."}
+{"prompt": "Q: What is the capital of Germany?\nA:", "response": "Berlin."}
+{"prompt": "Q: What is the largest volcano in the world?\nA:", "response": "Mauna Loa."}
+{"prompt": "Q: What is the main ingredient in pizza dough?\nA:", "response": "Flour."}
+{"prompt": "Q: What is the capital of Russia?\nA:", "response": "Moscow."}
+{"prompt": "Q: What is the largest bird in the world?\nA:", "response": "The ostrich."}
+{"prompt": "Q: What is the main ingredient in sushi?\nA:", "response": "Rice."}
+{"prompt": "Q: What is the capital of China?\nA:", "response": "Beijing."}
+{"prompt": "Q: What is the largest fish in the world?\nA:", "response": "The whale shark."}
+{"prompt": "Q: What is the main ingredient in guacamole?\nA:", "response": "Avocado."}
+{"prompt": "Q: What is the capital of India?\nA:", "response": "New Delhi."}
+{"prompt": "Q: What is the largest reptile in the world?\nA:", "response": "The saltwater crocodile."}
+{"prompt": "Q: What is the main ingredient in hummus?\nA:", "response": "Chickpeas."}
+{"prompt": "Q: What is the capital of Brazil?\nA:", "response": "Brasília."}
+{"prompt": "Q: What is the largest flower in the world?\nA:", "response": "Rafflesia arnoldii."}
+{"prompt": "Q: What is the capital of Spain?\nA:", "response": "Madrid."}
+{"prompt": "Q: What is the largest peninsula in the world?\nA:", "response": "The Arabian Peninsula."}
+{"prompt": "Q: What is the main ingredient in pasta?\nA:", "response": "Wheat."}
+{"prompt": "Q: What is the capital of Egypt?\nA:", "response": "Cairo."}
+{"prompt": "Q: What is the largest bay in the world?\nA:", "response": "The Bay of Bengal."}
+{"prompt": "Q: What is the main ingredient in mayonnaise?\nA:", "response": "Eggs."}
+{"prompt": "Q: What is the capital of South Africa?\nA:", "response": "Pretoria."}
+{"prompt": "Q: What is the largest canyon in the world?\nA:", "response": "The Grand Canyon."}
+{"prompt": "Q: What is the main ingredient in tofu?\nA:", "response": "Soybeans."}
+{"prompt": "Q: What is the capital of Mexico?\nA:", "response": "Mexico City."}
+{"prompt": "Q: What is the largest coral reef in the world?\nA:", "response": "The Great Barrier Reef."}
+{"prompt": "Q: What is the main ingredient in ketchup?\nA:", "response": "Tomatoes."}
+{"prompt": "Q: What is the capital of Argentina?\nA:", "response": "Buenos Aires."}
+{"prompt": "Q: What is the largest waterfall in the world?\nA:", "response": "Angel Falls."}
+{"prompt": "Q: What is the main ingredient in peanut butter?\nA:", "response": "Peanuts."}
+{"prompt": "Q: What is the capital of Turkey?\nA:", "response": "Ankara."}
+{"prompt": "Q: What is the largest stadium in the world?\nA:", "response": "Rungrado 1st of May Stadium."}
+{"prompt": "Q: What is the main ingredient in miso soup?\nA:", "response": "Miso paste."}
+{"prompt": "Q: What is the capital of Saudi Arabia?\nA:", "response": "Riyadh."}
+{"prompt": "Q: What is the largest archipelago in the world?\nA:", "response": "Indonesia."}
+{"prompt": "Q: What is the main ingredient in falafel?\nA:", "response": "Chickpeas."}
+{"prompt": "Q: What is the capital of Thailand?\nA:", "response": "Bangkok."}
+{"prompt": "Q: What is the largest cave in the world?\nA:", "response": "Son Doong Cave."}
+{"prompt": "Q: What is the main ingredient in curry?\nA:", "response": "Spices."}
+{"prompt": "Q: What is the capital of Sweden?\nA:", "response": "Stockholm."}
+{"prompt": "Q: What is the largest glacier in the world?\nA:", "response": "Lambert Glacier."}
+{"prompt": "Q: What is the main ingredient in risotto?\nA:", "response": "Rice."}
+{"prompt": "Q: What is the capital of Norway?\nA:", "response": "Oslo."}
+{"prompt": "Q: What is the largest hot desert in the world?\nA:", "response": "The Sahara Desert."}
+{"prompt": "Q: What is the main ingredient in borscht?\nA:", "response": "Beets."}
+{"prompt": "Q: What is the capital of Switzerland?\nA:", "response": "Bern."}
+{"prompt": "Q: What is the largest freshwater lake in the world?\nA:", "response": "Lake Superior."}
+{"prompt": "Q: What is the main ingredient in paella?\nA:", "response": "Rice."}
+{"prompt": "Q: What is the capital of the Netherlands?\nA:", "response": "Amsterdam."}
+{"prompt": "Q: What is the largest sea in the world?\nA:", "response": "The Philippine Sea."}
+{"prompt": "Q: What is the main ingredient in pesto?\nA:", "response": "Basil."}
+{"prompt": "Q: What is the capital of Greece?\nA:", "response": "Athens."}
+{"prompt": "Q: What is the largest plateau in the world?\nA:", "response": "The Tibetan Plateau."}
+{"prompt": "Q: What is the main ingredient in gazpacho?\nA:", "response": "Tomatoes."}
+{"prompt": "Q: What is the capital of Portugal?\nA:", "response": "Lisbon."}
+{"prompt": "Q: What is the largest delta in the world?\nA:", "response": "The Ganges Delta."}
+{"prompt": "Q: What is the main ingredient in ramen?\nA:", "response": "Noodles."}
+{"prompt": "Q: What is the capital of South Korea?\nA:", "response": "Seoul."}
+{"prompt": "Q: What is the largest island country in the world?\nA:", "response": "Indonesia."}
+{"prompt": "Q: What is the main ingredient in kimchi?\nA:", "response": "Cabbage."}
+{"prompt": "Q: What is the capital of New Zealand?\nA:", "response": "Wellington."}
+{"prompt": "Q: What is the largest peninsula in Europe?\nA:", "response": "The Scandinavian Peninsula."}
+{"prompt": "Q: What is the main ingredient in gnocchi?\nA:", "response": "Potatoes."}
+{"prompt": "Q: What is the capital of Denmark?\nA:", "response": "Copenhagen."}
+{"prompt": "Q: What is the largest gulf in the world?\nA:", "response": "The Gulf of Mexico."}
+{"prompt": "Q: What is the main ingredient in hummus?\nA:", "response": "Chickpeas."}
+{"prompt": "Q: What is the capital of Finland?\nA:", "response": "Helsinki."}
+{"prompt": "Q: What is the largest archipelago in Europe?\nA:", "response": "The British Isles."}
+{"prompt": "Q: What is the main ingredient in moussaka?\nA:", "response": "Eggplant."}
+{"prompt": "Q: What is the capital of Ireland?\nA:", "response": "Dublin."}
+{"prompt": "Q: What is the largest lake in Africa?\nA:", "response": "Lake Victoria."}
+{"prompt": "Q: What is the main ingredient in tabbouleh?\nA:", "response": "Parsley."}
+{"prompt": "Q: What is the capital of Poland?\nA:", "response": "Warsaw."}
+{"prompt": "Q: What is the largest river in Europe?\nA:", "response": "The Volga River."}
+{"prompt": "Q: What is the main ingredient in sauerkraut?\nA:", "response": "Cabbage."}
+{"prompt": "Q: What is the capital of Hungary?\nA:", "response": "Budapest."}
+{"prompt": "Q: What is the largest city in the world by population?\nA:", "response": "Tokyo."}
+{"prompt": "Q: What is the main ingredient in tempura?\nA:", "response": "Batter."}
+{"prompt": "Q: What is the capital of Austria?\nA:", "response": "Vienna."}
+{"prompt": "Q: What is the largest waterfall in Africa?\nA:", "response": "Victoria Falls."}
+{"prompt": "Q: What is the main ingredient in baklava?\nA:", "response": "Phyllo dough."}
+{"prompt": "Q: What is the capital of Belgium?\nA:", "response": "Brussels."}
+{"prompt": "Q: What is the largest peninsula in North America?\nA:", "response": "The Labrador Peninsula."}
+{"prompt": "Q: What is the main ingredient in couscous?\nA:", "response": "Semolina."}
+{"prompt": "Q: What is the capital of Czech Republic?\nA:", "response": "Prague."}
+{"prompt": "Q: What is the largest island in the Mediterranean Sea?\nA:", "response": "Sicily."}
+{"prompt": "Q: What is the main ingredient in tzatziki?\nA:", "response": "Yogurt."}
+{"prompt": "Q: What is the capital of Romania?\nA:", "response": "Bucharest."}
+{"prompt": "Q: What is the largest city in Africa?\nA:", "response": "Lagos."}
+{"prompt": "Q: What is the main ingredient in goulash?\nA:", "response": "Beef."}
+{"prompt": "Q: What is the capital of Ukraine?\nA:", "response": "Kyiv."}
+{"prompt": "Q: What is the largest city in South America?\nA:", "response": "São Paulo."}
+{"prompt": "Q: What is the main ingredient in feijoada?\nA:", "response": "Beans."}
+{"prompt": "Q: What is the capital of Chile?\nA:", "response": "Santiago."}
+{"prompt": "Q: What is the largest city in Australia?\nA:", "response": "Sydney."}
+{"prompt": "Q: What is the main ingredient in pavlova?\nA:", "response": "Egg whites."}
+{"prompt": "Q: What is the capital of Peru?\nA:", "response": "Lima."}
+{"prompt": "Q: What is the largest city in Canada?\nA:", "response": "Toronto."}
+{"prompt": "Q: What is the main ingredient in poutine?\nA:", "response": "French fries."}
+{"prompt": "Q: What is the capital of Colombia?\nA:", "response": "Bogotá."}
+{"prompt": "Q: What is the largest city in India?\nA:", "response": "Mumbai."}
+{"prompt": "Q: What is the main ingredient in biryani?\nA:", "response": "Rice."}
+{"prompt": "Q: What is the capital of Pakistan?\nA:", "response": "Islamabad."}
+{"prompt": "Q: What is the largest city in China?\nA:", "response": "Shanghai."}
+{"prompt": "Q: What is the main ingredient in hot pot?\nA:", "response": "Broth."}
+{"prompt": "Q: What is the capital of Indonesia?\nA:", "response": "Jakarta."}
+{"prompt": "Q: What is the largest city in Russia?\nA:", "response": "Moscow."}
+{"prompt": "Q: What is the main ingredient in borscht?\nA:", "response": "Beets."}
+{"prompt": "Q: What is the capital of Saudi Arabia?\nA:", "response": "Riyadh."}
+{"prompt": "Q: What is the largest city in Turkey?\nA:", "response": "Istanbul."}
+{"prompt": "Q: What is the main ingredient in kebab?\nA:", "response": "Meat."}
+{"prompt": "Q: What is the capital of Iran?\nA:", "response": "Tehran."}
+{"prompt": "Q: What is the largest city in Egypt?\nA:", "response": "Cairo."}
+{"prompt": "Q: What is the main ingredient in koshari?\nA:", "response": "Rice."}
+{"prompt": "Q: What is the capital of Nigeria?\nA:", "response": "Abuja."}
+{"prompt": "Q: What is the largest city in Nigeria?\nA:", "response": "Lagos."}
+{"prompt": "Q: What is the main ingredient in jollof rice?\nA:", "response": "Rice."}
+{"prompt": "Q: What is the capital of Kenya?\nA:", "response": "Nairobi."}
+{"prompt": "Q: What is the largest city in Kenya?\nA:", "response": "Nairobi."}
+{"prompt": "Q: What is the main ingredient in ugali?\nA:", "response": "Maize flour."}
+{"prompt": "Q: What is the capital of Ethiopia?\nA:", "response": "Addis Ababa."}
+{"prompt": "Q: What is the largest city in Ethiopia?\nA:", "response": "Addis Ababa."}
+{"prompt": "Q: What is the main ingredient in injera?\nA:", "response": "Teff flour."}

requirements.txt

@@ -0,0 +1,3 @@
+torch
+tokenizers
+tqdm

tokenizer/merges.txt

@@ -0,0 +1,745 @@
+#version: 0.2
+č Ċ
+Ġ i
+h e
+Ġ t
+Ġt he
+a t
+Ġ W
+ĠW h
+Ġi s
+ĠWh at
+Ġi n
+e s
+a r
+i n
+a l
+Ġ c
+Ġ o
+a n
+i t
+e n
+e r
+es t
+Ġo f
+Ġ l
+r e
+Ġ w
+g est
+o r
+ar gest
+Ġl argest
+Ġ m
+d i
+a in
+a p
+Ġ p
+en t
+g re
+Ġc ap
+it al
+Ġcap ital
+Ġm ain
+Ġ T
+o n
+Ġin gre
+di ent
+Ġingre dient
+i c
+l d
+Ġ s
+Ġw or
+Ġwor ld
+Ġ S
+ĠT he
+Ġ A
+Ġ f
+i a
+r a
+i l
+Ġ C
+o u
+Ġ b
+Ġ P
+Ġ B
+e l
+u s
+r o
+o l
+i s
+an d
+Ġ d
+e d
+e t
+Ġ M
+c e
+Ġ R
+Ġ g
+it y
+Ġ G
+at er
+n t
+Ġ F
+Ġ L
+ĠWh o
+Ġ E
+a k
+Ġw ater
+a m
+a s
+p e
+in g
+o t
+Ġ N
+a d
+l e
+m b
+o s
+t s
+al l
+Ġf or
+g h
+l an
+Ġ I
+ar d
+ou nt
+c i
+h ot
+s t
+t h
+v er
+Ġ D
+Ġ V
+Ġc ity
+Ġp lan
+Ġs y
+el s
+e c
+l and
+o m
+u r
+Ġ J
+Ġp o
+e gre
+i us
+u l
+Ġ H
+in t
+ĠC els
+Ġd egre
+all est
+Ġpo int
+ĠCels ius
+Ġdegre es
+a g
+a i
+c h
+c o
+e a
+i gh
+m ic
+o k
+r i
+v ity
+Ġ k
+he mic
+Ġt o
+Ġc hemic
+Ġw h
+ra vity
+ce an
+ak e
+mb ol
+Ġsy mbol
+igh t
+Ġchemic al
+l ou
+r an
+s ia
+Ġ 2
+er t
+on d
+ia m
+il l
+Ġplan et
+lou r
+a b
+c es
+s is
+t a
+u m
+u n
+u p
+x and
+y nt
+Ġ O
+Ġ h
+he sis
+Ġt allest
+es ert
+Ġp ro
+ic a
+ic e
+ĠA le
+ĠC h
+Ġb o
+Ġb y
+Ġb re
+os ynt
+hot osynt
+ces s
+xand er
+Ġpro cess
+ĠAle xander
+hotosynt hesis
+a ci
+e m
+g u
+h ar
+t e
+Ġ 1
+Ġ ar
+Ġ di
+it er
+re e
+on e
+ĠS h
+ĠP ar
+ĠB ra
+el l
+ĠR ice
+ĠF lour
+ĠL is
+0 0
+7 9
+9 9
+e p
+e on
+f f
+i d
+i r
+l o
+l et
+m al
+o c
+s ul
+t ed
+y o
+Ġ U
+Ġ n
+at t
+ĠW ill
+es pe
+ar e
+ar th
+in a
+in ci
+in sul
+en insul
+er s
+Ġl ight
+Ġw ro
+ain ted
+Ġp er
+Ġp ainted
+ĠT ok
+on a
+ic h
+Ġs pe
+Ġs ec
+ĠS a
+ĠS t
+il e
+il ing
+il om
+ĠC an
+ĠB e
+us sia
+Ġd a
+et ers
+ĠM ona
+ĠR ussia
+ĠF ran
+ĠL eon
+ĠE arth
+ak espe
+am let
+ĠI n
+ard o
+ĠV inci
+ĠJ up
+ĠH amlet
+Ġk ilom
+Ġ2 99
+Ġbo iling
+Ġ1 00
+ĠSh akespe
+ĠPar is
+ĠLis a
+79 2
+ĠWill iam
+eninsul a
+Ġwro te
+ĠTok yo
+Ġspe ed
+Ġsec ond
+ĠFran ce
+ĠLeon ardo
+ĠJup iter
+Ġkilom eters
+ĠShakespe are
+a y
+b j
+b on
+c ts
+f ic
+h at
+h am
+h one
+n it
+o d
+o ld
+o od
+r y
+u b
+v ent
+w ard
+z il
+z ing
+Ġ 0
+Ġ hot
+Ġ att
+Ġt el
+Ġt hat
+at es
+Ġis land
+Ġin vent
+es ia
+al e
+Ġc ent
+Ġo cean
+Ġo bj
+Ġm ount
+ap an
+ĠA u
+Ġf ree
+ra cts
+ra ham
+ĠP aci
+ĠB ell
+ĠM ount
+Ġg ravity
+Ġg old
+ĠG ravity
+ĠG raham
+ĠE g
+ĠE ver
+Ġfor ce
+ec ts
+ĠJ apan
+ag e
+Ġto ward
+Ġwh ale
+ĠO cean
+Ġh um
+Ġbre ad
+ĠBra zil
+ep hone
+ĠU nit
+ĠSt ates
+Ġatt racts
+Ġtel ephone
+Ġinvent ed
+Ġcent er
+Ġobj ects
+Ġmount ain
+Ġfree zing
+ĠPaci fic
+ĠEver est
+ĠUnit ed
+a v
+e an
+f r
+h i
+i an
+k pe
+l ight
+o x
+p t
+r en
+t r
+t u
+t in
+u mb
+Ġ K
+Ġ us
+Ġ and
+Ġ ri
+ar bon
+Ġc ount
+Ġc ur
+Ġc arbon
+an i
+Ġl ake
+or tu
+Ġm ake
+Ġp hotosynthesis
+ic kpe
+Ġs un
+ĠS ea
+ĠA us
+ĠA fr
+Ġf ro
+Ġf ood
+ĠP hotosynthesis
+ĠP ortu
+Ġd esert
+ĠM e
+ĠR om
+ĠG re
+ad a
+st an
+ĠD esert
+Ġplan ts
+ai ro
+ch i
+Ġwh ich
+ĠCh ickpe
+har a
+Ġdi ox
+id e
+ĠSa hara
+Ġus e
+Ġri ver
+Ġcount ry
+Ġsun light
+ĠAus tr
+ĠAfr ica
+Ġfro m
+ĠChickpe as
+Ġdiox ide
+b ab
+c y
+d on
+e w
+e y
+g al
+g en
+i j
+i ra
+l u
+l ar
+l in
+l as
+m e
+m us
+m ing
+m allest
+o p
+o es
+p ok
+r es
+s co
+t al
+t est
+t or
+t on
+u t
+u ro
+w ater
+x ic
+Ġ ai
+Ġ ani
+at he
+at ur
+at oes
+es e
+al ia
+Ġc on
+an y
+an gu
+it ro
+en ic
+en us
+en land
+Ġl on
+Ġl angu
+Ġw e
+Ġm am
+di a
+Ġp ri
+Ġp eninsula
+Ġp enic
+Ġs ol
+Ġs ub
+Ġs mallest
+Ġs pok
+ĠS ou
+ĠA m
+ĠA ra
+ĠA sia
+ou s
+ou gh
+Ġb ir
+Ġb lu
+ĠP eninsula
+ĠB u
+ĠB er
+ol lar
+is h
+Ġd ough
+Ġd ollar
+ĠM os
+ĠR i
+Ġg as
+Ġg ira
+ĠF le
+ĠE uro
+ing ton
+ĠN ile
+ĠN ew
+ĠN itro
+ĠI tal
+ard est
+st em
+ver ed
+ĠD el
+ĠD iam
+ĠV enus
+Ġsy stem
+co w
+ill in
+ab b
+Ġh ardest
+ĠCh ina
+Ġbre athe
+gu ese
+Ġar ea
+Ġdi sco
+ff e
+Ġn umb
+Ġn atur
+ĠCan ada
+ĠIn don
+ĠIn dia
+Ġhot test
+ĠEg g
+Ġhum mus
+ren cy
+tin ent
+Ġcur rency
+ĠPortu guese
+ĠMe xic
+ĠRom e
+ĠGre enland
+stan ce
+ĠAustr alia
+tor ia
+Ġai r
+Ġani mal
+Ġcon tinent
+Ġlon gest
+Ġlangu age
+Ġmam mal
+Ġpri me
+Ġpenic illin
+Ġsol ar
+Ġsub stance
+Ġspok en
+ĠSou th
+ĠAra b
+Ġblu e
+ĠMos cow
+Ġgira ffe
+ĠFle ming
+ĠEuro pe
+ĠNitro gen
+ĠItal y
+ĠDiam ond
+Ġdisco vered
+Ġnumb er
+Ġnatur al
+ĠIndon esia
+ĠMexic o
+a c
+a un
+a ff
+b i
+b er
+c an
+d y
+d di
+e f
+e or
+f all
+g e
+g er
+h oc
+h ington
+i m
+i z
+i op
+i ger
+k i
+k ey
+l at
+l es
+l ia
+l aci
+l esia
+m any
+n a
+n ol
+o a
+o g
+o co
+p el
+p ian
+r on
+r ich
+s Ã
+s ch
+t o
+t ta
+t water
+u di
+u ac
+v ol
+v oc
+w a
+w er
+y a
+y ad
+y pt
+z a
+Ġ r
+Ġ re
+Ġ vol
+Ń lia
+Ġi ron
+at e
+ĠW as
+al twater
+Ġc ro
+Ġc hoc
+Ġo st
+an g
+en ya
+er ica
+er many
+Ġw as
+or sch
+di i
+di le
+di um
+Ġp as
+Ġp iz
+ĠT e
+ĠT om
+ĠT ur
+ic toria
+Ġs us
+Ġs har
+Ġs altwater
+ĠS p
+ĠS w
+ĠS il
+ĠA n
+ĠA bab
+ĠA ddi
+ĠA voc
+Ġf lour
+Ġf em
+Ġf ir
+Ġf lo
+Ġf ish
+ra d
+ĠC as
+ĠC airo
+ĠC abb
+ĠC oco
+Ġb one
+Ġb orsch
+ĠP h
+ĠP res
+ĠB r
+ol e
+ol ate
+is o
+et s
+ĠM aun
+ĠR aff
+Ġg las
+Ġg uac
+ĠG eor
+ĠG ermany
+ĠF e
+ĠF all
+ĠL ag
+ĠL ake
+ĠL oa
+ĠE th
+Ġwater fall
+am ole
+ĠN or
+ĠN airo
+ĠN iger
+ad o
+ĠI s
+ĠV ictoria
+ul f
+ag o
+co dile
+ea u
+ta dium
+un g
+ĠO tta
+Ġbo dy
+Ġar chi
+Ġar nol
+ĠBra sÃ
+id ent
+ĠSa udi
+ĠCan ber
+ĠBe ij
+ĠBe ets
+ĠEg ypt
+Ġhum an
+av e
+pt ile
+ĠK enya
+ĠAm erica
+Ġbir d
+ĠBer lin
+ĠRi yad
+ĠDel hi
+ĠArab ia
+can o
+iop ia
+lat eau
+laci er
+pel ago
+Ġre ptile
+Ġvol cano
+ĠWas hington
+Ġcro codile
+Ġchoc olate
+Ġost rich
+Ġpiz za
+ĠTom atoes
+ĠTur key
+Ġsus hi
+Ġshar k
+ĠSil ica
+ĠAbab a
+ĠAddi s
+ĠAvoc ado
+Ġfem ur
+Ġfir st
+Ġflo wer
+ĠCas pian
+ĠCabb age
+ĠCoco a
+Ġborsch t
+ĠPres ident
+ĠMaun a

tokenizer/train_tokenizer.py

@@ -0,0 +1,10 @@
+from tokenizers import ByteLevelBPETokenizer
+import os
+
+input_path = os.path.join("..", "data", "input.txt")
+if not os.path.exists(input_path):
+    input_path = os.path.join("data", "input.txt")
+tokenizer = ByteLevelBPETokenizer()
+tokenizer.train(files=input_path, vocab_size=1000, min_frequency=2)
+tokenizer.save_model(os.path.dirname(__file__))
+print("Tokenizer trained and saved.")

tokenizer/vocab.json

@@ -0,0 +1 @@
+{"!":0,"\"":1,"#":2,"$":3,"%":4,"&":5,"'":6,"(":7,")":8,"*":9,"+":10,",":11,"-":12,".":13,"/":14,"0":15,"1":16,"2":17,"3":18,"4":19,"5":20,"6":21,"7":22,"8":23,"9":24,":":25,";":26,"<":27,"=":28,">":29,"?":30,"@":31,"A":32,"B":33,"C":34,"D":35,"E":36,"F":37,"G":38,"H":39,"I":40,"J":41,"K":42,"L":43,"M":44,"N":45,"O":46,"P":47,"Q":48,"R":49,"S":50,"T":51,"U":52,"V":53,"W":54,"X":55,"Y":56,"Z":57,"[":58,"\\":59,"]":60,"^":61,"_":62,"`":63,"a":64,"b":65,"c":66,"d":67,"e":68,"f":69,"g":70,"h":71,"i":72,"j":73,"k":74,"l":75,"m":76,"n":77,"o":78,"p":79,"q":80,"r":81,"s":82,"t":83,"u":84,"v":85,"w":86,"x":87,"y":88,"z":89,"{":90,"|":91,"}":92,"~":93,"¡":94,"¢":95,"£":96,"¤":97,"¥":98,"¦":99,"§":100,"¨":101,"©":102,"ª":103,"«":104,"¬":105,"®":106,"¯":107,"°":108,"±":109,"²":110,"³":111,"´":112,"µ":113,"¶":114,"·":115,"¸":116,"¹":117,"º":118,"»":119,"¼":120,"½":121,"¾":122,"¿":123,"À":124,"Á":125,"Â":126,"Ã":127,"Ä":128,"Å":129,"Æ":130,"Ç":131,"È":132,"É":133,"Ê":134,"Ë":135,"Ì":136,"Í":137,"Î":138,"Ï":139,"Ð":140,"Ñ":141,"Ò":142,"Ó":143,"Ô":144,"Õ":145,"Ö":146,"×":147,"Ø":148,"Ù":149,"Ú":150,"Û":151,"Ü":152,"Ý":153,"Þ":154,"ß":155,"à":156,"á":157,"â":158,"ã":159,"ä":160,"å":161,"æ":162,"ç":163,"è":164,"é":165,"ê":166,"ë":167,"ì":168,"í":169,"î":170,"ï":171,"ð":172,"ñ":173,"ò":174,"ó":175,"ô":176,"õ":177,"ö":178,"÷":179,"ø":180,"ù":181,"ú":182,"û":183,"ü":184,"ý":185,"þ":186,"ÿ":187,"Ā":188,"ā":189,"Ă":190,"ă":191,"Ą":192,"ą":193,"Ć":194,"ć":195,"Ĉ":196,"ĉ":197,"Ċ":198,"ċ":199,"Č":200,"č":201,"Ď":202,"ď":203,"Đ":204,"đ":205,"Ē":206,"ē":207,"Ĕ":208,"ĕ":209,"Ė":210,"ė":211,"Ę":212,"ę":213,"Ě":214,"ě":215,"Ĝ":216,"ĝ":217,"Ğ":218,"ğ":219,"Ġ":220,"ġ":221,"Ģ":222,"ģ":223,"Ĥ":224,"ĥ":225,"Ħ":226,"ħ":227,"Ĩ":228,"ĩ":229,"Ī":230,"ī":231,"Ĭ":232,"ĭ":233,"Į":234,"į":235,"İ":236,"ı":237,"IJ":238,"ij":239,"Ĵ":240,"ĵ":241,"Ķ":242,"ķ":243,"ĸ":244,"Ĺ":245,"ĺ":246,"Ļ":247,"ļ":248,"Ľ":249,"ľ":250,"Ŀ":251,"ŀ":252,"Ł":253,"ł":254,"Ń":255,"čĊ":256,"Ġi":257,"he":258,"Ġt":259,"Ġthe":260,"at":261,"ĠW":262,"ĠWh":263,"Ġis":264,"ĠWhat":265,"Ġin":266,"es":267,"ar":268,"in":269,"al":270,"Ġc":271,"Ġo":272,"an":273,"it":274,"en":275,"er":276,"est":277,"Ġof":278,"Ġl":279,"re":280,"Ġw":281,"gest":282,"or":283,"argest":284,"Ġlargest":285,"Ġm":286,"di":287,"ain":288,"ap":289,"Ġp":290,"ent":291,"gre":292,"Ġcap":293,"ital":294,"Ġcapital":295,"Ġmain":296,"ĠT":297,"on":298,"Ġingre":299,"dient":300,"Ġingredient":301,"ic":302,"ld":303,"Ġs":304,"Ġwor":305,"Ġworld":306,"ĠS":307,"ĠThe":308,"ĠA":309,"Ġf":310,"ia":311,"ra":312,"il":313,"ĠC":314,"ou":315,"Ġb":316,"ĠP":317,"ĠB":318,"el":319,"us":320,"ro":321,"ol":322,"is":323,"and":324,"Ġd":325,"ed":326,"et":327,"ĠM":328,"ce":329,"ĠR":330,"Ġg":331,"ity":332,"ĠG":333,"ater":334,"nt":335,"ĠF":336,"ĠL":337,"ĠWho":338,"ĠE":339,"ak":340,"Ġwater":341,"am":342,"as":343,"pe":344,"ing":345,"ot":346,"ĠN":347,"ad":348,"le":349,"mb":350,"os":351,"ts":352,"all":353,"Ġfor":354,"gh":355,"lan":356,"ĠI":357,"ard":358,"ount":359,"ci":360,"hot":361,"st":362,"th":363,"ver":364,"ĠD":365,"ĠV":366,"Ġcity":367,"Ġplan":368,"Ġsy":369,"els":370,"ec":371,"land":372,"om":373,"ur":374,"ĠJ":375,"Ġpo":376,"egre":377,"ius":378,"ul":379,"ĠH":380,"int":381,"ĠCels":382,"Ġdegre":383,"allest":384,"Ġpoint":385,"ĠCelsius":386,"Ġdegrees":387,"ag":388,"ai":389,"ch":390,"co":391,"ea":392,"igh":393,"mic":394,"ok":395,"ri":396,"vity":397,"Ġk":398,"hemic":399,"Ġto":400,"Ġchemic":401,"Ġwh":402,"ravity":403,"cean":404,"ake":405,"mbol":406,"Ġsymbol":407,"ight":408,"Ġchemical":409,"lou":410,"ran":411,"sia":412,"Ġ2":413,"ert":414,"ond":415,"iam":416,"ill":417,"Ġplanet":418,"lour":419,"ab":420,"ces":421,"sis":422,"ta":423,"um":424,"un":425,"up":426,"xand":427,"ynt":428,"ĠO":429,"Ġh":430,"hesis":431,"Ġtallest":432,"esert":433,"Ġpro":434,"ica":435,"ice":436,"ĠAle":437,"ĠCh":438,"Ġbo":439,"Ġby":440,"Ġbre":441,"osynt":442,"hotosynt":443,"cess":444,"xander":445,"Ġprocess":446,"ĠAlexander":447,"hotosynthesis":448,"aci":449,"em":450,"gu":451,"har":452,"te":453,"Ġ1":454,"Ġar":455,"Ġdi":456,"iter":457,"ree":458,"one":459,"ĠSh":460,"ĠPar":461,"ĠBra":462,"ell":463,"ĠRice":464,"ĠFlour":465,"ĠLis":466,"00":467,"79":468,"99":469,"ep":470,"eon":471,"ff":472,"id":473,"ir":474,"lo":475,"let":476,"mal":477,"oc":478,"sul":479,"ted":480,"yo":481,"ĠU":482,"Ġn":483,"att":484,"ĠWill":485,"espe":486,"are":487,"arth":488,"ina":489,"inci":490,"insul":491,"eninsul":492,"ers":493,"Ġlight":494,"Ġwro":495,"ainted":496,"Ġper":497,"Ġpainted":498,"ĠTok":499,"ona":500,"ich":501,"Ġspe":502,"Ġsec":503,"ĠSa":504,"ĠSt":505,"ile":506,"iling":507,"ilom":508,"ĠCan":509,"ĠBe":510,"ussia":511,"Ġda":512,"eters":513,"ĠMona":514,"ĠRussia":515,"ĠFran":516,"ĠLeon":517,"ĠEarth":518,"akespe":519,"amlet":520,"ĠIn":521,"ardo":522,"ĠVinci":523,"ĠJup":524,"ĠHamlet":525,"Ġkilom":526,"Ġ299":527,"Ġboiling":528,"Ġ100":529,"ĠShakespe":530,"ĠParis":531,"ĠLisa":532,"792":533,"ĠWilliam":534,"eninsula":535,"Ġwrote":536,"ĠTokyo":537,"Ġspeed":538,"Ġsecond":539,"ĠFrance":540,"ĠLeonardo":541,"ĠJupiter":542,"Ġkilometers":543,"ĠShakespeare":544,"ay":545,"bj":546,"bon":547,"cts":548,"fic":549,"hat":550,"ham":551,"hone":552,"nit":553,"od":554,"old":555,"ood":556,"ry":557,"ub":558,"vent":559,"ward":560,"zil":561,"zing":562,"Ġ0":563,"Ġhot":564,"Ġatt":565,"Ġtel":566,"Ġthat":567,"ates":568,"Ġisland":569,"Ġinvent":570,"esia":571,"ale":572,"Ġcent":573,"Ġocean":574,"Ġobj":575,"Ġmount":576,"apan":577,"ĠAu":578,"Ġfree":579,"racts":580,"raham":581,"ĠPaci":582,"ĠBell":583,"ĠMount":584,"Ġgravity":585,"Ġgold":586,"ĠGravity":587,"ĠGraham":588,"ĠEg":589,"ĠEver":590,"Ġforce":591,"ects":592,"ĠJapan":593,"age":594,"Ġtoward":595,"Ġwhale":596,"ĠOcean":597,"Ġhum":598,"Ġbread":599,"ĠBrazil":600,"ephone":601,"ĠUnit":602,"ĠStates":603,"Ġattracts":604,"Ġtelephone":605,"Ġinvented":606,"Ġcenter":607,"Ġobjects":608,"Ġmountain":609,"Ġfreezing":610,"ĠPacific":611,"ĠEverest":612,"ĠUnited":613,"av":614,"ean":615,"fr":616,"hi":617,"ian":618,"kpe":619,"light":620,"ox":621,"pt":622,"ren":623,"tr":624,"tu":625,"tin":626,"umb":627,"ĠK":628,"Ġus":629,"Ġand":630,"Ġri":631,"arbon":632,"Ġcount":633,"Ġcur":634,"Ġcarbon":635,"ani":636,"Ġlake":637,"ortu":638,"Ġmake":639,"Ġphotosynthesis":640,"ickpe":641,"Ġsun":642,"ĠSea":643,"ĠAus":644,"ĠAfr":645,"Ġfro":646,"Ġfood":647,"ĠPhotosynthesis":648,"ĠPortu":649,"Ġdesert":650,"ĠMe":651,"ĠRom":652,"ĠGre":653,"ada":654,"stan":655,"ĠDesert":656,"Ġplants":657,"airo":658,"chi":659,"Ġwhich":660,"ĠChickpe":661,"hara":662,"Ġdiox":663,"ide":664,"ĠSahara":665,"Ġuse":666,"Ġriver":667,"Ġcountry":668,"Ġsunlight":669,"ĠAustr":670,"ĠAfrica":671,"Ġfrom":672,"ĠChickpeas":673,"Ġdioxide":674,"bab":675,"cy":676,"don":677,"ew":678,"ey":679,"gal":680,"gen":681,"ij":682,"ira":683,"lu":684,"lar":685,"lin":686,"las":687,"me":688,"mus":689,"ming":690,"mallest":691,"op":692,"oes":693,"pok":694,"res":695,"sco":696,"tal":697,"test":698,"tor":699,"ton":700,"ut":701,"uro":702,"water":703,"xic":704,"Ġai":705,"Ġani":706,"athe":707,"atur":708,"atoes":709,"ese":710,"alia":711,"Ġcon":712,"any":713,"angu":714,"itro":715,"enic":716,"enus":717,"enland":718,"Ġlon":719,"Ġlangu":720,"Ġwe":721,"Ġmam":722,"dia":723,"Ġpri":724,"Ġpeninsula":725,"Ġpenic":726,"Ġsol":727,"Ġsub":728,"Ġsmallest":729,"Ġspok":730,"ĠSou":731,"ĠAm":732,"ĠAra":733,"ĠAsia":734,"ous":735,"ough":736,"Ġbir":737,"Ġblu":738,"ĠPeninsula":739,"ĠBu":740,"ĠBer":741,"ollar":742,"ish":743,"Ġdough":744,"Ġdollar":745,"ĠMos":746,"ĠRi":747,"Ġgas":748,"Ġgira":749,"ĠFle":750,"ĠEuro":751,"ington":752,"ĠNile":753,"ĠNew":754,"ĠNitro":755,"ĠItal":756,"ardest":757,"stem":758,"vered":759,"ĠDel":760,"ĠDiam":761,"ĠVenus":762,"Ġsystem":763,"cow":764,"illin":765,"abb":766,"Ġhardest":767,"ĠChina":768,"Ġbreathe":769,"guese":770,"Ġarea":771,"Ġdisco":772,"ffe":773,"Ġnumb":774,"Ġnatur":775,"ĠCanada":776,"ĠIndon":777,"ĠIndia":778,"Ġhottest":779,"ĠEgg":780,"Ġhummus":781,"rency":782,"tinent":783,"Ġcurrency":784,"ĠPortuguese":785,"ĠMexic":786,"ĠRome":787,"ĠGreenland":788,"stance":789,"ĠAustralia":790,"toria":791,"Ġair":792,"Ġanimal":793,"Ġcontinent":794,"Ġlongest":795,"Ġlanguage":796,"Ġmammal":797,"Ġprime":798,"Ġpenicillin":799,"Ġsolar":800,"Ġsubstance":801,"Ġspoken":802,"ĠSouth":803,"ĠArab":804,"Ġblue":805,"ĠMoscow":806,"Ġgiraffe":807,"ĠFleming":808,"ĠEurope":809,"ĠNitrogen":810,"ĠItaly":811,"ĠDiamond":812,"Ġdiscovered":813,"Ġnumber":814,"Ġnatural":815,"ĠIndonesia":816,"ĠMexico":817,"ac":818,"aun":819,"aff":820,"bi":821,"ber":822,"can":823,"dy":824,"ddi":825,"ef":826,"eor":827,"fall":828,"ge":829,"ger":830,"hoc":831,"hington":832,"im":833,"iz":834,"iop":835,"iger":836,"ki":837,"key":838,"lat":839,"les":840,"lia":841,"laci":842,"lesia":843,"many":844,"na":845,"nol":846,"oa":847,"og":848,"oco":849,"pel":850,"pian":851,"ron":852,"rich":853,"sÃ":854,"sch":855,"to":856,"tta":857,"twater":858,"udi":859,"uac":860,"vol":861,"voc":862,"wa":863,"wer":864,"ya":865,"yad":866,"ypt":867,"za":868,"Ġr":869,"Ġre":870,"Ġvol":871,"Ńlia":872,"Ġiron":873,"ate":874,"ĠWas":875,"altwater":876,"Ġcro":877,"Ġchoc":878,"Ġost":879,"ang":880,"enya":881,"erica":882,"ermany":883,"Ġwas":884,"orsch":885,"dii":886,"dile":887,"dium":888,"Ġpas":889,"Ġpiz":890,"ĠTe":891,"ĠTom":892,"ĠTur":893,"ictoria":894,"Ġsus":895,"Ġshar":896,"Ġsaltwater":897,"ĠSp":898,"ĠSw":899,"ĠSil":900,"ĠAn":901,"ĠAbab":902,"ĠAddi":903,"ĠAvoc":904,"Ġflour":905,"Ġfem":906,"Ġfir":907,"Ġflo":908,"Ġfish":909,"rad":910,"ĠCas":911,"ĠCairo":912,"ĠCabb":913,"ĠCoco":914,"Ġbone":915,"Ġborsch":916,"ĠPh":917,"ĠPres":918,"ĠBr":919,"ole":920,"olate":921,"iso":922,"ets":923,"ĠMaun":924,"ĠRaff":925,"Ġglas":926,"Ġguac":927,"ĠGeor":928,"ĠGermany":929,"ĠFe":930,"ĠFall":931,"ĠLag":932,"ĠLake":933,"ĠLoa":934,"ĠEth":935,"Ġwaterfall":936,"amole":937,"ĠNor":938,"ĠNairo":939,"ĠNiger":940,"ado":941,"ĠIs":942,"ĠVictoria":943,"ulf":944,"ago":945,"codile":946,"eau":947,"tadium":948,"ung":949,"ĠOtta":950,"Ġbody":951,"Ġarchi":952,"Ġarnol":953,"ĠBrasÃ":954,"ident":955,"ĠSaudi":956,"ĠCanber":957,"ĠBeij":958,"ĠBeets":959,"ĠEgypt":960,"Ġhuman":961,"ave":962,"ptile":963,"ĠKenya":964,"ĠAmerica":965,"Ġbird":966,"ĠBerlin":967,"ĠRiyad":968,"ĠDelhi":969,"ĠArabia":970,"cano":971,"iopia":972,"lateau":973,"lacier":974,"pelago":975,"Ġreptile":976,"Ġvolcano":977,"ĠWashington":978,"Ġcrocodile":979,"Ġchocolate":980,"Ġostrich":981,"Ġpizza":982,"ĠTomatoes":983,"ĠTurkey":984,"Ġsushi":985,"Ġshark":986,"ĠSilica":987,"ĠAbaba":988,"ĠAddis":989,"ĠAvocado":990,"Ġfemur":991,"Ġfirst":992,"Ġflower":993,"ĠCaspian":994,"ĠCabbage":995,"ĠCocoa":996,"Ġborscht":997,"ĠPresident":998,"ĠMauna":999}