Upload SFT Chain-of-Thought training script
Browse files- train_sft.py +487 -0
train_sft.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
# =====================================================================
|
| 3 |
+
# 🧠 TURKISH-1B: SFT & REASONING (CHAIN-OF-THOUGHT) EĞİTİM MOTORU
|
| 4 |
+
# Model: Rel0aded/Turkish-1B (1.014 Milyar Parametre)
|
| 5 |
+
# Hedef: Düşünme Yeteneği (<think>...</think>) + Doğal Türkçe Diyalog
|
| 6 |
+
# Geliştirici: Rel0aded / Yağız Gür
|
| 7 |
+
# Ortam: Cloud GPU (Kaggle Dual T4 veya Google Colab T4) - 0 Byte Yerel Yük
|
| 8 |
+
# =====================================================================
|
| 9 |
+
|
| 10 |
+
import os
|
| 11 |
+
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
| 12 |
+
import sys
|
| 13 |
+
import gc
|
| 14 |
+
import json
|
| 15 |
+
import time
|
| 16 |
+
import math
|
| 17 |
+
import base64
|
| 18 |
+
import subprocess
|
| 19 |
+
|
| 20 |
+
# --- 0. GEREKLİ KÜTÜPHANELERİ OTOMATİK KUR ---
|
| 21 |
+
def ensure_installed(pkg, import_name=None):
|
| 22 |
+
if import_name is None:
|
| 23 |
+
import_name = pkg
|
| 24 |
+
try:
|
| 25 |
+
__import__(import_name)
|
| 26 |
+
except ImportError:
|
| 27 |
+
print(f"📦 Otomatik paket kuruluyor: {pkg}...")
|
| 28 |
+
subprocess.run([sys.executable, "-m", "pip", "install", "-q", pkg], check=True)
|
| 29 |
+
|
| 30 |
+
for _pkg, _imp in [
|
| 31 |
+
("bitsandbytes", "bitsandbytes"),
|
| 32 |
+
("datasets", "datasets"),
|
| 33 |
+
("tokenizers", "tokenizers"),
|
| 34 |
+
("wandb", "wandb"),
|
| 35 |
+
("huggingface_hub", "huggingface_hub")
|
| 36 |
+
]:
|
| 37 |
+
ensure_installed(_pkg, _imp)
|
| 38 |
+
|
| 39 |
+
import torch
|
| 40 |
+
import torch.nn as nn
|
| 41 |
+
import torch.nn.functional as F
|
| 42 |
+
from torch.utils.checkpoint import checkpoint
|
| 43 |
+
from datasets import load_dataset
|
| 44 |
+
from tokenizers import Tokenizer
|
| 45 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 46 |
+
import bitsandbytes as bnb
|
| 47 |
+
import wandb
|
| 48 |
+
|
| 49 |
+
# TF32 Hızlandırmaları
|
| 50 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 51 |
+
torch.backends.cudnn.allow_tf32 = True
|
| 52 |
+
|
| 53 |
+
# --- 1. KİMLİK & KİMLİK BİLGİLERİ (Base64 Korumalı) ---
|
| 54 |
+
_dec = lambda s: base64.b64decode(s).decode("utf-8")
|
| 55 |
+
HF_TOKEN = _dec("aGZfQUF5b1J3ZHBnbENRSExGeGhTeURFYUFRamJDUkxmV1pKbQ==")
|
| 56 |
+
REPO_ID = "Rel0aded/Turkish-1B"
|
| 57 |
+
WANDB_API_KEY = _dec("d2FuZGJfdjFfUXZ6eHNPOVdvZk85dElUN3VPWFVBUlo1SnhuX2pPTldOWEhlU2lhNGZteEtpVUE0QktaanMyNzVDelVFVjRTbU54Q0xPRFQzQUFJSVk=")
|
| 58 |
+
|
| 59 |
+
api = HfApi(token=HF_TOKEN)
|
| 60 |
+
|
| 61 |
+
# WandB Başlatma
|
| 62 |
+
wandb_aktif = False
|
| 63 |
+
try:
|
| 64 |
+
wandb.login(key=WANDB_API_KEY)
|
| 65 |
+
wandb.init(
|
| 66 |
+
project="Turkish-1B",
|
| 67 |
+
name="Turkish-1B-SFT-Reasoning-CoT",
|
| 68 |
+
config={
|
| 69 |
+
"architecture": "1.014B Llama-3 GQA",
|
| 70 |
+
"task": "SFT + Chain-of-Thought (<think>...</think>)",
|
| 71 |
+
"learning_rate": 5e-5,
|
| 72 |
+
"batch_size": 2,
|
| 73 |
+
"grad_accum": 8,
|
| 74 |
+
"max_seq_len": 768,
|
| 75 |
+
"target_steps": 800
|
| 76 |
+
}
|
| 77 |
+
)
|
| 78 |
+
wandb_aktif = True
|
| 79 |
+
print("🚀 WandB Canlı Takip Başarıyla Başlatıldı!")
|
| 80 |
+
except Exception as e:
|
| 81 |
+
print(f"⚠️ WandB uyarısı: {e}")
|
| 82 |
+
|
| 83 |
+
# --- 2. 1.014B MODEL MİMARİSİ (Llama-3 Standardı + GQA) ---
|
| 84 |
+
class ModelConfig1B:
|
| 85 |
+
vocab_size = 16384
|
| 86 |
+
dim = 2048
|
| 87 |
+
n_layers = 21 # Tam 1.014 Milyar parametre
|
| 88 |
+
n_heads = 16
|
| 89 |
+
n_kv_heads = 4 # GQA (Grouped Query Attention 16:4)
|
| 90 |
+
head_dim = 128
|
| 91 |
+
intermediate_size = 5632 # SwiGLU MLP
|
| 92 |
+
max_seq_len = 1024 # Bağlam Boyutu
|
| 93 |
+
rope_theta = 500000.0
|
| 94 |
+
|
| 95 |
+
cfg = ModelConfig1B()
|
| 96 |
+
|
| 97 |
+
class RMSNorm(nn.Module):
|
| 98 |
+
def __init__(self, dim, eps=1e-5):
|
| 99 |
+
super().__init__()
|
| 100 |
+
self.eps = eps
|
| 101 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 102 |
+
def forward(self, x):
|
| 103 |
+
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
|
| 104 |
+
|
| 105 |
+
def precompute_rope_frequencies(head_dim, max_seq_len, theta=500000.0):
|
| 106 |
+
theta_factors = theta ** (-torch.arange(0, head_dim, 2).float() / head_dim)
|
| 107 |
+
t = torch.arange(max_seq_len)
|
| 108 |
+
freqs = torch.outer(t, theta_factors)
|
| 109 |
+
return torch.polar(torch.ones_like(freqs), freqs)
|
| 110 |
+
|
| 111 |
+
def apply_rotary_emb(x, freqs_cis):
|
| 112 |
+
x_complex = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
|
| 113 |
+
freqs_cis = freqs_cis.view(1, x.size(1), 1, x_complex.size(-1))
|
| 114 |
+
return torch.view_as_real(x_complex * freqs_cis).flatten(-2).type_as(x)
|
| 115 |
+
|
| 116 |
+
class GQAttention(nn.Module):
|
| 117 |
+
def __init__(self, cfg):
|
| 118 |
+
super().__init__()
|
| 119 |
+
self.n_heads = cfg.n_heads
|
| 120 |
+
self.n_kv_heads = cfg.n_kv_heads
|
| 121 |
+
self.head_dim = cfg.head_dim
|
| 122 |
+
self.num_rep = self.n_heads // self.n_kv_heads
|
| 123 |
+
|
| 124 |
+
self.q_proj = nn.Linear(cfg.dim, self.n_heads * self.head_dim, bias=False)
|
| 125 |
+
self.k_proj = nn.Linear(cfg.dim, self.n_kv_heads * self.head_dim, bias=False)
|
| 126 |
+
self.v_proj = nn.Linear(cfg.dim, self.n_kv_heads * self.head_dim, bias=False)
|
| 127 |
+
self.out_proj = nn.Linear(self.n_heads * self.head_dim, cfg.dim, bias=False)
|
| 128 |
+
|
| 129 |
+
def forward(self, x, freqs_cis):
|
| 130 |
+
B, S, _ = x.shape
|
| 131 |
+
q = self.q_proj(x).view(B, S, self.n_heads, self.head_dim)
|
| 132 |
+
k = self.k_proj(x).view(B, S, self.n_kv_heads, self.head_dim)
|
| 133 |
+
v = self.v_proj(x).view(B, S, self.n_kv_heads, self.head_dim)
|
| 134 |
+
|
| 135 |
+
q = apply_rotary_emb(q, freqs_cis)
|
| 136 |
+
k = apply_rotary_emb(k, freqs_cis)
|
| 137 |
+
|
| 138 |
+
k = k.repeat_interleave(self.num_rep, dim=2)
|
| 139 |
+
v = v.repeat_interleave(self.num_rep, dim=2)
|
| 140 |
+
|
| 141 |
+
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 142 |
+
out = F.scaled_dot_product_attention(q, k, v, is_causal=True)
|
| 143 |
+
out = out.transpose(1, 2).contiguous().view(B, S, -1)
|
| 144 |
+
return self.out_proj(out)
|
| 145 |
+
|
| 146 |
+
class SwiGLUMLP(nn.Module):
|
| 147 |
+
def __init__(self, cfg):
|
| 148 |
+
super().__init__()
|
| 149 |
+
self.gate_proj = nn.Linear(cfg.dim, cfg.intermediate_size, bias=False)
|
| 150 |
+
self.up_proj = nn.Linear(cfg.dim, cfg.intermediate_size, bias=False)
|
| 151 |
+
self.down_proj = nn.Linear(cfg.intermediate_size, cfg.dim, bias=False)
|
| 152 |
+
|
| 153 |
+
def forward(self, x):
|
| 154 |
+
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
|
| 155 |
+
|
| 156 |
+
class TransformerBlock(nn.Module):
|
| 157 |
+
def __init__(self, cfg):
|
| 158 |
+
super().__init__()
|
| 159 |
+
self.attn_norm = RMSNorm(cfg.dim)
|
| 160 |
+
self.attn = GQAttention(cfg)
|
| 161 |
+
self.mlp_norm = RMSNorm(cfg.dim)
|
| 162 |
+
self.mlp = SwiGLUMLP(cfg)
|
| 163 |
+
|
| 164 |
+
def forward(self, x, freqs_cis):
|
| 165 |
+
x = x + self.attn(self.attn_norm(x), freqs_cis)
|
| 166 |
+
x = x + self.mlp(self.mlp_norm(x))
|
| 167 |
+
return x
|
| 168 |
+
|
| 169 |
+
class Rel0aded1B(nn.Module):
|
| 170 |
+
def __init__(self, cfg):
|
| 171 |
+
super().__init__()
|
| 172 |
+
self.cfg = cfg
|
| 173 |
+
self.has_2_gpus = torch.cuda.device_count() > 1
|
| 174 |
+
self.dev0 = "cuda:0" if torch.cuda.is_available() else "cpu"
|
| 175 |
+
self.dev1 = "cuda:1" if self.has_2_gpus else self.dev0
|
| 176 |
+
|
| 177 |
+
self.tok_embeddings = nn.Embedding(cfg.vocab_size, cfg.dim).to(self.dev0)
|
| 178 |
+
self.layers_0 = nn.ModuleList([TransformerBlock(cfg).to(self.dev0) for _ in range(11)])
|
| 179 |
+
self.layers_1 = nn.ModuleList([TransformerBlock(cfg).to(self.dev1) for _ in range(10)])
|
| 180 |
+
self.norm = RMSNorm(cfg.dim).to(self.dev1)
|
| 181 |
+
self.output = nn.Linear(cfg.dim, cfg.vocab_size, bias=False).to(self.dev1)
|
| 182 |
+
|
| 183 |
+
freqs = precompute_rope_frequencies(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta)
|
| 184 |
+
self.register_buffer("freqs_cis", freqs, persistent=False)
|
| 185 |
+
|
| 186 |
+
def forward(self, input_ids):
|
| 187 |
+
S = input_ids.size(1)
|
| 188 |
+
freqs_cis_0 = self.freqs_cis[:S].to(self.dev0)
|
| 189 |
+
freqs_cis_1 = self.freqs_cis[:S].to(self.dev1)
|
| 190 |
+
|
| 191 |
+
h = self.tok_embeddings(input_ids.to(self.dev0))
|
| 192 |
+
for layer in self.layers_0:
|
| 193 |
+
h = checkpoint(layer, h, freqs_cis_0, use_reentrant=False)
|
| 194 |
+
|
| 195 |
+
h = h.to(self.dev1)
|
| 196 |
+
for layer in self.layers_1:
|
| 197 |
+
h = checkpoint(layer, h, freqs_cis_1, use_reentrant=False)
|
| 198 |
+
|
| 199 |
+
h = self.norm(h)
|
| 200 |
+
return self.output(h)
|
| 201 |
+
|
| 202 |
+
# --- 3. MODEL VE CHECKPOINT YÜKLEME ---
|
| 203 |
+
print("🚀 Tokenizer ve 1.014B Model hazırlanıyor...")
|
| 204 |
+
tok_path = hf_hub_download(repo_id=REPO_ID, filename="turkce_tokenizer.json", token=HF_TOKEN)
|
| 205 |
+
tokenizer = Tokenizer.from_file(tok_path)
|
| 206 |
+
|
| 207 |
+
model = Rel0aded1B(cfg)
|
| 208 |
+
toplam_param = sum(p.numel() for p in model.parameters())
|
| 209 |
+
print(f"🧠 Model parametre sayısı: {toplam_param:,} (Tam 1.014 Milyar!)")
|
| 210 |
+
if model.has_2_gpus:
|
| 211 |
+
print("⚡ Çift GPU Model Paralelizmi Devrede (Kaggle Dual T4)!")
|
| 212 |
+
else:
|
| 213 |
+
print(f"⚡ Tek GPU Modu Devrede ({model.dev0})!")
|
| 214 |
+
|
| 215 |
+
# Son Checkpoint'i İndir ve Ağırlıkları Yükle
|
| 216 |
+
print("📥 Step 13,730 checkpoint'i Hugging Face'ten indiriliyor...")
|
| 217 |
+
ckpt_path = hf_hub_download(repo_id=REPO_ID, filename="checkpoint.pt", token=HF_TOKEN)
|
| 218 |
+
ckpt = torch.load(ckpt_path, map_location="cpu")
|
| 219 |
+
state_dict = ckpt["model_state_dict"] if "model_state_dict" in ckpt else ckpt
|
| 220 |
+
|
| 221 |
+
target_dict = model.state_dict()
|
| 222 |
+
adapted_sd = {}
|
| 223 |
+
for k, v in state_dict.items():
|
| 224 |
+
if k.startswith("layers."):
|
| 225 |
+
parts = k.split(".")
|
| 226 |
+
l_idx = int(parts[1])
|
| 227 |
+
rem = ".".join(parts[2:])
|
| 228 |
+
if l_idx < 11:
|
| 229 |
+
adapted_sd[f"layers_0.{l_idx}.{rem}"] = v
|
| 230 |
+
else:
|
| 231 |
+
adapted_sd[f"layers_1.{l_idx - 11}.{rem}"] = v
|
| 232 |
+
else:
|
| 233 |
+
adapted_sd[k] = v
|
| 234 |
+
|
| 235 |
+
matched = {k: v for k, v in adapted_sd.items() if k in target_dict and v.shape == target_dict[k].shape}
|
| 236 |
+
model.load_state_dict(matched, strict=False)
|
| 237 |
+
print(f"✅ Checkpoint başarıyla yüklendi! ({len(matched)} / {len(target_dict)} katman)")
|
| 238 |
+
del ckpt, state_dict, adapted_sd
|
| 239 |
+
gc.collect()
|
| 240 |
+
|
| 241 |
+
# --- 4. SFT & CHAIN-OF-THOUGHT VERİ SETİ HAZIRLIĞI ---
|
| 242 |
+
print("📚 SFT Veri Setleri Birleştiriliyor (Reasoning CoT + Doğal Diyalog)...")
|
| 243 |
+
sft_examples = []
|
| 244 |
+
|
| 245 |
+
# A. AlicanKiraz0/Turkish-CoT-Instruct-Dataset (4.8k yerel CoT verisi)
|
| 246 |
+
try:
|
| 247 |
+
print("🔹 Yükleniyor: Turkish-CoT-Instruct-Dataset...")
|
| 248 |
+
ds_cot1 = load_dataset("AlicanKiraz0/Turkish-CoT-Instruct-Dataset", split="train")
|
| 249 |
+
for row in ds_cot1:
|
| 250 |
+
msgs = row.get("messages", [])
|
| 251 |
+
if len(msgs) >= 3:
|
| 252 |
+
user_text = msgs[1].get("content", "").strip()
|
| 253 |
+
asst_text = msgs[2].get("content", "").strip()
|
| 254 |
+
if user_text and asst_text and ("<think>" in asst_text or len(asst_text) > 30):
|
| 255 |
+
sft_examples.append({"prompt": user_text, "response": asst_text})
|
| 256 |
+
print(f" -> {len(sft_examples):,} CoT örneği eklendi.")
|
| 257 |
+
except Exception as e:
|
| 258 |
+
print(f"⚠️ CoT 1 yüklenemedi: {e}")
|
| 259 |
+
|
| 260 |
+
# B. selimc/OpenThoughts-TR-18k (Analitik Muhakeme İzleri - İlk 3,000 örnek)
|
| 261 |
+
try:
|
| 262 |
+
print("🔹 Yükleniyor: OpenThoughts-TR-18k (Örneklem)...")
|
| 263 |
+
ds_cot2 = load_dataset("selimc/OpenThoughts-TR-18k", split="train[:3000]")
|
| 264 |
+
cnt_cot2 = 0
|
| 265 |
+
for row in ds_cot2:
|
| 266 |
+
prompt = row.get("prompt", "").strip()
|
| 267 |
+
resp = row.get("response", "").strip()
|
| 268 |
+
if prompt and resp:
|
| 269 |
+
resp_formatted = resp.replace("<|begin_of_thought|>", "<think>\n").replace("<|end_of_thought|>", "\n</think>\n")
|
| 270 |
+
if "<think>" not in resp_formatted:
|
| 271 |
+
resp_formatted = f"<think>\nProblemi analiz edip adım adım çözüme ulaşalım.\n</think>\n{resp_formatted}"
|
| 272 |
+
sft_examples.append({"prompt": prompt, "response": resp_formatted})
|
| 273 |
+
cnt_cot2 += 1
|
| 274 |
+
print(f" -> {cnt_cot2:,} OpenThoughts örneği eklendi.")
|
| 275 |
+
except Exception as e:
|
| 276 |
+
print(f"⚠️ CoT 2 yüklenemedi: {e}")
|
| 277 |
+
|
| 278 |
+
# C. merve/turkish_instructions (Doğal Türkçe Sohbet & Akıcılık - İlk 2,500 örnek)
|
| 279 |
+
try:
|
| 280 |
+
print("🔹 Yükleniyor: turkish_instructions (Doğal Sohbet & Diyalog)...")
|
| 281 |
+
ds_chat = load_dataset("merve/turkish_instructions", split="train[:2500]")
|
| 282 |
+
cnt_chat = 0
|
| 283 |
+
for row in ds_chat:
|
| 284 |
+
talimat = row.get("talimat", "").strip()
|
| 285 |
+
cikti = row.get(" çıktı", "") or row.get("cikti", "")
|
| 286 |
+
if talimat and cikti:
|
| 287 |
+
cikti = cikti.strip()
|
| 288 |
+
# Doğal düşünme adımı ekle
|
| 289 |
+
think_text = f"<think>\nKullanıcı doğal bir talimat veya soru sordu. Türkçe dil bilgisine uygun, samimi, anlaşılır ve akıcı bir şekilde yanıt veriyorum.\n</think>\n{cikti}"
|
| 290 |
+
sft_examples.append({"prompt": talimat, "response": think_text})
|
| 291 |
+
cnt_chat += 1
|
| 292 |
+
print(f" -> {cnt_chat:,} Doğal Sohbet örneği eklendi.")
|
| 293 |
+
except Exception as e:
|
| 294 |
+
print(f"⚠️ Chat verisi yüklenemedi: {e}")
|
| 295 |
+
|
| 296 |
+
import random
|
| 297 |
+
random.seed(42)
|
| 298 |
+
random.shuffle(sft_examples)
|
| 299 |
+
print(f"🎯 TOPLAM HAZIRLANAN SFT & CoT ÖRNEĞİ: {len(sft_examples):,}")
|
| 300 |
+
|
| 301 |
+
# --- 5. TOKENİZASYON VE PROMPT MASKING LOADER ---
|
| 302 |
+
MAX_SEQ_LEN = 768
|
| 303 |
+
EOS_ID = 2
|
| 304 |
+
PAD_ID = 0
|
| 305 |
+
|
| 306 |
+
def create_training_batch(examples, batch_size=2):
|
| 307 |
+
batch_input_ids = []
|
| 308 |
+
batch_labels = []
|
| 309 |
+
|
| 310 |
+
for ex in examples:
|
| 311 |
+
prompt_str = f"### Kullanıcı:\n{ex['prompt']}\n\n### Asistan:\n"
|
| 312 |
+
full_str = f"{prompt_str}{ex['response']}</s>"
|
| 313 |
+
|
| 314 |
+
prompt_ids = tokenizer.encode(prompt_str).ids
|
| 315 |
+
full_ids = tokenizer.encode(full_str).ids
|
| 316 |
+
|
| 317 |
+
if len(full_ids) > MAX_SEQ_LEN:
|
| 318 |
+
full_ids = full_ids[:MAX_SEQ_LEN]
|
| 319 |
+
if EOS_ID not in full_ids[-5:]:
|
| 320 |
+
full_ids[-1] = EOS_ID
|
| 321 |
+
|
| 322 |
+
# Prompt kısmını kayıp hesaplamasından dışla (-100 maskesi)
|
| 323 |
+
prefix_len = min(len(prompt_ids), len(full_ids))
|
| 324 |
+
labels = [-100] * prefix_len + full_ids[prefix_len:]
|
| 325 |
+
|
| 326 |
+
# Padding uygula
|
| 327 |
+
pad_len = MAX_SEQ_LEN - len(full_ids)
|
| 328 |
+
if pad_len > 0:
|
| 329 |
+
full_ids = full_ids + [PAD_ID] * pad_len
|
| 330 |
+
labels = labels + [-100] * pad_len
|
| 331 |
+
|
| 332 |
+
batch_input_ids.append(full_ids)
|
| 333 |
+
batch_labels.append(labels)
|
| 334 |
+
|
| 335 |
+
return torch.tensor(batch_input_ids, dtype=torch.long), torch.tensor(batch_labels, dtype=torch.long)
|
| 336 |
+
|
| 337 |
+
# --- 6. EĞİTİM DÖNGÜSÜ (SFT Fine-Tuning) ---
|
| 338 |
+
LR = 5.0e-5 # Taban modeli bozmadan ince ayar
|
| 339 |
+
optimizer = bnb.optim.AdamW8bit(model.parameters(), lr=LR, weight_decay=0.01)
|
| 340 |
+
scaler = torch.amp.GradScaler('cuda')
|
| 341 |
+
|
| 342 |
+
TOTAL_STEPS = 800
|
| 343 |
+
BATCH_SIZE = 2
|
| 344 |
+
GRAD_ACCUM = 8 # Efektif batch size = 16
|
| 345 |
+
LOG_INTERVAL = 20
|
| 346 |
+
SAVE_INTERVAL = 200
|
| 347 |
+
|
| 348 |
+
print(f"\n=======================================================")
|
| 349 |
+
print(f"🔥 SFT CHAIN-OF-THOUGHT EĞİTİMİ BAŞLIYOR!")
|
| 350 |
+
print(f"📊 Adım Sayısı: {TOTAL_STEPS} | Efektif Batch: {BATCH_SIZE * GRAD_ACCUM}")
|
| 351 |
+
print(f"🧠 Düşünme formatı: <think> ... </think> + Doğal Türkçe Cevap")
|
| 352 |
+
print(f"=======================================================\n")
|
| 353 |
+
|
| 354 |
+
model.train()
|
| 355 |
+
optimizer.zero_grad()
|
| 356 |
+
accum_loss = 0.0
|
| 357 |
+
step_start_time = time.time()
|
| 358 |
+
sample_idx = 0
|
| 359 |
+
|
| 360 |
+
for step in range(1, TOTAL_STEPS + 1):
|
| 361 |
+
# Dinamik Cosine LR Zamanlaması
|
| 362 |
+
progress = step / TOTAL_STEPS
|
| 363 |
+
cur_lr = LR * 0.5 * (1.0 + math.cos(math.pi * progress))
|
| 364 |
+
for pg in optimizer.param_groups:
|
| 365 |
+
pg['lr'] = cur_lr
|
| 366 |
+
|
| 367 |
+
step_loss_total = 0.0
|
| 368 |
+
for micro in range(GRAD_ACCUM):
|
| 369 |
+
if sample_idx + BATCH_SIZE > len(sft_examples):
|
| 370 |
+
random.shuffle(sft_examples)
|
| 371 |
+
sample_idx = 0
|
| 372 |
+
|
| 373 |
+
batch_samples = sft_examples[sample_idx:sample_idx + BATCH_SIZE]
|
| 374 |
+
sample_idx += BATCH_SIZE
|
| 375 |
+
|
| 376 |
+
x, y = create_training_batch(batch_samples, BATCH_SIZE)
|
| 377 |
+
x = x.to(model.dev0)
|
| 378 |
+
y = y.to(model.dev1)
|
| 379 |
+
|
| 380 |
+
with torch.amp.autocast('cuda', dtype=torch.float16):
|
| 381 |
+
logits = model(x)
|
| 382 |
+
# targets view (-1), logits view (-1, vocab_size)
|
| 383 |
+
shift_logits = logits[:, :-1, :].contiguous()
|
| 384 |
+
shift_labels = y[:, 1:].contiguous()
|
| 385 |
+
loss = F.cross_entropy(shift_logits.view(-1, cfg.vocab_size), shift_labels.view(-1), ignore_index=-100)
|
| 386 |
+
loss = loss / GRAD_ACCUM
|
| 387 |
+
|
| 388 |
+
scaler.scale(loss).backward()
|
| 389 |
+
step_loss_total += loss.item() * GRAD_ACCUM
|
| 390 |
+
|
| 391 |
+
scaler.unscale_(optimizer)
|
| 392 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 393 |
+
scaler.step(optimizer)
|
| 394 |
+
scaler.update()
|
| 395 |
+
optimizer.zero_grad()
|
| 396 |
+
|
| 397 |
+
accum_loss += step_loss_total
|
| 398 |
+
|
| 399 |
+
if step % LOG_INTERVAL == 0:
|
| 400 |
+
avg_loss = accum_loss / LOG_INTERVAL
|
| 401 |
+
elapsed = time.time() - step_start_time
|
| 402 |
+
sec_per_step = elapsed / LOG_INTERVAL
|
| 403 |
+
step_start_time = time.time()
|
| 404 |
+
accum_loss = 0.0
|
| 405 |
+
|
| 406 |
+
print(f"📍 Step [{step:03d}/{TOTAL_STEPS}] | SFT Loss: {avg_loss:.4f} | LR: {cur_lr:.2e} | {sec_per_step:.2f} s/step")
|
| 407 |
+
|
| 408 |
+
if wandb_aktif:
|
| 409 |
+
wandb.log({
|
| 410 |
+
"sft_step": step,
|
| 411 |
+
"sft_loss": avg_loss,
|
| 412 |
+
"learning_rate": cur_lr,
|
| 413 |
+
"sec_per_step": sec_per_step
|
| 414 |
+
})
|
| 415 |
+
|
| 416 |
+
# Ara Checkpoint Kaydet
|
| 417 |
+
if step % SAVE_INTERVAL == 0 or step == TOTAL_STEPS:
|
| 418 |
+
print(f"\n💾 Model Kaydediliyor (Step {step})...")
|
| 419 |
+
save_path = "turkish_1b_instruct_cot.pt"
|
| 420 |
+
|
| 421 |
+
# Ağırlıkları FP16 olarak kaydet (Hafif ve ultra hızlı yükleme: ~2.03 GB)
|
| 422 |
+
fp16_dict = {k: v.half().cpu() for k, v in model.state_dict().items()}
|
| 423 |
+
torch.save({
|
| 424 |
+
"step": step,
|
| 425 |
+
"model_state_dict": fp16_dict,
|
| 426 |
+
"loss": avg_loss if 'avg_loss' in locals() else step_loss_total,
|
| 427 |
+
"architecture": "Rel0aded1B-SFT-CoT"
|
| 428 |
+
}, save_path)
|
| 429 |
+
|
| 430 |
+
print("☁️ Hugging Face Kasasına Yükleniyor...")
|
| 431 |
+
try:
|
| 432 |
+
api.upload_file(
|
| 433 |
+
path_or_fileobj=save_path,
|
| 434 |
+
path_in_repo=save_path,
|
| 435 |
+
repo_id=REPO_ID,
|
| 436 |
+
commit_message=f"Upload SFT Reasoning model checkpoint (Step {step}, Loss {avg_loss:.4f})"
|
| 437 |
+
)
|
| 438 |
+
print(f"✅ Hugging Face'e yüklendi: {REPO_ID}/{save_path}")
|
| 439 |
+
except Exception as ue:
|
| 440 |
+
print(f"⚠️ Yükleme uyarısı: {ue}")
|
| 441 |
+
|
| 442 |
+
print("\n" + "="*60)
|
| 443 |
+
print("🎉 SFT & REASONING EĞİTİMİ BAŞARIYLA TAMAMLANDI!")
|
| 444 |
+
print("="*60)
|
| 445 |
+
|
| 446 |
+
# --- 7. CANLI TEST & AKIL YÜRÜTME ÜRETİMİ (INFERENCE TEST) ---
|
| 447 |
+
print("\n🧪 Model Düşünme & Cevap Üretim Testi Yapılıyor...")
|
| 448 |
+
model.eval()
|
| 449 |
+
|
| 450 |
+
test_sorulari = [
|
| 451 |
+
"Merhaba! Kendini tanıtır mısın, sen kimsin ve ne yapabilirsin?",
|
| 452 |
+
"Bir göletteki nilüfer yaprakları her gün 2 katına çıkıyor. Gölet 48 günde tamamen doluyorsa, göletin yarısı kaçıncı günde dolmuştur?",
|
| 453 |
+
"Bana yapay zekanın geleceği hakkında kısa, doğal ve samimi bir görüş belirtir misin?"
|
| 454 |
+
]
|
| 455 |
+
|
| 456 |
+
for soru in test_sorulari:
|
| 457 |
+
print(f"\n❓ Soru: {soru}")
|
| 458 |
+
prompt = f"### Kullanıcı:\n{soru}\n\n### Asistan:\n<think>\n"
|
| 459 |
+
inp_ids = torch.tensor([tokenizer.encode(prompt).ids], dtype=torch.long).to(model.dev0)
|
| 460 |
+
|
| 461 |
+
generated = list(inp_ids[0].cpu().numpy())
|
| 462 |
+
with torch.no_grad():
|
| 463 |
+
for _ in range(256):
|
| 464 |
+
cur_inp = torch.tensor([generated[-cfg.max_seq_len:]], dtype=torch.long).to(model.dev0)
|
| 465 |
+
logits = model(cur_inp)
|
| 466 |
+
next_logits = logits[0, -1, :] / 0.7 # Temperature = 0.7
|
| 467 |
+
|
| 468 |
+
# Top-p (nucleus sampling)
|
| 469 |
+
probs = F.softmax(next_logits, dim=-1)
|
| 470 |
+
next_token = torch.multinomial(probs, num_samples=1).item()
|
| 471 |
+
|
| 472 |
+
generated.append(next_token)
|
| 473 |
+
if next_token == EOS_ID:
|
| 474 |
+
break
|
| 475 |
+
|
| 476 |
+
out_text = tokenizer.decode(generated)
|
| 477 |
+
# Sadece asistan cevabını göster
|
| 478 |
+
if "### Asistan:\n" in out_text:
|
| 479 |
+
resp = out_text.split("### Asistan:\n")[1].replace("</s>", "").strip()
|
| 480 |
+
print(f"🤖 Yanıt:\n{resp}")
|
| 481 |
+
else:
|
| 482 |
+
print(f"🤖 Yanıt:\n{out_text}")
|
| 483 |
+
print("-" * 50)
|
| 484 |
+
|
| 485 |
+
if wandb_aktif:
|
| 486 |
+
wandb.finish()
|
| 487 |
+
print("\n🏁 Tüm süreç tamamlandı!")
|