File size: 3,707 Bytes
4c531a5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | # ---------- veryfy.py ----------
# (주식회사엠케이디 | MKD)
# Compare base vs fine-tuned model generations — Fully compatible with your mkd_core.py
import os
import torch
from mkd_core import MKDCore, MyTokenizer
# ============================================================
# CONFIGURATION
# ============================================================
BASE_DIR = "/home/mkd-s003/AI/finetune"
TOKENIZER_JSON = os.path.join(BASE_DIR, "prod_tokenizer_v2.json")
BASE_MODEL = os.path.join(BASE_DIR, "mkdcore_v2.pt")
# Automatically find the latest fine-tuned model
def find_latest_finetuned():
files = [
f for f in os.listdir(BASE_DIR)
if f.endswith(".pt") and ("finetuned" in f or "segment_finetuned" in f)
]
if not files:
return None
files.sort(key=lambda n: os.path.getmtime(os.path.join(BASE_DIR, n)))
return os.path.join(BASE_DIR, files[-1])
LATEST_MODEL = find_latest_finetuned()
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
# ============================================================
# TOKENIZER
# ============================================================
tok = MyTokenizer(TOKENIZER_JSON)
vocab_size = tok.vocab_size
# ============================================================
# TEXT GENERATION
# ============================================================
@torch.no_grad()
def generate_text(model, prompt, max_new_tokens=60, temperature=0.8):
model.eval()
# ✅ Safe tokenizer handling (works with your MyTokenizer)
try:
ids = tok.encode(prompt)
if not isinstance(ids, list):
ids = ids.ids
except Exception as e:
print(f"⚠️ Encoding error: {e}")
return ""
input_ids = torch.tensor(ids, dtype=torch.long, device=DEVICE).unsqueeze(0)
for _ in range(max_new_tokens):
with torch.amp.autocast("cuda", enabled=torch.cuda.is_available()):
logits = model(input_ids)
probs = torch.softmax(logits[0, -1] / temperature, dim=-1)
next_token = torch.multinomial(probs, 1)
input_ids = torch.cat([input_ids, next_token.unsqueeze(0)], dim=1)
# ✅ Safe decode
try:
return tok.decode(input_ids[0].tolist())
except Exception:
return "⚠️ Decode failed"
# ============================================================
# MODEL LOADING
# ============================================================
def load_model(path):
print(f"🔹 Loading model → {path}")
model = MKDCore(vocab_size).to(DEVICE)
state = torch.load(path, map_location=DEVICE)
model.load_state_dict(state, strict=False)
return model
base_model = load_model(BASE_MODEL)
latest_model = load_model(LATEST_MODEL) if LATEST_MODEL else None
# ============================================================
# TEST PROMPTS
# ============================================================
prompts = [
"Artificial Intelligence will",
"The future of digital technology",
"AI in healthcare and robotics",
"How climate change impacts innovation",
]
# ============================================================
# COMPARISON
# ============================================================
print("\n================ COMPARISON START ================\n")
for prompt in prompts:
print(f"🧩 PROMPT: {prompt}\n")
base_output = generate_text(base_model, prompt)
print(f"--- BASE MODEL OUTPUT ---\n{base_output}\n")
if latest_model:
ft_output = generate_text(latest_model, prompt)
print(f"--- FINE-TUNED MODEL OUTPUT ---\n{ft_output}\n")
print("=" * 80 + "\n")
print("✅ Comparison complete. Observe how the fine-tuned model evolves with each segment.")
|