Yaongi/hybridko-exp6

HybriKo - Korean Hybrid LLM

Griffin-inspired hybrid architecture combining RNN and Attention mechanisms.

Model Details

• Architecture: Hybrid RNN + Attention (2:1 ratio)
• Parameters: 117.8M
• Training: Continued pretraining on exp4_plus dataset
• Base Model: exp4 (Wikipedia pretrained)

Training Data

• korean_textbooks_tiny (50K samples)
• korean_textbooks_edu (50K samples)
• korean_public_corpus (50K samples)

Usage

from hybridko.model import HybriKoModel, HybriKoConfig

config = HybriKoConfig.from_yaml("config.yaml")
model = HybriKoModel(config)

# Load checkpoint
checkpoint = torch.load("checkpoints/checkpoint_step_XXX.pt")
model.load_state_dict(checkpoint["model_state_dict"])

Colab

# @title HybriKo-117M (Exp6 - Step 1962) Final
import os
import sys
import yaml
import torch
import torch.nn.functional as F
import sentencepiece as spm
from huggingface_hub import hf_hub_download, list_repo_files

# 1. 의존성 설치 및 설정
!pip install transformers sentencepiece pyyaml huggingface_hub -q

# 설정
REPO_ID = "Yaongi/hybridko-exp6"
CODE_REPO_ID = "Yaongi/HybriKo-117M"
TARGET_CHECKPOINT = "checkpoint_step_1962.pt"
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
print(f"🖥️ 사용 디바이스: {DEVICE}")

# 2. 모델 코드 다운로드
try:
    print(f"📥 모델 코드 다운로드 시도 (from {REPO_ID})...")
    config_py_path = hf_hub_download(REPO_ID, "configuration_hybridko.py")
    model_py_path = hf_hub_download(REPO_ID, "modeling_hybridko.py")
except Exception:
    print(f"⚠️ 코드 파일이 없어 {CODE_REPO_ID}에서 다운로드합니다.")
    config_py_path = hf_hub_download(CODE_REPO_ID, "configuration_hybridko.py")
    model_py_path = hf_hub_download(CODE_REPO_ID, "modeling_hybridko.py")

sys.path.insert(0, os.path.dirname(config_py_path))
try:
    from configuration_hybridko import HybriKoConfig
    from modeling_hybridko import HybriKoModel
except ImportError:
    # Colab Import 오류 방지
    import importlib
    spec = importlib.util.spec_from_file_location("configuration_hybridko", config_py_path)
    configuration_hybridko = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(configuration_hybridko)
    HybriKoConfig = configuration_hybridko.HybriKoConfig
    spec = importlib.util.spec_from_file_location("modeling_hybridko", model_py_path)
    modeling_hybridko = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(modeling_hybridko)
    HybriKoModel = modeling_hybridko.HybriKoModel

# 3. 모델 및 체크포인트 로드
try:
    config_path = hf_hub_download(REPO_ID, "config.yaml")
    with open(config_path, 'r') as f:
        config_data = yaml.safe_load(f)
    config = HybriKoConfig(**config_data)
except:
    config = HybriKoConfig()

model = HybriKoModel(config)

try:
    # 1. checkpoints 폴더 2. 루트 순서 탐색
    try:
        ckpt_path = hf_hub_download(REPO_ID, f"checkpoints/{TARGET_CHECKPOINT}")
    except:
        ckpt_path = hf_hub_download(REPO_ID, TARGET_CHECKPOINT)
        
    checkpoint = torch.load(ckpt_path, map_location=DEVICE)
    state_dict = checkpoint.get("model_state_dict", checkpoint)
    model.load_state_dict(state_dict)
    print(f"✅ 모델 로드 완료: {TARGET_CHECKPOINT}")
except Exception as e:
    print(f"❌ 모델 로드 실패: {e}")

model.to(DEVICE)
model.eval()

# 4. 토크나이저 로드
try:
    try:
        tokenizer_path = hf_hub_download(REPO_ID, "tokenizer/HybriKo_tok.model")
    except:
        tokenizer_path = hf_hub_download(REPO_ID, "HybriKo_tok.model")
    sp = spm.SentencePieceProcessor()
    sp.Load(tokenizer_path)
except Exception as e:
    print(f"❌ 토크나이저 로드 실패: {e}")
    sp = None

# 5. 고급 생성 함수 (Repetition Penalty 지원)
def generate(
    text, 
    max_len=128, 
    temp=0.6,               # 최적값: 0.6
    top_k=50, 
    top_p=0.9,              # 최적값: 0.9
    repetition_penalty=1.15 # 최적값: 1.15
):
    if not sp or not text: return ""
    
    input_ids = torch.tensor([[2] + sp.EncodeAsIds(text)]).to(DEVICE)
    with torch.no_grad():
        for _ in range(max_len):
            idx = input_ids[:, -model.config.max_seq_len:]
            outputs = model(idx)
            logits = outputs.logits[:, -1] / temp

            # 1. Repetition Penalty
            if repetition_penalty > 1.0:
                for i in range(input_ids.shape[1]):
                    token_id = input_ids[0, i]
                    if logits[0, token_id] < 0:
                        logits[0, token_id] *= repetition_penalty
                    else:
                        logits[0, token_id] /= repetition_penalty

            # 2. Top-K
            if top_k is not None:
                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
                logits[logits < v[:, [-1]]] = float("-inf")

            # 3. Top-P
            if top_p is not None:
                sorted_logits, sorted_indices = torch.sort(logits, descending=True)
                cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
                sorted_indices_to_remove = cumulative_probs > top_p
                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
                sorted_indices_to_remove[:, 0] = 0
                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
                logits[indices_to_remove] = float("-inf")

            probs = F.softmax(logits, dim=-1)
            next_token = torch.multinomial(probs, 1)
            
            if next_token.item() == 3: # EOS Token
                break
                
            input_ids = torch.cat([input_ids, next_token], dim=1)

    decoded = sp.DecodeIds(input_ids[0].tolist())
    return decoded
    
print("\n" + "="*30)
print(f"🤖 HybriKo 채팅 시작 (종료: 'q')")
print(f"⚙️ 옵션: Temp={0.6}, Rep_Pen={1.15}")
print("="*30)

while True:
    try:
        user_input = input("\nUser: ")
        if user_input.lower() in ['q', 'quit', 'exit']:
            break
        
        response = generate(user_input)
        print(f"HybriKo: {response}")
    except KeyboardInterrupt:
        break
    except Exception as e:
        print(f"Error: {e}")

Citation

@misc{hybridko2026,
  title={HybriKo: Korean Hybrid LLM},
  year={2026},
}