Upload folder using huggingface_hub

.gitignore

@@ -3,3 +3,4 @@ __pycache__/
 out/
 *.pt
 .git/
+data/redacted/

README.md

@@ -1,3 +1,63 @@
----
-license: unknown
----
+---
+language:
+- id
+license: mit
+tags:
+- text-generation
+- from-scratch
+- gpt
+- transformer
+- indonesian
+datasets:
+- custom
+---
+
+# Indigo
+
+Model bahasa kecil GPT-style yang dibangun **dari nol** (tanpa library transformers) sebagai proyek pembelajaran. Dual-backend: **PyTorch** dan **TensorFlow/Keras**, dengan format bobot aman `.safetensors`.
+
+## Arsitektur
+
+| | Nilai default |
+|---|---|
+| Tipe | Decoder-only transformer (pre-LN) |
+| Parameter | ~0.81M |
+| Layer / Head | 4 / 4 |
+| Dimensi | 128 |
+| Konteks | 96 token |
+| Tokenizer | Level karakter (~90 vocab) |
+
+## File penting
+
+```
+indigo/model.py        arsitektur PyTorch (SDPA)
+indigo/model_keras.py  arsitektur Keras 3
+train.py / train_tf.py training per backend
+generate.py / generate_tf.py  generasi teks
+out/indigo_best.safetensors   bobot terbaik + _meta.json
+```
+
+## Cara pakai
+
+```bash
+pip install -r requirements.txt   # torch (+ tensorflow opsional)
+
+# PyTorch
+python train.py --data data/sample.txt --steps 2000
+python generate.py --prompt "Indigo" --max-new 300
+
+# TensorFlow/Keras
+python train_tf.py --data data/sample.txt --out out_tf
+python generate_tf.py --prompt "Indigo"
+```
+
+Fitur: pembersihan teks otomatis, split validasi per-file, best-checkpoint, resume (`--init-from`), cosine LR + warmup, top-k sampling, pilihan `--device`.
+
+## Batasan
+
+- Dilatih pada data sangat kecil (~16 ribu token) → output belum koheren, cocok untuk edukasi bukan produksi.
+- Checkpoint berlabel `_best` dipilih berdasarkan validasi; gunakan itu, bukan checkpoint akhir.
+
+## Keamanan
+
+Bobot disimpan sebagai `.safetensors` (tanpa pickle, tidak mengeksekusi kode saat dimuat). State optimizer (`*_optimizer.pt`) hanya untuk resume lokal — jangan dibagikan.

generate.py

@@ -1,4 +1,6 @@
 import argparse
+import json
+import os
 
 import torch
 
@@ -6,26 +8,41 @@ from indigo.model import GPT, GPTConfig
 from indigo.tokenizer import CharTokenizer
 
 
+def load_model(path):
+    if path.endswith(".safetensors"):
+        from safetensors.torch import load_file
+
+        state = load_file(path)
+        with open(os.path.splitext(path)[0] + "_meta.json", encoding="utf-8") as f:
+            meta = json.load(f)
+        return state, meta["config"], meta["vocab"]
+    ckpt = torch.load(path, map_location="cpu", weights_only=True)
+    return ckpt["model"], ckpt["config"], ckpt["vocab"]
+
+
 def main():
     parser = argparse.ArgumentParser(description="Generate teks dari checkpoint Indigo")
-    parser.add_argument("--ckpt", default="out/indigo.pt")
+    parser.add_argument("--ckpt", default="out/indigo_best.safetensors")
     parser.add_argument("--prompt", default="")
     parser.add_argument("--max-new", type=int, default=300)
     parser.add_argument("--temperature", type=float, default=0.8)
     parser.add_argument("--top-k", type=int, default=40)
     parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda"])
     args = parser.parse_args()
 
     if args.seed is not None:
         torch.manual_seed(args.seed)
+    device = "cuda" if torch.cuda.is_available() else "cpu" if args.device == "auto" else args.device
 
-    ckpt = torch.load(args.ckpt, map_location="cpu", weights_only=True)
-    model = GPT(GPTConfig(**ckpt["config"]))
-    model.load_state_dict(ckpt["model"])
-    tokenizer = CharTokenizer(ckpt["vocab"])
+    state, config_d, vocab = load_model(args.ckpt)
+    model = GPT(GPTConfig(**config_d))
+    model.load_state_dict(state, strict=False)
+    model = model.to(device)
+    tokenizer = CharTokenizer(vocab)
 
     ids = tokenizer.encode(args.prompt) or [0]
-    idx = torch.tensor([ids], dtype=torch.long)
+    idx = torch.tensor([ids], dtype=torch.long, device=device)
     out = model.generate(idx, args.max_new, temperature=args.temperature, top_k=args.top_k)
     print(tokenizer.decode(out[0].tolist()))
 

generate_tf.py

@@ -0,0 +1,68 @@
+import argparse
+import os
+import random
+
+import numpy as np
+import tensorflow as tf
+
+from indigo.common import load_meta
+from indigo.model_keras import build_gpt, generate
+from indigo.tokenizer import CharTokenizer
+
+CONFIG_KEYS = ("vocab_size", "block_size", "n_layer", "n_head", "n_embd", "dropout")
+
+
+def load_model(path):
+    from safetensors.numpy import load_file
+
+    meta = load_meta(path)
+    if meta.get("backend") != "tensorflow":
+        raise SystemExit(
+            f"{path} berasal dari backend {meta.get('backend')}, gunakan generate.py (PyTorch)"
+        )
+    state = load_file(path)
+    by_path = {k.replace("/", "_"): v for k, v in state.items()}
+    model = build_gpt(**{k: meta["config"][k] for k in CONFIG_KEYS})
+    missing = [v.path for v in model.weights if v.path.replace("/", "_") not in by_path]
+    if missing:
+        raise SystemExit(f"bobot tidak cocok dengan checkpoint: {missing[:5]}")
+    model.set_weights([by_path[v.path.replace("/", "_")] for v in model.weights])
+    return model, meta
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate teks dari checkpoint Indigo (TensorFlow)")
+    parser.add_argument("--ckpt", default="out_tf/indigo_best.safetensors")
+    parser.add_argument("--prompt", default="")
+    parser.add_argument("--max-new", type=int, default=300)
+    parser.add_argument("--temperature", type=float, default=0.8)
+    parser.add_argument("--top-k", type=int, default=40)
+    parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument("--device", default="auto", choices=["auto", "cpu", "gpu"])
+    args = parser.parse_args()
+
+    if args.device == "cpu":
+        tf.config.set_visible_devices([], "GPU")
+    if args.seed is not None:
+        random.seed(args.seed)
+        np.random.seed(args.seed)
+        tf.random.set_seed(args.seed)
+
+    model, meta = load_model(args.ckpt)
+    tokenizer = CharTokenizer(meta["vocab"])
+
+    ids = tokenizer.encode(args.prompt) or [0]
+    idx = tf.constant([ids], dtype=tf.int64)
+    out = generate(
+        model,
+        idx,
+        args.max_new,
+        block_size=meta["config"]["block_size"],
+        temperature=args.temperature,
+        top_k=args.top_k,
+    )
+    print(tokenizer.decode(out.numpy()[0].tolist()))
+
+
+if __name__ == "__main__":
+    main()

indigo/common.py

@@ -0,0 +1,45 @@
+import json
+import os
+import re
+
+DECOR_LINE = re.compile(r"^[\s=\-_~*#.]{4,}$")
+
+
+def clean_text(text):
+    lines = [ln for ln in text.splitlines() if not DECOR_LINE.match(ln)]
+    text = "\n".join(lines)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip() + "\n"
+
+
+def collect_text_files(paths):
+    files = []
+    for p in paths:
+        if os.path.isdir(p):
+            for root, _, names in os.walk(p):
+                files.extend(os.path.join(root, n) for n in sorted(names) if n.lower().endswith(".txt"))
+        else:
+            files.append(p)
+    return sorted(files)
+
+
+def read_clean(path):
+    with open(path, encoding="utf-8") as f:
+        return clean_text(f.read())
+
+
+def save_meta(base_path, config, vocab, step, val_loss, backend):
+    meta = {
+        "config": config,
+        "vocab": vocab,
+        "step": step,
+        "val_loss": val_loss,
+        "backend": backend,
+    }
+    with open(os.path.splitext(base_path)[0] + "_meta.json", "w", encoding="utf-8") as f:
+        json.dump(meta, f, ensure_ascii=False)
+
+
+def load_meta(path):
+    with open(os.path.splitext(path)[0] + "_meta.json", encoding="utf-8") as f:
+        return json.load(f)

indigo/model.py

@@ -1,4 +1,3 @@
-import math
 from dataclasses import dataclass
 
 import torch
@@ -27,8 +26,6 @@ class CausalSelfAttention(nn.Module):
         self.proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
         self.attn_dropout = nn.Dropout(config.dropout)
         self.resid_dropout = nn.Dropout(config.dropout)
-        mask = torch.tril(torch.ones(config.block_size, config.block_size))
-        self.register_buffer("mask", mask.view(1, 1, config.block_size, config.block_size))
 
     def forward(self, x):
         B, T, C = x.shape
@@ -36,11 +33,12 @@ class CausalSelfAttention(nn.Module):
         q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
         k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
         v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
-        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
-        att = att.masked_fill(self.mask[:, :, :T, :T] == 0, float("-inf"))
-        att = F.softmax(att, dim=-1)
-        att = self.attn_dropout(att)
-        y = (att @ v).transpose(1, 2).contiguous().view(B, T, C)
+        y = F.scaled_dot_product_attention(
+            q, k, v,
+            dropout_p=self.attn_dropout.p if self.training else 0.0,
+            is_causal=True,
+        )
+        y = y.transpose(1, 2).contiguous().view(B, T, C)
         return self.resid_dropout(self.proj(y))
 
 

indigo/model_keras.py

@@ -0,0 +1,61 @@
+import tensorflow as tf
+from keras import layers
+
+
+class PositionEmbedding(layers.Layer):
+    def __init__(self, block_size, **kwargs):
+        super().__init__(**kwargs)
+        self.block_size = block_size
+
+    def build(self, input_shape):
+        self.pos_emb = self.add_weight(
+            name="pos_emb", shape=(self.block_size, input_shape[-1]), initializer="random_normal"
+        )
+
+    def call(self, x):
+        T = tf.shape(x)[1]
+        return x + self.pos_emb[tf.newaxis, :T, :]
+
+    def get_config(self):
+        config = super().get_config()
+        config["block_size"] = self.block_size
+        return config
+
+
+def build_gpt(vocab_size, block_size, n_layer=4, n_head=4, n_embd=128, dropout=0.1, name="indigo"):
+    tokens = tf.keras.Input(shape=(None,), dtype="int64", name="tokens")
+    x = layers.Embedding(vocab_size, n_embd, name="tok_emb")(tokens)
+    x = PositionEmbedding(block_size, name="pos_emb")(x)
+    x = layers.Dropout(dropout)(x)
+    for i in range(n_layer):
+        xn = layers.LayerNormalization(epsilon=1e-5, name=f"ln1_{i}")(x)
+        attn = layers.MultiHeadAttention(
+            num_heads=n_head, key_dim=n_embd // n_head, dropout=dropout, name=f"attn_{i}"
+        )
+        x = x + attn(xn, xn, use_causal_mask=True)
+        xn = layers.LayerNormalization(epsilon=1e-5, name=f"ln2_{i}")(x)
+        h = layers.Dense(4 * n_embd, activation="gelu", name=f"fc_{i}")(xn)
+        h = layers.Dense(n_embd, name=f"proj_{i}")(h)
+        h = layers.Dropout(dropout)(h)
+        x = x + h
+    x = layers.LayerNormalization(epsilon=1e-5, name="ln_f")(x)
+    logits = layers.Dense(vocab_size, use_bias=False, name="head")(x)
+    return tf.keras.Model(tokens, logits, name=name)
+
+
+def generate(model, idx, max_new_tokens, block_size, temperature=1.0, top_k=None):
+    for _ in range(max_new_tokens):
+        idx_cond = idx[:, -block_size:]
+        logits = model(idx_cond, training=False)[:, -1, :]
+        logits = logits / max(temperature, 1e-8)
+        if top_k is not None:
+            k = min(top_k, int(logits.shape[-1]))
+            vals, _ = tf.math.top_k(logits, k=k)
+            logits = tf.where(
+                logits < vals[:, -1:],
+                tf.fill(tf.shape(logits), tf.float32.min),
+                logits,
+            )
+        next_id = tf.random.categorical(logits, num_samples=1, dtype=tf.int64)
+        idx = tf.concat([idx, next_id], axis=1)
+    return idx

out/indigo.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8438242a855ba259d59d07b7e8a9a29461311c2f008993cb20d5e9f5828dbc72
+size 3294144

out/indigo_best.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2f67a103c33e00f21bb78c13716b70ff97c0de33352a27f0b6fff8c01c480ab
+size 3294144

out/indigo_best_meta.json

@@ -0,0 +1 @@
+{"config": {"vocab_size": 90, "block_size": 96, "n_layer": 4, "n_head": 4, "n_embd": 128, "dropout": 0.1, "bias": false}, "vocab": ["\n", " ", "!", "\"", "#", "%", "&", "'", "(", ")", "+", ",", "-", ".", "/", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", ">", "?", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "]", "_", "`", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "{", "|", "}", "°"], "step": 900, "val_loss": 3.390139579772949}

out/indigo_meta.json

@@ -0,0 +1 @@
+{"config": {"vocab_size": 90, "block_size": 96, "n_layer": 4, "n_head": 4, "n_embd": 128, "dropout": 0.1, "bias": false}, "vocab": ["\n", " ", "!", "\"", "#", "%", "&", "'", "(", ")", "+", ",", "-", ".", "/", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", ">", "?", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "]", "_", "`", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "{", "|", "}", "°"], "step": 900, "val_loss": 3.423954129219055}

requirements.txt

@@ -1 +1,3 @@
 torch>=2.0
+safetensors>=0.4
+tensorflow>=2.16

train.py

@@ -1,14 +1,35 @@
 import argparse
 import math
 import os
+import random
 import time
 
 import torch
+from safetensors.torch import save_file
 
+from indigo.common import collect_text_files, load_meta, read_clean, save_meta
 from indigo.model import GPT, GPTConfig
 from indigo.tokenizer import CharTokenizer
 
 
+def load_init(path):
+    if path.endswith(".safetensors"):
+        from safetensors.torch import load_file
+
+        state = load_file(path)
+        meta = load_meta(path)
+        opt_path = os.path.splitext(path)[0].replace("_best", "") + "_optimizer.pt"
+        opt = None
+        if os.path.exists(opt_path):
+            try:
+                opt = torch.load(opt_path, map_location="cpu", weights_only=True)
+            except Exception as e:
+                print(f"optimizer state dilewati: {e}")
+        return state, meta["config"], meta.get("step", 0), opt
+    ckpt = torch.load(path, map_location="cpu", weights_only=True)
+    return ckpt["model"], ckpt["config"], ckpt.get("step", 0), ckpt.get("optimizer")
+
+
 def get_batch(data, block_size, batch_size, device):
     ix = torch.randint(len(data) - block_size - 1, (batch_size,))
     x = torch.stack([data[i : i + block_size] for i in ix])
@@ -30,7 +51,7 @@ def estimate_loss(model, data, args, device):
 
 def main():
     parser = argparse.ArgumentParser(description="Latih model Indigo dari scratch")
-    parser.add_argument("--data", default="data/sample.txt", help="path file teks untuk training")
+    parser.add_argument("--data", nargs="+", default=["data/sample.txt"], help="path file/folder teks untuk training")
     parser.add_argument("--out", default="out", help="folder output checkpoint")
     parser.add_argument("--steps", type=int, default=2000)
     parser.add_argument("--batch-size", type=int, default=32)
@@ -45,80 +66,129 @@ def main():
     parser.add_argument("--eval-interval", type=int, default=200)
     parser.add_argument("--eval-iters", type=int, default=20)
     parser.add_argument("--seed", type=int, default=1337)
+    parser.add_argument("--init-from", default=None, help="checkpoint untuk melanjutkan training")
+    parser.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda"])
+    parser.add_argument("--val-fraction", type=float, default=0.1, help="proporsi file untuk validasi")
     args = parser.parse_args()
 
     torch.manual_seed(args.seed)
-    device = "cuda" if torch.cuda.is_available() else "cpu"
+    if args.device == "auto":
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+    else:
+        device = args.device
     os.makedirs(args.out, exist_ok=True)
 
-    with open(args.data, encoding="utf-8") as f:
-        text = f.read()
-
-    tokenizer = CharTokenizer.from_text(text)
-    data = torch.tensor(tokenizer.encode(text), dtype=torch.long)
-    if len(data) < args.block_size * 2:
-        raise SystemExit(f"data terlalu pendek ({len(data)} token), minimal {args.block_size * 2}")
-    n = int(0.9 * len(data))
-    train_data, val_data = data[:n], data[n:]
-
-    config = GPTConfig(
-        vocab_size=tokenizer.vocab_size,
-        block_size=args.block_size,
-        n_layer=args.n_layer,
-        n_head=args.n_head,
-        n_embd=args.n_embd,
-        dropout=args.dropout,
+    paths = collect_text_files(args.data)
+    if not paths:
+        raise SystemExit("tidak ada file teks ditemukan")
+
+    files = sorted(paths)
+    rng = random.Random(args.seed)
+    rng.shuffle(files)
+    n_val = max(1, round(len(files) * args.val_fraction)) if len(files) > 1 else 0
+    print(f"file latih={len(files) - n_val} | file validasi={n_val}")
+
+    train_text = "".join(read_clean(p) for p in files[n_val:])
+    val_text = "".join(read_clean(p) for p in files[:n_val])
+
+    tokenizer = CharTokenizer.from_text(train_text + val_text)
+    train_data = torch.tensor(tokenizer.encode(train_text), dtype=torch.long)
+    val_data = torch.tensor(tokenizer.encode(val_text), dtype=torch.long)
+    if len(train_data) < args.block_size * 2:
+        raise SystemExit(f"data latih terlalu pendek ({len(train_data)} token), minimal {args.block_size * 2}")
+    print(
+        f"tokens latih={len(train_data):,} | tokens validasi={len(val_data):,} | vocab={tokenizer.vocab_size}"
     )
-    model = GPT(config).to(device)
+
+    init_state = None
+    init_opt = None
+    start_step = 0
+    if args.init_from:
+        init_state, init_config, start_step, init_opt = load_init(args.init_from)
+        config = GPTConfig(**init_config)
+        print(f"melanjutkan dari {args.init_from} (step {start_step})")
+    else:
+        config = GPTConfig(
+            vocab_size=tokenizer.vocab_size,
+            block_size=args.block_size,
+            n_layer=args.n_layer,
+            n_head=args.n_head,
+            n_embd=args.n_embd,
+            dropout=args.dropout,
+        )
+    if config.vocab_size != tokenizer.vocab_size:
+        raise SystemExit(
+            f"vocab tidak cocok: checkpoint={config.vocab_size}, data={tokenizer.vocab_size}"
+        )
+    model = GPT(config)
+    if init_state is not None:
+        missing, unexpected = model.load_state_dict(init_state, strict=False)
+        if missing or unexpected:
+            print(f"state_dict: missing={missing} unexpected={unexpected}")
+    model = model.to(device)
+    total_steps = start_step + args.steps
     print(
         f"device={device} | params={model.num_params() / 1e6:.2f}M | "
-        f"vocab={tokenizer.vocab_size} | tokens={len(data):,}"
+        f"vocab={tokenizer.vocab_size} | total_steps={total_steps}"
     )
 
     optimizer = torch.optim.AdamW(
         model.parameters(), lr=args.lr, betas=(0.9, 0.95), weight_decay=args.weight_decay
     )
+    if init_opt is not None:
+        try:
+            optimizer.load_state_dict(init_opt)
+            print("state optimizer dipulihkan")
+        except Exception as e:
+            print(f"optimizer state dilewati: {e}")
+
+    def save_model(base_path, val_loss):
+        tensors = {k: v.detach().clone().contiguous() for k, v in model.state_dict().items()}
+        save_file(tensors, base_path)
+        save_meta(base_path, config.__dict__, tokenizer.itos, total_steps, val_loss, backend="pytorch")
 
     def lr_at(step):
         if step < args.warmup:
             return args.lr * (step + 1) / args.warmup
-        progress = (step - args.warmup) / max(1, args.steps - args.warmup)
+        progress = (step - args.warmup) / max(1, total_steps - args.warmup)
         return 0.1 * args.lr + 0.45 * args.lr * (1 + math.cos(math.pi * progress))
 
+    best_val = float("inf")
+    last_val = None
     model.train()
     t0 = time.time()
-    for step in range(args.steps):
+    for step in range(start_step, total_steps):
         lr = lr_at(step)
         for g in optimizer.param_groups:
             g["lr"] = lr
-        x, y = get_batch(train_data, args.block_size, args.batch_size, device)
+        x, y = get_batch(train_data, config.block_size, args.batch_size, device)
         _, loss = model(x, y)
         optimizer.zero_grad(set_to_none=True)
         loss.backward()
         torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
         optimizer.step()
 
-        if step % args.eval_interval == 0 or step == args.steps - 1:
-            if len(val_data) > args.block_size + 1:
+        if step % args.eval_interval == 0 or step == total_steps - 1:
+            if len(val_data) > config.block_size + 1:
                 val_loss = estimate_loss(model, val_data, args, device)
-                val_str = f"{val_loss:.4f}"
+                marker = ""
+                if val_loss < best_val:
+                    best_val = val_loss
+                    save_model(os.path.join(args.out, "indigo_best.safetensors"), val_loss)
+                    marker = " <- best"
+                last_val = val_loss
+                val_str = f"{val_loss:.4f}{marker}"
             else:
                 val_str = "n/a"
             print(
-                f"step {step:5d}/{args.steps} | lr {lr:.2e} | "
+                f"step {step + 1:5d}/{total_steps} | lr {lr:.2e} | "
                 f"loss {loss.item():.4f} | val {val_str} | {time.time() - t0:.1f}s"
             )
 
-    ckpt_path = os.path.join(args.out, "indigo.pt")
-    torch.save(
-        {
-            "model": model.state_dict(),
-            "config": config.__dict__,
-            "vocab": tokenizer.itos,
-        },
-        ckpt_path,
-    )
-    print(f"checkpoint tersimpan di {ckpt_path}")
+    final_path = os.path.join(args.out, "indigo.safetensors")
+    save_model(final_path, last_val)
+    torch.save(optimizer.state_dict(), os.path.join(args.out, "indigo_optimizer.pt"))
+    print(f"model tersimpan di {final_path} (+_meta.json, indigo_optimizer.pt)")
 
 
 if __name__ == "__main__":

train_tf.py

@@ -0,0 +1,197 @@
+import argparse
+import math
+import os
+import random
+import time
+
+import numpy as np
+import tensorflow as tf
+from safetensors.numpy import load_file, save_file
+
+from indigo.common import collect_text_files, load_meta, read_clean, save_meta
+from indigo.model_keras import build_gpt
+from indigo.tokenizer import CharTokenizer
+
+CONFIG_KEYS = ("vocab_size", "block_size", "n_layer", "n_head", "n_embd", "dropout")
+
+
+def get_batch(np_data, block_size, batch_size):
+    ix = np.random.randint(0, len(np_data) - block_size - 1, size=batch_size)
+    x = np.stack([np_data[i : i + block_size] for i in ix])
+    y = np.stack([np_data[i + 1 : i + block_size + 1] for i in ix])
+    return tf.constant(x), tf.constant(y)
+
+
+@tf.function(reduce_retracing=True)
+def train_step(model, optimizer, loss_fn, x, y):
+    with tf.GradientTape() as tape:
+        logits = model(x, training=True)
+        loss = loss_fn(y, logits)
+    grads = tape.gradient(loss, model.trainable_variables)
+    optimizer.apply_gradients(zip(grads, model.trainable_variables))
+    return loss
+
+
+@tf.function(reduce_retracing=True)
+def eval_step(model, loss_fn, x, y):
+    logits = model(x, training=False)
+    return loss_fn(y, logits)
+
+
+@tf.function(reduce_retracing=True)
+def forward_last(model, x):
+    return model(x, training=False)[:, -1, :]
+
+
+def save_weights_tf(model, base_path, config, vocab, step, val_loss):
+    tensors = {v.path: np.asarray(v) for v in model.weights}
+    save_file(tensors, base_path)
+    save_meta(base_path, config, vocab, step, val_loss, backend="tensorflow")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Latih model Indigo (backend TensorFlow/Keras)")
+    parser.add_argument("--data", nargs="+", default=["data/sample.txt"])
+    parser.add_argument("--out", default="out_tf")
+    parser.add_argument("--steps", type=int, default=2000)
+    parser.add_argument("--batch-size", type=int, default=32)
+    parser.add_argument("--block-size", type=int, default=128)
+    parser.add_argument("--n-layer", type=int, default=4)
+    parser.add_argument("--n-head", type=int, default=4)
+    parser.add_argument("--n-embd", type=int, default=128)
+    parser.add_argument("--dropout", type=float, default=0.1)
+    parser.add_argument("--lr", type=float, default=3e-4)
+    parser.add_argument("--warmup", type=int, default=100)
+    parser.add_argument("--weight-decay", type=float, default=0.1)
+    parser.add_argument("--eval-interval", type=int, default=200)
+    parser.add_argument("--eval-iters", type=int, default=20)
+    parser.add_argument("--seed", type=int, default=1337)
+    parser.add_argument("--init-from", default=None, help="checkpoint safetensors dari backend TF")
+    parser.add_argument("--device", default="auto", choices=["auto", "cpu", "gpu"])
+    args = parser.parse_args()
+
+    if args.device == "cpu":
+        tf.config.set_visible_devices([], "GPU")
+    gpus = tf.config.list_physical_devices("GPU")
+    device_label = f"gpu({len(gpus)})" if gpus and args.device != "cpu" else "cpu"
+
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    tf.random.set_seed(args.seed)
+    os.makedirs(args.out, exist_ok=True)
+
+    files = sorted(collect_text_files(args.data))
+    if not files:
+        raise SystemExit("tidak ada file teks ditemukan")
+    rng = random.Random(args.seed)
+    rng.shuffle(files)
+    n_val = max(1, round(len(files) * 0.1)) if len(files) > 1 else 0
+    print(f"file latih={len(files) - n_val} | file validasi={n_val}")
+
+    train_text = "".join(read_clean(p) for p in files[n_val:])
+    val_text = "".join(read_clean(p) for p in files[:n_val])
+
+    tokenizer = CharTokenizer.from_text(train_text + val_text)
+    train_np = np.array(tokenizer.encode(train_text), dtype=np.int64)
+    val_np = np.array(tokenizer.encode(val_text), dtype=np.int64)
+    if len(train_np) < args.block_size * 2:
+        raise SystemExit(f"data latih terlalu pendek ({len(train_np)} token)")
+    print(
+        f"tokens latih={len(train_np):,} | tokens validasi={len(val_np):,} | vocab={tokenizer.vocab_size}"
+    )
+
+    start_step = 0
+    init_state = None
+    if args.init_from:
+        meta = load_meta(args.init_from)
+        if meta.get("backend") != "tensorflow":
+            raise SystemExit(f"{args.init_from} bukan checkpoint backend TensorFlow")
+        config_d = meta["config"]
+        start_step = meta.get("step", 0)
+        init_state = {k.replace("/", "_"): v for k, v in load_file(args.init_from).items()}
+        print(f"melanjutkan dari {args.init_from} (step {start_step})")
+    else:
+        config_d = {
+            "vocab_size": tokenizer.vocab_size,
+            "block_size": args.block_size,
+            "n_layer": args.n_layer,
+            "n_head": args.n_head,
+            "n_embd": args.n_embd,
+            "dropout": args.dropout,
+            "bias": False,
+        }
+        if config_d["vocab_size"] != tokenizer.vocab_size:
+            raise SystemExit("vocab tidak cocok")
+    total_steps = start_step + args.steps
+
+    model = build_gpt(
+        vocab_size=config_d["vocab_size"],
+        block_size=config_d["block_size"],
+        n_layer=config_d["n_layer"],
+        n_head=config_d["n_head"],
+        n_embd=config_d["n_embd"],
+        dropout=config_d["dropout"],
+    )
+    if init_state is not None:
+        by_path = {v.path.replace("/", "_"): v for v in model.weights}
+        missing = [p for p in by_path if p not in init_state]
+        if missing:
+            raise SystemExit(f"bobot tidak cocok dengan checkpoint: {missing[:5]}")
+        model.set_weights([init_state[v.path.replace("/", "_")] for v in model.weights])
+
+    n_params = int(sum(int(np.prod(v.shape)) for v in model.weights))
+    print(f"device={device_label} | params={n_params / 1e6:.2f}M | vocab={config_d['vocab_size']} | total_steps={total_steps}")
+
+    optimizer = tf.keras.optimizers.AdamW(
+        learning_rate=args.lr, beta_1=0.9, beta_2=0.95, weight_decay=args.weight_decay, clipnorm=1.0
+    )
+    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+
+    def lr_at(step):
+        if step < args.warmup:
+            return args.lr * (step + 1) / args.warmup
+        progress = (step - args.warmup) / max(1, total_steps - args.warmup)
+        return 0.1 * args.lr + 0.45 * args.lr * (1 + math.cos(math.pi * progress))
+
+    best_val = float("inf")
+    last_val = None
+    t0 = time.time()
+    for step in range(start_step, total_steps):
+        optimizer.learning_rate.assign(lr_at(step))
+        x, y = get_batch(train_np, config_d["block_size"], args.batch_size)
+        loss = train_step(model, optimizer, loss_fn, x, y)
+        if step % args.eval_interval == 0 or step == total_steps - 1:
+            if len(val_np) > config_d["block_size"] + 1:
+                losses = []
+                for _ in range(args.eval_iters):
+                    vx, vy = get_batch(val_np, config_d["block_size"], args.batch_size)
+                    losses.append(float(eval_step(model, loss_fn, vx, vy)))
+                val_loss = sum(losses) / len(losses)
+                marker = ""
+                if val_loss < best_val:
+                    best_val = val_loss
+                    save_weights_tf(
+                        model,
+                        os.path.join(args.out, "indigo_best.safetensors"),
+                        config_d,
+                        tokenizer.itos,
+                        total_steps,
+                        val_loss,
+                    )
+                    marker = " <- best"
+                last_val = val_loss
+                val_str = f"{val_loss:.4f}{marker}"
+            else:
+                val_str = "n/a"
+            print(
+                f"step {step + 1:5d}/{total_steps} | "
+                f"loss {float(loss):.4f} | val {val_str} | {time.time() - t0:.1f}s"
+            )
+
+    final_path = os.path.join(args.out, "indigo.safetensors")
+    save_weights_tf(model, final_path, config_d, tokenizer.itos, total_steps, last_val)
+    print(f"model tersimpan di {final_path} (+_meta.json)")
+
+
+if __name__ == "__main__":
+    main()