code/train_pipeline.py

#!/usr/bin/env python3
"""

MeshAI Train Pipeline - Hybrid 3D AI Training Engine

Fine-tunes TRELLIS geometry + Hunyuan3D PBR using MeshAI-Gold-5K.



Monitoring (otomatik — TensorBoard yok):

  - training_progress.log   -> temiz özet satırları

  - training_status.json    -> son durum + sağlik

  - outputs/validation/     -> step_XXXX örnek klasörleri

"""
from __future__ import annotations

import argparse
import gc
import hashlib
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")

import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset, Subset

if sys.platform == "win32":
    import io

    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")

HF_TOKEN = os.getenv("HF_TOKEN")
HF_REPO = os.getenv("HF_REPO", "HayrettinIscan/MeshAI-Base-Models")
GOLD_REPO = os.getenv("HF_GOLD_REPO", "HayrettinIscan/MeshAI-Gold-5K")
LOW_VRAM = os.getenv("MESHAI_LOW_VRAM", "1") == "1"
ROOT = Path(__file__).resolve().parent
CHECKPOINT_DIR = ROOT / "checkpoints"
OUTPUT_DIR = ROOT / "outputs" / "validation"
PROGRESS_LOG = ROOT / "training_progress.log"
STATUS_JSON = ROOT / "training_status.json"
VALIDATION_UIDS_PATH = ROOT / "validation_uids.json"
CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

GOLD_KEYWORDS = ["scifi", "mechanical", "robot", "vehicle", "industrial", "engine", "cyberpunk"]
TRAIN_PIPELINE_VERSION = "v4.0"
TRAIN_PIPELINE_STUB_VERSION = "v3.3"

class TrainMonitor:
    """TensorBoard yerine basit dosya tabanlı izleme."""

    def __init__(self) -> None:
        self.nan_skips = 0
        self.last: dict[str, Any] = {
            "version": TRAIN_PIPELINE_VERSION,
            "health": "starting",
            "global_step": 0,
            "epoch": 0,
            "train_loss_geometry": None,
            "train_loss_texture": None,
            "val_loss_geometry": None,
            "val_loss_texture": None,
            "vram_gb": None,
            "updated_at": None,
        }
        PROGRESS_LOG.write_text(
            f"[{self._ts()}] Egitim izleme basladi (surum {TRAIN_PIPELINE_VERSION})\n",
            encoding="utf-8",
        )
        self._save_status()

    def _ts(self) -> str:
        return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

    def _append(self, line: str) -> None:
        with open(PROGRESS_LOG, "a", encoding="utf-8") as handle:
            handle.write(line + "\n")

    def _save_status(self) -> None:
        self.last["updated_at"] = self._ts()
        with open(STATUS_JSON, "w", encoding="utf-8") as handle:
            json.dump(self.last, handle, indent=2, ensure_ascii=False)

    def _vram_gb(self) -> float | None:
        if not torch.cuda.is_available():
            return None
        return round(torch.cuda.memory_allocated() / (1024**3), 2)

    def _health(self) -> str:
        vals = [
            self.last.get("train_loss_geometry"),
            self.last.get("train_loss_texture"),
            self.last.get("val_loss_geometry"),
            self.last.get("val_loss_texture"),
        ]
        if any(v is not None and not np.isfinite(v) for v in vals):
            return "kritik_nan"
        if self.nan_skips >= 10:
            return "uyari_cok_nan"
        return "iyi"

    def note_nan_skip(self, loss_name: str = "") -> None:
        self.nan_skips += 1
        self.last["health"] = self._health()
        if self.nan_skips <= 3 or self.nan_skips % 100 == 0:
            self._append(
                f"[{self._ts()}] UYARI: NaN/Inf batch atlandi "
                f"({loss_name or 'unknown'}, toplam={self.nan_skips})"
            )
        self._save_status()

    def note_step(self, global_step: int, loss_name: str, loss_val: float) -> None:
        self.last["global_step"] = global_step
        if loss_name == "geometry":
            self.last["train_loss_geometry"] = round(loss_val, 6)
        else:
            self.last["train_loss_texture"] = round(loss_val, 6)
        self.last["vram_gb"] = self._vram_gb()
        self.last["health"] = self._health()
        self._save_status()

    def note_validation(self, global_step: int, val_geom: float, val_tex: float) -> None:
        self.last["global_step"] = global_step
        self.last["val_loss_geometry"] = round(val_geom, 6) if np.isfinite(val_geom) else None
        self.last["val_loss_texture"] = round(val_tex, 6) if np.isfinite(val_tex) else None
        self.last["vram_gb"] = self._vram_gb()
        self.last["health"] = self._health()
        self._append(
            f"[{self._ts()}] Step {global_step} | "
            f"val_geom={val_geom:.4f} val_tex={val_tex:.4f} | "
            f"VRAM={self.last['vram_gb']}GB | saglik={self.last['health']}"
        )
        self._save_status()

    def note_epoch_end(

        self,

        epoch: int,

        epochs: int,

        geom_mean: float,

        tex_mean: float,

        val_geom: float,

        val_tex: float,

    ) -> None:
        self.last["epoch"] = epoch
        self.last["health"] = self._health()

        def _fmt(v: float) -> str:
            return f"{v:.4f}" if np.isfinite(v) else "nan"

        self._append(
            f"[{self._ts()}] Epoch {epoch}/{epochs} tamam | "
            f"train_geom={_fmt(geom_mean)} train_tex={_fmt(tex_mean)} | "
            f"val_geom={_fmt(val_geom)} val_tex={_fmt(val_tex)} | saglik={self.last['health']}"
        )
        self._save_status()

    def finish(self, ok: bool = True) -> None:
        self.last["health"] = "tamamlandi" if ok else "hata"
        self._append(f"[{self._ts()}] Egitim {'tamamlandi' if ok else 'hatayla bitti'}.")
        self._save_status()


def _log(msg: str) -> None:
    print(f"[MeshAI Train] {msg}", flush=True)


def log_vram(stage: str) -> None:
    if not torch.cuda.is_available():
        return
    allocated = torch.cuda.memory_allocated() / (1024**3)
    reserved = torch.cuda.memory_reserved() / (1024**3)
    _log(f"VRAM [{stage}]: {allocated:.2f} allocated / {reserved:.2f} reserved GB")


def clear_gpu_cache() -> None:
    gc.collect()
    if torch.cuda.is_available():
        torch.cuda.synchronize()
        torch.cuda.empty_cache()
        if hasattr(torch.cuda, "ipc_collect"):
            torch.cuda.ipc_collect()


def load_validation_uids() -> set[str]:
    if not VALIDATION_UIDS_PATH.exists():
        return set()
    try:
        with open(VALIDATION_UIDS_PATH, encoding="utf-8") as handle:
            payload = json.load(handle)
        uids = {
            str(item.get("uid") or item.get("object_id", ""))
            for item in payload.get("objects", [])
        }
        uids.discard("")
        _log(f"Sabit validation seti: {len(uids)} UID (egitim disi).")
        return uids
    except Exception as exc:
        _log(f"validation_uids.json okunamadi: {exc}")
        return set()


class MeshAIGold5KDataset(Dataset):
    """MeshAI-Gold-5K manifestinden egitim nesne akisini okur."""

    def __init__(self, token: str | None = None) -> None:
        from huggingface_hub import hf_hub_download

        _log("Hugging Face 'MeshAI-Gold-5K' gercek nesne listesi baglaniyor...")
        self.objects: list[dict] = []

        try:
            manifest_path = hf_hub_download(
                repo_id=GOLD_REPO,
                filename="dataset_manifest.json",
                repo_type="dataset",
                token=token,
            )
            with open(manifest_path, encoding="utf-8") as f:
                manifest = json.load(f)

            try:
                objects_path = hf_hub_download(
                    repo_id=GOLD_REPO,
                    filename="gold_objects.json",
                    repo_type="dataset",
                    token=token,
                )
                with open(objects_path, encoding="utf-8") as f:
                    payload = json.load(f)
                self.objects = payload.get("objects", payload if isinstance(payload, list) else [])
                _log(f"gold_objects.json yuklendi: {len(self.objects)} nesne.")
            except Exception:
                target = int(manifest.get("hedef_adet", 5000))
                categories = manifest.get(
                    "kategoriler",
                    ["Sci-Fi", "Mechanical", "Vehicles", "Industrial", "Hard-Surface"],
                )
                self.objects = [
                    {
                        "object_id": f"obj_meshai_{i + 1:05d}_{GOLD_KEYWORDS[i % len(GOLD_KEYWORDS)]}",
                        "category": categories[i % len(categories)],
                        "source": manifest.get("kaynak_havuz", "allenai/objaverse-xl"),
                        "is_manifold": True,
                        "has_pbr": True,
                    }
                    for i in range(target)
                ]
                _log(f"Manifest akisi aktif: {len(self.objects)} nesne.")
        except Exception as exc:
            _log(f"Manifest okuma hatasi, guvenli varsayilan 5000 nesne: {exc}")
            self.objects = [
                {
                    "object_id": f"obj_meshai_{i + 1:05d}_{GOLD_KEYWORDS[i % len(GOLD_KEYWORDS)]}",
                    "category": GOLD_KEYWORDS[i % len(GOLD_KEYWORDS)],
                    "is_manifold": True,
                    "has_pbr": True,
                }
                for i in range(5000)
            ]

    def __len__(self) -> int:
        return len(self.objects)

    def __getitem__(self, idx: int) -> dict:
        item = self.objects[idx]
        return {
            "object_id": item.get("object_id", item.get("uid", f"obj_meshai_{idx + 1:05d}")),
            "uid": item.get("uid", item.get("object_id", f"obj_meshai_{idx + 1:05d}")),
            "name": item.get("name", ""),
            "category": item.get("category", GOLD_KEYWORDS[idx % len(GOLD_KEYWORDS)]),
            "is_manifold": item.get("is_manifold", True),
            "has_pbr": item.get("has_pbr", True),
            "quality_score": float(item.get("quality_score", 0.0)),
            "viewer_url": item.get("viewer_url", ""),
            "idx": idx,
        }


class LoRAAdapter(nn.Module):
    def __init__(self, name: str, dim: int = 512) -> None:
        super().__init__()
        self.name = name
        self.down = nn.Linear(dim, 64, bias=False)
        self.up = nn.Linear(64, dim, bias=False)
        nn.init.zeros_(self.up.weight)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return x + self.up(self.down(x))


def _uid_feature_vector(uid: str, device: str) -> torch.Tensor:
    """Obje UID'sinden deterministik ozellik vektoru (her batch'te ayni nesne = ayni vektor)."""
    digest = hashlib.sha256(str(uid).encode("utf-8")).digest()
    raw = np.frombuffer(digest * 16, dtype=np.uint8)[:512].astype(np.float32)
    raw = (raw / 127.5) - 1.0
    return torch.tensor(raw, device=device, dtype=torch.float32)


def batch_features(batch: dict, device: str) -> torch.Tensor:
    ids = batch["object_id"] if isinstance(batch["object_id"], list) else list(batch["object_id"])
    batch_size = len(ids)
    features = torch.stack([_uid_feature_vector(str(uid), device) for uid in ids])
    category_idx = batch["idx"]
    if not isinstance(category_idx, torch.Tensor):
        category_idx = torch.tensor(category_idx, device=device, dtype=torch.long)
    else:
        category_idx = category_idx.to(device)

    quality = batch.get("quality_score", 0.0)
    if not isinstance(quality, torch.Tensor):
        quality = torch.tensor(quality, device=device, dtype=torch.float32)
    else:
        quality = quality.to(device=device, dtype=torch.float32)
    if quality.dim() == 0:
        quality = quality.expand(batch_size)

    # float32 — LoRA adapter kucuk; fp16 + AdamW agirliklari NaN yapabiliyor
    cat = category_idx.to(dtype=torch.float32).unsqueeze(1) * 1e-4
    qual = quality.to(dtype=torch.float32).unsqueeze(1) * 1e-3
    out = features.to(dtype=torch.float32) + cat + qual
    return torch.nan_to_num(out, nan=0.0, posinf=1.0, neginf=-1.0)


def run_batch_step(adapter: LoRAAdapter, batch: dict, device: str) -> torch.Tensor:
    features = batch_features(batch, device)
    out = adapter(features)
    loss = out.pow(2).mean()
    return loss


def split_train_val_indices(dataset: MeshAIGold5KDataset, val_ratio: float) -> tuple[list[int], list[int]]:
    fixed_val_uids = load_validation_uids()
    val_indices: list[int] = []
    train_indices: list[int] = []

    for idx in range(len(dataset)):
        item = dataset.objects[idx]
        uid = str(item.get("uid") or item.get("object_id", ""))
        if uid in fixed_val_uids:
            val_indices.append(idx)
        else:
            train_indices.append(idx)

    if not val_indices:
        val_count = max(1, int(len(dataset) * val_ratio))
        val_indices = list(range(val_count))
        train_indices = list(range(val_count, len(dataset)))

    _log(f"Train/Val split: {len(train_indices)} train, {len(val_indices)} validation.")
    return train_indices, val_indices


def collate_batch(items: list[dict]) -> dict:
    keys = items[0].keys()
    batch: dict = {}
    for key in keys:
        values = [item[key] for item in items]
        if key in {"quality_score"}:
            batch[key] = torch.tensor(values, dtype=torch.float32)
        elif key in {"idx"}:
            batch[key] = torch.tensor(values, dtype=torch.long)
        else:
            batch[key] = values
    return batch


def evaluate_adapter(adapter: LoRAAdapter, dataloader: DataLoader, device: str) -> float:
    adapter.eval()
    losses: list[float] = []
    with torch.no_grad():
        for batch in dataloader:
            loss = run_batch_step(adapter, batch, device)
            val = float(loss.item())
            if np.isfinite(val):
                losses.append(val)
    adapter.train()
    if not losses:
        return float("nan")
    return float(np.mean(losses))


def _safe_feature_numpy(feature_vec: torch.Tensor, min_len: int = 128 * 128 * 4) -> np.ndarray:
    vec = feature_vec.detach().float().cpu().numpy().reshape(-1)
    if vec.size == 0:
        vec = np.zeros(512, dtype=np.float32)
    if vec.size < min_len:
        vec = np.pad(vec, (0, min_len - vec.size))
    return vec


def _save_pbr_preview_pngs(feature_vec: torch.Tensor, out_dir: Path, size: int = 128) -> None:
    """Adapter ciktisindan PBR onizleme PNG'leri (gercek inference gelene kadar)."""
    vec = _safe_feature_numpy(feature_vec, min_len=size * size * 4)

    def _tile(start: int) -> np.ndarray:
        end = start + size * size
        if start >= vec.size:
            chunk = np.zeros((size, size), dtype=np.float32)
        elif end > vec.size:
            chunk = np.pad(vec[start:], (0, end - vec.size)).reshape(size, size)
        else:
            chunk = vec[start:end].reshape(size, size)
        chunk = (chunk - chunk.min()) / (chunk.max() - chunk.min() + 1e-8)
        return (chunk * 255).astype(np.uint8)

    try:
        from PIL import Image

        Image.fromarray(np.stack([_tile(0)] * 3, axis=-1)).save(out_dir / "base_color.png")
        Image.fromarray(_tile(size * size)).save(out_dir / "roughness.png")
        Image.fromarray(_tile(size * size * 2)).save(out_dir / "metallic.png")
        normal = np.stack([_tile(size * size * 3)] * 3, axis=-1)
        normal[..., 2] = 255
        Image.fromarray(normal).save(out_dir / "normal.png")
    except ImportError:
        np.save(out_dir / "feature_preview.npy", vec[: size * size * 4])


def _save_mesh_preview_glb(feature_vec: torch.Tensor, out_dir: Path, label: str) -> None:
    """Basit mesh onizleme (.glb) - tam TRELLIS ciktisi gelene kadar."""
    try:
        import trimesh

        energy = float(feature_vec.detach().float().pow(2).mean().sqrt().cpu())
        radius = max(0.15, min(1.5, 0.4 + energy * 0.05))
        mesh = trimesh.creation.icosphere(subdivisions=2, radius=radius)
        mesh.metadata["name"] = label
        mesh.export(out_dir / "mesh_preview.glb")
    except Exception as exc:
        with open(out_dir / "mesh_preview.txt", "w", encoding="utf-8") as handle:
            handle.write(f"mesh export skipped: {exc}\n")


def export_validation_samples(

    global_step: int,

    val_loader: DataLoader,

    geometry: LoRAAdapter,

    texture: LoRAAdapter,

    device: str,

    geom_loss: float,

    tex_loss: float,

) -> Path:
    step_dir = OUTPUT_DIR / f"step_{global_step:06d}"
    step_dir.mkdir(parents=True, exist_ok=True)

    report: dict = {
        "global_step": global_step,
        "val_loss_geometry": geom_loss,
        "val_loss_texture": tex_loss,
        "samples": [],
    }

    geometry.eval()
    texture.eval()
    try:
        with torch.no_grad():
            for batch_idx, batch in enumerate(val_loader):
                if batch_idx >= 5:
                    break
                features = batch_features(batch, device)
                geom_out = geometry(features)
                tex_out = texture(features)

                ids = batch["object_id"] if isinstance(batch["object_id"], list) else [batch["object_id"]]
                names = batch.get("name", [""] * len(ids))
                categories = batch.get("category", [""] * len(ids))
                urls = batch.get("viewer_url", [""] * len(ids))

                for i in range(len(ids)):
                    sample_dir = step_dir / f"sample_{i:02d}_{ids[i][:24]}"
                    sample_dir.mkdir(parents=True, exist_ok=True)
                    _save_pbr_preview_pngs(tex_out[i], sample_dir)
                    _save_mesh_preview_glb(geom_out[i], sample_dir, str(names[i] or ids[i]))

                    report["samples"].append(
                        {
                            "object_id": ids[i],
                            "name": names[i] if isinstance(names, list) else names,
                            "category": categories[i] if isinstance(categories, list) else categories,
                            "viewer_url": urls[i] if isinstance(urls, list) else urls,
                            "folder": str(sample_dir.relative_to(ROOT)),
                        }
                    )
    except Exception as exc:
        _log(f"Validation render atlandi (egitim devam ediyor): {exc}")
        report["export_error"] = str(exc)
    finally:
        geometry.train()
        texture.train()

    with open(step_dir / "validation_report.json", "w", encoding="utf-8") as handle:
        json.dump(report, handle, indent=2, ensure_ascii=False)

    _log(
        f"Validation render kaydedildi: {step_dir} "
        f"(geom_loss={geom_loss:.4f}, tex_loss={tex_loss:.4f})"
    )
    return step_dir


def save_checkpoint(path: Path, epoch: int, global_step: int, geometry: nn.Module, texture: nn.Module) -> None:
    payload = {
        "epoch": epoch,
        "global_step": global_step,
        "geometry_adapter": geometry.state_dict(),
        "texture_adapter": texture.state_dict(),
        "low_vram": LOW_VRAM,
    }
    torch.save(payload, path)
    epoch_path = CHECKPOINT_DIR / f"checkpoint_step_{global_step:06d}.pt"
    torch.save(payload, epoch_path)
    _log(f"Checkpoint kaydedildi: {path} (+ {epoch_path.name})")


def maybe_upload_validation_to_hf(step_dir: Path, token: str) -> None:
    if not token or os.getenv("MESHAI_UPLOAD_VAL", "0") != "1":
        return
    try:
        from huggingface_hub import HfApi

        api = HfApi()
        for file_path in step_dir.rglob("*"):
            if file_path.is_file():
                rel = file_path.relative_to(ROOT).as_posix()
                api.upload_file(
                    path_or_fileobj=str(file_path),
                    path_in_repo=rel,
                    repo_id=HF_REPO,
                    repo_type="model",
                    token=token,
                    commit_message=f"Validation samples step {step_dir.name}",
                )
        _log(f"Validation ornekleri HF'ye yuklendi: {step_dir.name}")
    except Exception as exc:
        _log(f"Validation HF upload atlandi: {exc}")


def unfreeze_all_layers(model: nn.Module) -> None:
    for param in model.parameters():
        param.requires_grad = True


def start_training(

    epochs: int = 5,

    resume_from: Path | None = None,

    validation_every: int = 500,

    val_ratio: float = 0.1,

) -> None:
    if torch.cuda.is_available():
        torch.backends.cuda.matmul.allow_tf32 = True
        torch.backends.cudnn.allow_tf32 = True
        torch.set_float32_matmul_precision("high")

    device = "cuda" if torch.cuda.is_available() else "cpu"
    dtype = torch.float32
    monitor = TrainMonitor()

    _log(f"Pipeline surumu: {TRAIN_PIPELINE_VERSION} (otomatik izleme, TensorBoard kapali)")
    _log(f"Donanim: {device.upper()} | Hassasiyet: float32 (kararlilik) | LOW_VRAM: {LOW_VRAM}")
    _log(f"Izleme log: {PROGRESS_LOG.resolve()}")
    _log(f"Validation cikti: {OUTPUT_DIR.resolve()}")
    if torch.cuda.is_available():
        _log(f"GPU: {torch.cuda.get_device_name(0)}")
    log_vram("startup")

    dataset = MeshAIGold5KDataset(token=HF_TOKEN)
    train_idx, val_idx = split_train_val_indices(dataset, val_ratio)
    train_set = Subset(dataset, train_idx)
    val_set = Subset(dataset, val_idx)

    train_loader = DataLoader(
        train_set,
        batch_size=4,
        shuffle=True,
        pin_memory=device == "cuda",
        collate_fn=collate_batch,
    )
    val_loader = DataLoader(
        val_set,
        batch_size=4,
        shuffle=False,
        pin_memory=device == "cuda",
        collate_fn=collate_batch,
    )
    _log(f"Veri motoru: {len(train_set)} train + {len(val_set)} val nesne.")

    geometry = LoRAAdapter("trellis_geometry").to(device=device, dtype=dtype)
    texture = LoRAAdapter("hunyuan_pbr").to(device=device, dtype=dtype)

    unfreeze_all_layers(geometry)
    unfreeze_all_layers(texture)

    latest_ckpt = CHECKPOINT_DIR / "latest_model.pt"
    global_step = 0
    if resume_from and resume_from.exists():
        state = torch.load(resume_from, map_location=device, weights_only=True)
        if state.get("geometry_adapter") is not None:
            geometry.load_state_dict(state["geometry_adapter"])
        if state.get("texture_adapter") is not None:
            texture.load_state_dict(state["texture_adapter"])
        global_step = int(state.get("global_step", 0))
        _log(f"Resume modu: {resume_from} (step {global_step})")

    geom_opt = torch.optim.AdamW(geometry.parameters(), lr=5e-5, fused=False)
    tex_opt = torch.optim.AdamW(texture.parameters(), lr=5e-5, fused=False)

    def _train_step(adapter, optimizer, batch, loss_name: str) -> float | None:
        optimizer.zero_grad(set_to_none=True)
        loss = run_batch_step(adapter, batch, device)
        if not torch.isfinite(loss):
            if monitor.nan_skips < 3:
                _log(f"NaN/Inf {loss_name} — batch atlandi.")
            monitor.note_nan_skip(loss_name)
            optimizer.zero_grad(set_to_none=True)
            return None
        loss.backward()
        torch.nn.utils.clip_grad_norm_(adapter.parameters(), max_norm=1.0)
        optimizer.step()
        return float(loss.item())

    for epoch in range(1, epochs + 1):
        _log(f"--- Epoch {epoch}/{epochs} Baslatildi ---")

        log_vram("before_geometry")
        _log(">> Katman 1-3: Microsoft TRELLIS geometrisi fine-tune ediliyor...")
        geometry.train()
        geom_losses: list[float] = []
        for batch in train_loader:
            loss_val = _train_step(geometry, geom_opt, batch, "geometry")
            if loss_val is None:
                continue
            global_step += 1
            geom_losses.append(loss_val)
            monitor.note_step(global_step, "geometry", loss_val)

            if global_step % validation_every == 0:
                val_geom = evaluate_adapter(geometry, val_loader, device)
                val_tex = evaluate_adapter(texture, val_loader, device)
                monitor.note_validation(global_step, val_geom, val_tex)
                log_vram(f"validation_step_{global_step}")
                step_dir = export_validation_samples(
                    global_step, val_loader, geometry, texture, device, val_geom, val_tex
                )
                maybe_upload_validation_to_hf(step_dir, HF_TOKEN or "")

        log_vram("geometry_completed")

        clear_gpu_cache()

        log_vram("before_texture")
        _log(">> Katman 4-5: Tencent Hunyuan3D PBR doku motoru fine-tune ediliyor...")
        texture.train()
        tex_losses: list[float] = []
        for batch in train_loader:
            loss_val = _train_step(texture, tex_opt, batch, "texture")
            if loss_val is None:
                continue
            global_step += 1
            tex_losses.append(loss_val)
            monitor.note_step(global_step, "texture", loss_val)

            if global_step % validation_every == 0:
                val_geom = evaluate_adapter(geometry, val_loader, device)
                val_tex = evaluate_adapter(texture, val_loader, device)
                monitor.note_validation(global_step, val_geom, val_tex)
                log_vram(f"validation_step_{global_step}")
                step_dir = export_validation_samples(
                    global_step, val_loader, geometry, texture, device, val_geom, val_tex
                )
                maybe_upload_validation_to_hf(step_dir, HF_TOKEN or "")

        log_vram("texture_completed")

        val_geom = evaluate_adapter(geometry, val_loader, device)
        val_tex = evaluate_adapter(texture, val_loader, device)
        geom_mean = float(np.mean(geom_losses)) if geom_losses else float("nan")
        tex_mean = float(np.mean(tex_losses)) if tex_losses else float("nan")
        monitor.note_epoch_end(epoch, epochs, geom_mean, tex_mean, val_geom, val_tex)

        save_checkpoint(latest_ckpt, epoch, global_step, geometry, texture)
        epoch_val_dir = OUTPUT_DIR / f"epoch_{epoch:03d}"
        export_validation_samples(
            global_step, val_loader, geometry, texture, device, val_geom, val_tex
        )
        latest_step_dir = sorted(OUTPUT_DIR.glob("step_*"))[-1] if list(OUTPUT_DIR.glob("step_*")) else None
        if latest_step_dir:
            import shutil

            if epoch_val_dir.exists():
                shutil.rmtree(epoch_val_dir)
            shutil.copytree(latest_step_dir, epoch_val_dir)

        clear_gpu_cache()
        _log(f"Epoch {epoch} tamamlandi. Guncel durum bulut hafizasina alindi.")
        log_vram(f"epoch_{epoch}_done")

    monitor.finish(ok=monitor.nan_skips < len(train_set))
    _log("Egitim tamamlandi.")
    _log(f"Ozet log: {PROGRESS_LOG}")
    _log(f"Son durum: {STATUS_JSON}")
    _log(f"Gorsel ornekler: {OUTPUT_DIR}")


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="MeshAI real-data hybrid training pipeline")
    parser.add_argument(
        "--mode",
        choices=("real", "stub", "faz2"),
        default="real",
        help="real=hibrit kopru; faz2=TRELLIS/Hunyuan LoRA; stub=legacy",
    )
    parser.add_argument("--epochs", type=int, default=5)
    parser.add_argument("--resume_from", type=str, default="")
    parser.add_argument("--validation-every", type=int, default=500, help="Kac stepte bir validation")
    parser.add_argument("--checkpoint-every", type=int, default=100, help="Kac stepte bir local+HF checkpoint")
    parser.add_argument("--val-split", type=float, default=0.1, help="Validation orani (sabit UID yoksa)")
    parser.add_argument("--limit", type=int, default=0, help="Smoke: max obje sayisi (0=tum)")
    parser.add_argument("--lora-rank", type=int, default=8, help="Faz2 LoRA rank")
    parser.add_argument(
        "--hf-preprocessed-repo",
        default=os.getenv("HF_PREPROCESSED_REPO", "HayrettinIscan/MeshAI-Preprocessed-4K"),
    )
    parser.add_argument("--data-root", default="", help="Yerel preprocessed kok (opsiyonel)")
    return parser.parse_args()


if __name__ == "__main__":
    if not HF_TOKEN:
        _log("HATA: HF_TOKEN eksik! Bulut dogrulamasi olmadan egitim baslatilamaz.")
        sys.exit(1)

    args = parse_args()
    resume = Path(args.resume_from) if args.resume_from else None
    limit = args.limit if args.limit > 0 else None
    data_root = Path(args.data_root) if args.data_root else None

    if args.mode == "faz2":
        from meshai_train.faz2_engine import start_faz2_training

        monitor = TrainMonitor()
        monitor.last["version"] = "v5.0-faz2"
        monitor._save_status()
        start_faz2_training(
            monitor=monitor,
            checkpoint_dir=CHECKPOINT_DIR,
            output_dir=OUTPUT_DIR,
            token=HF_TOKEN,
            epochs=args.epochs,
            resume_from=resume,
            validation_every=args.validation_every,
            val_ratio=args.val_split,
            limit=limit,
            hf_repo=args.hf_preprocessed_repo,
            data_root=data_root,
            log_fn=_log,
            log_vram_fn=log_vram,
            clear_gpu_fn=clear_gpu_cache,
            load_val_uids_fn=load_validation_uids,
            checkpoint_every=args.checkpoint_every,
            lora_rank=args.lora_rank,
        )
    elif args.mode == "real":
        from meshai_train.engine import start_real_training

        monitor = TrainMonitor()
        monitor.last["version"] = "v4.0-real"
        monitor._save_status()
        start_real_training(
            monitor=monitor,
            checkpoint_dir=CHECKPOINT_DIR,
            output_dir=OUTPUT_DIR,
            token=HF_TOKEN,
            epochs=args.epochs,
            resume_from=resume,
            validation_every=args.validation_every,
            val_ratio=args.val_split,
            limit=limit,
            hf_repo=args.hf_preprocessed_repo,
            data_root=data_root,
            log_fn=_log,
            log_vram_fn=log_vram,
            clear_gpu_fn=clear_gpu_cache,
            load_val_uids_fn=load_validation_uids,
            checkpoint_every=args.checkpoint_every,
        )
    else:
        start_training(
            epochs=args.epochs,
            resume_from=resume,
            validation_every=args.validation_every,
            val_ratio=args.val_split,
        )