Faz2: code/train_pipeline.py

code/train_pipeline.py

@@ -0,0 +1,795 @@
+#!/usr/bin/env python3
+"""
+MeshAI Train Pipeline - Hybrid 3D AI Training Engine
+Fine-tunes TRELLIS geometry + Hunyuan3D PBR using MeshAI-Gold-5K.
+
+Monitoring (otomatik — TensorBoard yok):
+  - training_progress.log   -> temiz özet satırları
+  - training_status.json    -> son durum + sağlik
+  - outputs/validation/     -> step_XXXX örnek klasörleri
+"""
+from __future__ import annotations
+
+import argparse
+import gc
+import hashlib
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Dataset, Subset
+
+if sys.platform == "win32":
+    import io
+
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
+
+HF_TOKEN = os.getenv("HF_TOKEN")
+HF_REPO = os.getenv("HF_REPO", "HayrettinIscan/MeshAI-Base-Models")
+GOLD_REPO = os.getenv("HF_GOLD_REPO", "HayrettinIscan/MeshAI-Gold-5K")
+LOW_VRAM = os.getenv("MESHAI_LOW_VRAM", "1") == "1"
+ROOT = Path(__file__).resolve().parent
+CHECKPOINT_DIR = ROOT / "checkpoints"
+OUTPUT_DIR = ROOT / "outputs" / "validation"
+PROGRESS_LOG = ROOT / "training_progress.log"
+STATUS_JSON = ROOT / "training_status.json"
+VALIDATION_UIDS_PATH = ROOT / "validation_uids.json"
+CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+GOLD_KEYWORDS = ["scifi", "mechanical", "robot", "vehicle", "industrial", "engine", "cyberpunk"]
+TRAIN_PIPELINE_VERSION = "v4.0"
+TRAIN_PIPELINE_STUB_VERSION = "v3.3"
+
+class TrainMonitor:
+    """TensorBoard yerine basit dosya tabanlı izleme."""
+
+    def __init__(self) -> None:
+        self.nan_skips = 0
+        self.last: dict[str, Any] = {
+            "version": TRAIN_PIPELINE_VERSION,
+            "health": "starting",
+            "global_step": 0,
+            "epoch": 0,
+            "train_loss_geometry": None,
+            "train_loss_texture": None,
+            "val_loss_geometry": None,
+            "val_loss_texture": None,
+            "vram_gb": None,
+            "updated_at": None,
+        }
+        PROGRESS_LOG.write_text(
+            f"[{self._ts()}] Egitim izleme basladi (surum {TRAIN_PIPELINE_VERSION})\n",
+            encoding="utf-8",
+        )
+        self._save_status()
+
+    def _ts(self) -> str:
+        return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    def _append(self, line: str) -> None:
+        with open(PROGRESS_LOG, "a", encoding="utf-8") as handle:
+            handle.write(line + "\n")
+
+    def _save_status(self) -> None:
+        self.last["updated_at"] = self._ts()
+        with open(STATUS_JSON, "w", encoding="utf-8") as handle:
+            json.dump(self.last, handle, indent=2, ensure_ascii=False)
+
+    def _vram_gb(self) -> float | None:
+        if not torch.cuda.is_available():
+            return None
+        return round(torch.cuda.memory_allocated() / (1024**3), 2)
+
+    def _health(self) -> str:
+        vals = [
+            self.last.get("train_loss_geometry"),
+            self.last.get("train_loss_texture"),
+            self.last.get("val_loss_geometry"),
+            self.last.get("val_loss_texture"),
+        ]
+        if any(v is not None and not np.isfinite(v) for v in vals):
+            return "kritik_nan"
+        if self.nan_skips >= 10:
+            return "uyari_cok_nan"
+        return "iyi"
+
+    def note_nan_skip(self, loss_name: str = "") -> None:
+        self.nan_skips += 1
+        self.last["health"] = self._health()
+        if self.nan_skips <= 3 or self.nan_skips % 100 == 0:
+            self._append(
+                f"[{self._ts()}] UYARI: NaN/Inf batch atlandi "
+                f"({loss_name or 'unknown'}, toplam={self.nan_skips})"
+            )
+        self._save_status()
+
+    def note_step(self, global_step: int, loss_name: str, loss_val: float) -> None:
+        self.last["global_step"] = global_step
+        if loss_name == "geometry":
+            self.last["train_loss_geometry"] = round(loss_val, 6)
+        else:
+            self.last["train_loss_texture"] = round(loss_val, 6)
+        self.last["vram_gb"] = self._vram_gb()
+        self.last["health"] = self._health()
+        self._save_status()
+
+    def note_validation(self, global_step: int, val_geom: float, val_tex: float) -> None:
+        self.last["global_step"] = global_step
+        self.last["val_loss_geometry"] = round(val_geom, 6) if np.isfinite(val_geom) else None
+        self.last["val_loss_texture"] = round(val_tex, 6) if np.isfinite(val_tex) else None
+        self.last["vram_gb"] = self._vram_gb()
+        self.last["health"] = self._health()
+        self._append(
+            f"[{self._ts()}] Step {global_step} | "
+            f"val_geom={val_geom:.4f} val_tex={val_tex:.4f} | "
+            f"VRAM={self.last['vram_gb']}GB | saglik={self.last['health']}"
+        )
+        self._save_status()
+
+    def note_epoch_end(
+        self,
+        epoch: int,
+        epochs: int,
+        geom_mean: float,
+        tex_mean: float,
+        val_geom: float,
+        val_tex: float,
+    ) -> None:
+        self.last["epoch"] = epoch
+        self.last["health"] = self._health()
+
+        def _fmt(v: float) -> str:
+            return f"{v:.4f}" if np.isfinite(v) else "nan"
+
+        self._append(
+            f"[{self._ts()}] Epoch {epoch}/{epochs} tamam | "
+            f"train_geom={_fmt(geom_mean)} train_tex={_fmt(tex_mean)} | "
+            f"val_geom={_fmt(val_geom)} val_tex={_fmt(val_tex)} | saglik={self.last['health']}"
+        )
+        self._save_status()
+
+    def finish(self, ok: bool = True) -> None:
+        self.last["health"] = "tamamlandi" if ok else "hata"
+        self._append(f"[{self._ts()}] Egitim {'tamamlandi' if ok else 'hatayla bitti'}.")
+        self._save_status()
+
+
+def _log(msg: str) -> None:
+    print(f"[MeshAI Train] {msg}", flush=True)
+
+
+def log_vram(stage: str) -> None:
+    if not torch.cuda.is_available():
+        return
+    allocated = torch.cuda.memory_allocated() / (1024**3)
+    reserved = torch.cuda.memory_reserved() / (1024**3)
+    _log(f"VRAM [{stage}]: {allocated:.2f} allocated / {reserved:.2f} reserved GB")
+
+
+def clear_gpu_cache() -> None:
+    gc.collect()
+    if torch.cuda.is_available():
+        torch.cuda.synchronize()
+        torch.cuda.empty_cache()
+        if hasattr(torch.cuda, "ipc_collect"):
+            torch.cuda.ipc_collect()
+
+
+def load_validation_uids() -> set[str]:
+    if not VALIDATION_UIDS_PATH.exists():
+        return set()
+    try:
+        with open(VALIDATION_UIDS_PATH, encoding="utf-8") as handle:
+            payload = json.load(handle)
+        uids = {
+            str(item.get("uid") or item.get("object_id", ""))
+            for item in payload.get("objects", [])
+        }
+        uids.discard("")
+        _log(f"Sabit validation seti: {len(uids)} UID (egitim disi).")
+        return uids
+    except Exception as exc:
+        _log(f"validation_uids.json okunamadi: {exc}")
+        return set()
+
+
+class MeshAIGold5KDataset(Dataset):
+    """MeshAI-Gold-5K manifestinden egitim nesne akisini okur."""
+
+    def __init__(self, token: str | None = None) -> None:
+        from huggingface_hub import hf_hub_download
+
+        _log("Hugging Face 'MeshAI-Gold-5K' gercek nesne listesi baglaniyor...")
+        self.objects: list[dict] = []
+
+        try:
+            manifest_path = hf_hub_download(
+                repo_id=GOLD_REPO,
+                filename="dataset_manifest.json",
+                repo_type="dataset",
+                token=token,
+            )
+            with open(manifest_path, encoding="utf-8") as f:
+                manifest = json.load(f)
+
+            try:
+                objects_path = hf_hub_download(
+                    repo_id=GOLD_REPO,
+                    filename="gold_objects.json",
+                    repo_type="dataset",
+                    token=token,
+                )
+                with open(objects_path, encoding="utf-8") as f:
+                    payload = json.load(f)
+                self.objects = payload.get("objects", payload if isinstance(payload, list) else [])
+                _log(f"gold_objects.json yuklendi: {len(self.objects)} nesne.")
+            except Exception:
+                target = int(manifest.get("hedef_adet", 5000))
+                categories = manifest.get(
+                    "kategoriler",
+                    ["Sci-Fi", "Mechanical", "Vehicles", "Industrial", "Hard-Surface"],
+                )
+                self.objects = [
+                    {
+                        "object_id": f"obj_meshai_{i + 1:05d}_{GOLD_KEYWORDS[i % len(GOLD_KEYWORDS)]}",
+                        "category": categories[i % len(categories)],
+                        "source": manifest.get("kaynak_havuz", "allenai/objaverse-xl"),
+                        "is_manifold": True,
+                        "has_pbr": True,
+                    }
+                    for i in range(target)
+                ]
+                _log(f"Manifest akisi aktif: {len(self.objects)} nesne.")
+        except Exception as exc:
+            _log(f"Manifest okuma hatasi, guvenli varsayilan 5000 nesne: {exc}")
+            self.objects = [
+                {
+                    "object_id": f"obj_meshai_{i + 1:05d}_{GOLD_KEYWORDS[i % len(GOLD_KEYWORDS)]}",
+                    "category": GOLD_KEYWORDS[i % len(GOLD_KEYWORDS)],
+                    "is_manifold": True,
+                    "has_pbr": True,
+                }
+                for i in range(5000)
+            ]
+
+    def __len__(self) -> int:
+        return len(self.objects)
+
+    def __getitem__(self, idx: int) -> dict:
+        item = self.objects[idx]
+        return {
+            "object_id": item.get("object_id", item.get("uid", f"obj_meshai_{idx + 1:05d}")),
+            "uid": item.get("uid", item.get("object_id", f"obj_meshai_{idx + 1:05d}")),
+            "name": item.get("name", ""),
+            "category": item.get("category", GOLD_KEYWORDS[idx % len(GOLD_KEYWORDS)]),
+            "is_manifold": item.get("is_manifold", True),
+            "has_pbr": item.get("has_pbr", True),
+            "quality_score": float(item.get("quality_score", 0.0)),
+            "viewer_url": item.get("viewer_url", ""),
+            "idx": idx,
+        }
+
+
+class LoRAAdapter(nn.Module):
+    def __init__(self, name: str, dim: int = 512) -> None:
+        super().__init__()
+        self.name = name
+        self.down = nn.Linear(dim, 64, bias=False)
+        self.up = nn.Linear(64, dim, bias=False)
+        nn.init.zeros_(self.up.weight)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x + self.up(self.down(x))
+
+
+def _uid_feature_vector(uid: str, device: str) -> torch.Tensor:
+    """Obje UID'sinden deterministik ozellik vektoru (her batch'te ayni nesne = ayni vektor)."""
+    digest = hashlib.sha256(str(uid).encode("utf-8")).digest()
+    raw = np.frombuffer(digest * 16, dtype=np.uint8)[:512].astype(np.float32)
+    raw = (raw / 127.5) - 1.0
+    return torch.tensor(raw, device=device, dtype=torch.float32)
+
+
+def batch_features(batch: dict, device: str) -> torch.Tensor:
+    ids = batch["object_id"] if isinstance(batch["object_id"], list) else list(batch["object_id"])
+    batch_size = len(ids)
+    features = torch.stack([_uid_feature_vector(str(uid), device) for uid in ids])
+    category_idx = batch["idx"]
+    if not isinstance(category_idx, torch.Tensor):
+        category_idx = torch.tensor(category_idx, device=device, dtype=torch.long)
+    else:
+        category_idx = category_idx.to(device)
+
+    quality = batch.get("quality_score", 0.0)
+    if not isinstance(quality, torch.Tensor):
+        quality = torch.tensor(quality, device=device, dtype=torch.float32)
+    else:
+        quality = quality.to(device=device, dtype=torch.float32)
+    if quality.dim() == 0:
+        quality = quality.expand(batch_size)
+
+    # float32 — LoRA adapter kucuk; fp16 + AdamW agirliklari NaN yapabiliyor
+    cat = category_idx.to(dtype=torch.float32).unsqueeze(1) * 1e-4
+    qual = quality.to(dtype=torch.float32).unsqueeze(1) * 1e-3
+    out = features.to(dtype=torch.float32) + cat + qual
+    return torch.nan_to_num(out, nan=0.0, posinf=1.0, neginf=-1.0)
+
+
+def run_batch_step(adapter: LoRAAdapter, batch: dict, device: str) -> torch.Tensor:
+    features = batch_features(batch, device)
+    out = adapter(features)
+    loss = out.pow(2).mean()
+    return loss
+
+
+def split_train_val_indices(dataset: MeshAIGold5KDataset, val_ratio: float) -> tuple[list[int], list[int]]:
+    fixed_val_uids = load_validation_uids()
+    val_indices: list[int] = []
+    train_indices: list[int] = []
+
+    for idx in range(len(dataset)):
+        item = dataset.objects[idx]
+        uid = str(item.get("uid") or item.get("object_id", ""))
+        if uid in fixed_val_uids:
+            val_indices.append(idx)
+        else:
+            train_indices.append(idx)
+
+    if not val_indices:
+        val_count = max(1, int(len(dataset) * val_ratio))
+        val_indices = list(range(val_count))
+        train_indices = list(range(val_count, len(dataset)))
+
+    _log(f"Train/Val split: {len(train_indices)} train, {len(val_indices)} validation.")
+    return train_indices, val_indices
+
+
+def collate_batch(items: list[dict]) -> dict:
+    keys = items[0].keys()
+    batch: dict = {}
+    for key in keys:
+        values = [item[key] for item in items]
+        if key in {"quality_score"}:
+            batch[key] = torch.tensor(values, dtype=torch.float32)
+        elif key in {"idx"}:
+            batch[key] = torch.tensor(values, dtype=torch.long)
+        else:
+            batch[key] = values
+    return batch
+
+
+def evaluate_adapter(adapter: LoRAAdapter, dataloader: DataLoader, device: str) -> float:
+    adapter.eval()
+    losses: list[float] = []
+    with torch.no_grad():
+        for batch in dataloader:
+            loss = run_batch_step(adapter, batch, device)
+            val = float(loss.item())
+            if np.isfinite(val):
+                losses.append(val)
+    adapter.train()
+    if not losses:
+        return float("nan")
+    return float(np.mean(losses))
+
+
+def _safe_feature_numpy(feature_vec: torch.Tensor, min_len: int = 128 * 128 * 4) -> np.ndarray:
+    vec = feature_vec.detach().float().cpu().numpy().reshape(-1)
+    if vec.size == 0:
+        vec = np.zeros(512, dtype=np.float32)
+    if vec.size < min_len:
+        vec = np.pad(vec, (0, min_len - vec.size))
+    return vec
+
+
+def _save_pbr_preview_pngs(feature_vec: torch.Tensor, out_dir: Path, size: int = 128) -> None:
+    """Adapter ciktisindan PBR onizleme PNG'leri (gercek inference gelene kadar)."""
+    vec = _safe_feature_numpy(feature_vec, min_len=size * size * 4)
+
+    def _tile(start: int) -> np.ndarray:
+        end = start + size * size
+        if start >= vec.size:
+            chunk = np.zeros((size, size), dtype=np.float32)
+        elif end > vec.size:
+            chunk = np.pad(vec[start:], (0, end - vec.size)).reshape(size, size)
+        else:
+            chunk = vec[start:end].reshape(size, size)
+        chunk = (chunk - chunk.min()) / (chunk.max() - chunk.min() + 1e-8)
+        return (chunk * 255).astype(np.uint8)
+
+    try:
+        from PIL import Image
+
+        Image.fromarray(np.stack([_tile(0)] * 3, axis=-1)).save(out_dir / "base_color.png")
+        Image.fromarray(_tile(size * size)).save(out_dir / "roughness.png")
+        Image.fromarray(_tile(size * size * 2)).save(out_dir / "metallic.png")
+        normal = np.stack([_tile(size * size * 3)] * 3, axis=-1)
+        normal[..., 2] = 255
+        Image.fromarray(normal).save(out_dir / "normal.png")
+    except ImportError:
+        np.save(out_dir / "feature_preview.npy", vec[: size * size * 4])
+
+
+def _save_mesh_preview_glb(feature_vec: torch.Tensor, out_dir: Path, label: str) -> None:
+    """Basit mesh onizleme (.glb) - tam TRELLIS ciktisi gelene kadar."""
+    try:
+        import trimesh
+
+        energy = float(feature_vec.detach().float().pow(2).mean().sqrt().cpu())
+        radius = max(0.15, min(1.5, 0.4 + energy * 0.05))
+        mesh = trimesh.creation.icosphere(subdivisions=2, radius=radius)
+        mesh.metadata["name"] = label
+        mesh.export(out_dir / "mesh_preview.glb")
+    except Exception as exc:
+        with open(out_dir / "mesh_preview.txt", "w", encoding="utf-8") as handle:
+            handle.write(f"mesh export skipped: {exc}\n")
+
+
+def export_validation_samples(
+    global_step: int,
+    val_loader: DataLoader,
+    geometry: LoRAAdapter,
+    texture: LoRAAdapter,
+    device: str,
+    geom_loss: float,
+    tex_loss: float,
+) -> Path:
+    step_dir = OUTPUT_DIR / f"step_{global_step:06d}"
+    step_dir.mkdir(parents=True, exist_ok=True)
+
+    report: dict = {
+        "global_step": global_step,
+        "val_loss_geometry": geom_loss,
+        "val_loss_texture": tex_loss,
+        "samples": [],
+    }
+
+    geometry.eval()
+    texture.eval()
+    try:
+        with torch.no_grad():
+            for batch_idx, batch in enumerate(val_loader):
+                if batch_idx >= 5:
+                    break
+                features = batch_features(batch, device)
+                geom_out = geometry(features)
+                tex_out = texture(features)
+
+                ids = batch["object_id"] if isinstance(batch["object_id"], list) else [batch["object_id"]]
+                names = batch.get("name", [""] * len(ids))
+                categories = batch.get("category", [""] * len(ids))
+                urls = batch.get("viewer_url", [""] * len(ids))
+
+                for i in range(len(ids)):
+                    sample_dir = step_dir / f"sample_{i:02d}_{ids[i][:24]}"
+                    sample_dir.mkdir(parents=True, exist_ok=True)
+                    _save_pbr_preview_pngs(tex_out[i], sample_dir)
+                    _save_mesh_preview_glb(geom_out[i], sample_dir, str(names[i] or ids[i]))
+
+                    report["samples"].append(
+                        {
+                            "object_id": ids[i],
+                            "name": names[i] if isinstance(names, list) else names,
+                            "category": categories[i] if isinstance(categories, list) else categories,
+                            "viewer_url": urls[i] if isinstance(urls, list) else urls,
+                            "folder": str(sample_dir.relative_to(ROOT)),
+                        }
+                    )
+    except Exception as exc:
+        _log(f"Validation render atlandi (egitim devam ediyor): {exc}")
+        report["export_error"] = str(exc)
+    finally:
+        geometry.train()
+        texture.train()
+
+    with open(step_dir / "validation_report.json", "w", encoding="utf-8") as handle:
+        json.dump(report, handle, indent=2, ensure_ascii=False)
+
+    _log(
+        f"Validation render kaydedildi: {step_dir} "
+        f"(geom_loss={geom_loss:.4f}, tex_loss={tex_loss:.4f})"
+    )
+    return step_dir
+
+
+def save_checkpoint(path: Path, epoch: int, global_step: int, geometry: nn.Module, texture: nn.Module) -> None:
+    payload = {
+        "epoch": epoch,
+        "global_step": global_step,
+        "geometry_adapter": geometry.state_dict(),
+        "texture_adapter": texture.state_dict(),
+        "low_vram": LOW_VRAM,
+    }
+    torch.save(payload, path)
+    epoch_path = CHECKPOINT_DIR / f"checkpoint_step_{global_step:06d}.pt"
+    torch.save(payload, epoch_path)
+    _log(f"Checkpoint kaydedildi: {path} (+ {epoch_path.name})")
+
+
+def maybe_upload_validation_to_hf(step_dir: Path, token: str) -> None:
+    if not token or os.getenv("MESHAI_UPLOAD_VAL", "0") != "1":
+        return
+    try:
+        from huggingface_hub import HfApi
+
+        api = HfApi()
+        for file_path in step_dir.rglob("*"):
+            if file_path.is_file():
+                rel = file_path.relative_to(ROOT).as_posix()
+                api.upload_file(
+                    path_or_fileobj=str(file_path),
+                    path_in_repo=rel,
+                    repo_id=HF_REPO,
+                    repo_type="model",
+                    token=token,
+                    commit_message=f"Validation samples step {step_dir.name}",
+                )
+        _log(f"Validation ornekleri HF'ye yuklendi: {step_dir.name}")
+    except Exception as exc:
+        _log(f"Validation HF upload atlandi: {exc}")
+
+
+def unfreeze_all_layers(model: nn.Module) -> None:
+    for param in model.parameters():
+        param.requires_grad = True
+
+
+def start_training(
+    epochs: int = 5,
+    resume_from: Path | None = None,
+    validation_every: int = 500,
+    val_ratio: float = 0.1,
+) -> None:
+    if torch.cuda.is_available():
+        torch.backends.cuda.matmul.allow_tf32 = True
+        torch.backends.cudnn.allow_tf32 = True
+        torch.set_float32_matmul_precision("high")
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    dtype = torch.float32
+    monitor = TrainMonitor()
+
+    _log(f"Pipeline surumu: {TRAIN_PIPELINE_VERSION} (otomatik izleme, TensorBoard kapali)")
+    _log(f"Donanim: {device.upper()} | Hassasiyet: float32 (kararlilik) | LOW_VRAM: {LOW_VRAM}")
+    _log(f"Izleme log: {PROGRESS_LOG.resolve()}")
+    _log(f"Validation cikti: {OUTPUT_DIR.resolve()}")
+    if torch.cuda.is_available():
+        _log(f"GPU: {torch.cuda.get_device_name(0)}")
+    log_vram("startup")
+
+    dataset = MeshAIGold5KDataset(token=HF_TOKEN)
+    train_idx, val_idx = split_train_val_indices(dataset, val_ratio)
+    train_set = Subset(dataset, train_idx)
+    val_set = Subset(dataset, val_idx)
+
+    train_loader = DataLoader(
+        train_set,
+        batch_size=4,
+        shuffle=True,
+        pin_memory=device == "cuda",
+        collate_fn=collate_batch,
+    )
+    val_loader = DataLoader(
+        val_set,
+        batch_size=4,
+        shuffle=False,
+        pin_memory=device == "cuda",
+        collate_fn=collate_batch,
+    )
+    _log(f"Veri motoru: {len(train_set)} train + {len(val_set)} val nesne.")
+
+    geometry = LoRAAdapter("trellis_geometry").to(device=device, dtype=dtype)
+    texture = LoRAAdapter("hunyuan_pbr").to(device=device, dtype=dtype)
+
+    unfreeze_all_layers(geometry)
+    unfreeze_all_layers(texture)
+
+    latest_ckpt = CHECKPOINT_DIR / "latest_model.pt"
+    global_step = 0
+    if resume_from and resume_from.exists():
+        state = torch.load(resume_from, map_location=device, weights_only=True)
+        if state.get("geometry_adapter") is not None:
+            geometry.load_state_dict(state["geometry_adapter"])
+        if state.get("texture_adapter") is not None:
+            texture.load_state_dict(state["texture_adapter"])
+        global_step = int(state.get("global_step", 0))
+        _log(f"Resume modu: {resume_from} (step {global_step})")
+
+    geom_opt = torch.optim.AdamW(geometry.parameters(), lr=5e-5, fused=False)
+    tex_opt = torch.optim.AdamW(texture.parameters(), lr=5e-5, fused=False)
+
+    def _train_step(adapter, optimizer, batch, loss_name: str) -> float | None:
+        optimizer.zero_grad(set_to_none=True)
+        loss = run_batch_step(adapter, batch, device)
+        if not torch.isfinite(loss):
+            if monitor.nan_skips < 3:
+                _log(f"NaN/Inf {loss_name} — batch atlandi.")
+            monitor.note_nan_skip(loss_name)
+            optimizer.zero_grad(set_to_none=True)
+            return None
+        loss.backward()
+        torch.nn.utils.clip_grad_norm_(adapter.parameters(), max_norm=1.0)
+        optimizer.step()
+        return float(loss.item())
+
+    for epoch in range(1, epochs + 1):
+        _log(f"--- Epoch {epoch}/{epochs} Baslatildi ---")
+
+        log_vram("before_geometry")
+        _log(">> Katman 1-3: Microsoft TRELLIS geometrisi fine-tune ediliyor...")
+        geometry.train()
+        geom_losses: list[float] = []
+        for batch in train_loader:
+            loss_val = _train_step(geometry, geom_opt, batch, "geometry")
+            if loss_val is None:
+                continue
+            global_step += 1
+            geom_losses.append(loss_val)
+            monitor.note_step(global_step, "geometry", loss_val)
+
+            if global_step % validation_every == 0:
+                val_geom = evaluate_adapter(geometry, val_loader, device)
+                val_tex = evaluate_adapter(texture, val_loader, device)
+                monitor.note_validation(global_step, val_geom, val_tex)
+                log_vram(f"validation_step_{global_step}")
+                step_dir = export_validation_samples(
+                    global_step, val_loader, geometry, texture, device, val_geom, val_tex
+                )
+                maybe_upload_validation_to_hf(step_dir, HF_TOKEN or "")
+
+        log_vram("geometry_completed")
+
+        clear_gpu_cache()
+
+        log_vram("before_texture")
+        _log(">> Katman 4-5: Tencent Hunyuan3D PBR doku motoru fine-tune ediliyor...")
+        texture.train()
+        tex_losses: list[float] = []
+        for batch in train_loader:
+            loss_val = _train_step(texture, tex_opt, batch, "texture")
+            if loss_val is None:
+                continue
+            global_step += 1
+            tex_losses.append(loss_val)
+            monitor.note_step(global_step, "texture", loss_val)
+
+            if global_step % validation_every == 0:
+                val_geom = evaluate_adapter(geometry, val_loader, device)
+                val_tex = evaluate_adapter(texture, val_loader, device)
+                monitor.note_validation(global_step, val_geom, val_tex)
+                log_vram(f"validation_step_{global_step}")
+                step_dir = export_validation_samples(
+                    global_step, val_loader, geometry, texture, device, val_geom, val_tex
+                )
+                maybe_upload_validation_to_hf(step_dir, HF_TOKEN or "")
+
+        log_vram("texture_completed")
+
+        val_geom = evaluate_adapter(geometry, val_loader, device)
+        val_tex = evaluate_adapter(texture, val_loader, device)
+        geom_mean = float(np.mean(geom_losses)) if geom_losses else float("nan")
+        tex_mean = float(np.mean(tex_losses)) if tex_losses else float("nan")
+        monitor.note_epoch_end(epoch, epochs, geom_mean, tex_mean, val_geom, val_tex)
+
+        save_checkpoint(latest_ckpt, epoch, global_step, geometry, texture)
+        epoch_val_dir = OUTPUT_DIR / f"epoch_{epoch:03d}"
+        export_validation_samples(
+            global_step, val_loader, geometry, texture, device, val_geom, val_tex
+        )
+        latest_step_dir = sorted(OUTPUT_DIR.glob("step_*"))[-1] if list(OUTPUT_DIR.glob("step_*")) else None
+        if latest_step_dir:
+            import shutil
+
+            if epoch_val_dir.exists():
+                shutil.rmtree(epoch_val_dir)
+            shutil.copytree(latest_step_dir, epoch_val_dir)
+
+        clear_gpu_cache()
+        _log(f"Epoch {epoch} tamamlandi. Guncel durum bulut hafizasina alindi.")
+        log_vram(f"epoch_{epoch}_done")
+
+    monitor.finish(ok=monitor.nan_skips < len(train_set))
+    _log("Egitim tamamlandi.")
+    _log(f"Ozet log: {PROGRESS_LOG}")
+    _log(f"Son durum: {STATUS_JSON}")
+    _log(f"Gorsel ornekler: {OUTPUT_DIR}")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="MeshAI real-data hybrid training pipeline")
+    parser.add_argument(
+        "--mode",
+        choices=("real", "stub", "faz2"),
+        default="real",
+        help="real=hibrit kopru; faz2=TRELLIS/Hunyuan LoRA; stub=legacy",
+    )
+    parser.add_argument("--epochs", type=int, default=5)
+    parser.add_argument("--resume_from", type=str, default="")
+    parser.add_argument("--validation-every", type=int, default=500, help="Kac stepte bir validation")
+    parser.add_argument("--checkpoint-every", type=int, default=100, help="Kac stepte bir local+HF checkpoint")
+    parser.add_argument("--val-split", type=float, default=0.1, help="Validation orani (sabit UID yoksa)")
+    parser.add_argument("--limit", type=int, default=0, help="Smoke: max obje sayisi (0=tum)")
+    parser.add_argument("--lora-rank", type=int, default=8, help="Faz2 LoRA rank")
+    parser.add_argument(
+        "--hf-preprocessed-repo",
+        default=os.getenv("HF_PREPROCESSED_REPO", "HayrettinIscan/MeshAI-Preprocessed-4K"),
+    )
+    parser.add_argument("--data-root", default="", help="Yerel preprocessed kok (opsiyonel)")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    if not HF_TOKEN:
+        _log("HATA: HF_TOKEN eksik! Bulut dogrulamasi olmadan egitim baslatilamaz.")
+        sys.exit(1)
+
+    args = parse_args()
+    resume = Path(args.resume_from) if args.resume_from else None
+    limit = args.limit if args.limit > 0 else None
+    data_root = Path(args.data_root) if args.data_root else None
+
+    if args.mode == "faz2":
+        from meshai_train.faz2_engine import start_faz2_training
+
+        monitor = TrainMonitor()
+        monitor.last["version"] = "v5.0-faz2"
+        monitor._save_status()
+        start_faz2_training(
+            monitor=monitor,
+            checkpoint_dir=CHECKPOINT_DIR,
+            output_dir=OUTPUT_DIR,
+            token=HF_TOKEN,
+            epochs=args.epochs,
+            resume_from=resume,
+            validation_every=args.validation_every,
+            val_ratio=args.val_split,
+            limit=limit,
+            hf_repo=args.hf_preprocessed_repo,
+            data_root=data_root,
+            log_fn=_log,
+            log_vram_fn=log_vram,
+            clear_gpu_fn=clear_gpu_cache,
+            load_val_uids_fn=load_validation_uids,
+            checkpoint_every=args.checkpoint_every,
+            lora_rank=args.lora_rank,
+        )
+    elif args.mode == "real":
+        from meshai_train.engine import start_real_training
+
+        monitor = TrainMonitor()
+        monitor.last["version"] = "v4.0-real"
+        monitor._save_status()
+        start_real_training(
+            monitor=monitor,
+            checkpoint_dir=CHECKPOINT_DIR,
+            output_dir=OUTPUT_DIR,
+            token=HF_TOKEN,
+            epochs=args.epochs,
+            resume_from=resume,
+            validation_every=args.validation_every,
+            val_ratio=args.val_split,
+            limit=limit,
+            hf_repo=args.hf_preprocessed_repo,
+            data_root=data_root,
+            log_fn=_log,
+            log_vram_fn=log_vram,
+            clear_gpu_fn=clear_gpu_cache,
+            load_val_uids_fn=load_validation_uids,
+            checkpoint_every=args.checkpoint_every,
+        )
+    else:
+        start_training(
+            epochs=args.epochs,
+            resume_from=resume,
+            validation_every=args.validation_every,
+            val_ratio=args.val_split,
+        )