Upload Mongle RunPod test package
Browse files- .gitattributes +4 -32
- .gitignore +12 -0
- README.md +33 -0
- models/lora_bg_v1/pytorch_lora_weights.safetensors +3 -0
- models/lora_v3_32bit/pytorch_lora_weights.safetensors +3 -0
- requirements.txt +23 -0
- src/feed/feed_generator_1.py +125 -0
- src/feed/feed_generator_2.py +147 -0
- src/pipeline/persona2prompt.py +102 -0
- test_image2feed_pipeline.py +351 -0
- test_qwen25_vl_extract.py +375 -0
- test_text2feed_pipeline.py +398 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,7 @@
|
|
| 1 |
-
*.
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.
|
| 25 |
-
*.
|
| 26 |
-
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 2 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 4 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.png filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.jpg filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.jpeg filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.gitignore
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.env
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
.DS_Store
|
| 5 |
+
outputs/
|
| 6 |
+
data/
|
| 7 |
+
image/
|
| 8 |
+
grid/
|
| 9 |
+
*.log
|
| 10 |
+
train_log.txt
|
| 11 |
+
extract_log.txt
|
| 12 |
+
extract_err.txt
|
README.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Mongle HF Test Package
|
| 2 |
+
|
| 3 |
+
This folder is a clean test package for uploading to a Hugging Face repository and cloning on RunPod.
|
| 4 |
+
|
| 5 |
+
## Included
|
| 6 |
+
|
| 7 |
+
- `test_text2feed_pipeline.py`
|
| 8 |
+
- `test_image2feed_pipeline.py`
|
| 9 |
+
- `test_qwen25_vl_extract.py`
|
| 10 |
+
- `src/pipeline/persona2prompt.py`
|
| 11 |
+
- `src/feed/feed_generator_1.py`
|
| 12 |
+
- `src/feed/feed_generator_2.py`
|
| 13 |
+
- `models/lora_v3_32bit/pytorch_lora_weights.safetensors`
|
| 14 |
+
- `models/lora_bg_v1/pytorch_lora_weights.safetensors`
|
| 15 |
+
- `requirements.txt`
|
| 16 |
+
|
| 17 |
+
## Not Included
|
| 18 |
+
|
| 19 |
+
- `.env`
|
| 20 |
+
- local outputs
|
| 21 |
+
- training datasets
|
| 22 |
+
- logs
|
| 23 |
+
- caches
|
| 24 |
+
|
| 25 |
+
## RunPod Quick Test
|
| 26 |
+
|
| 27 |
+
```bash
|
| 28 |
+
pip install -r requirements.txt
|
| 29 |
+
python test_text2feed_pipeline.py --name runpod_text_test --force
|
| 30 |
+
python test_image2feed_pipeline.py --name runpod_image_test --character-image image/01.jpg --force
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
For `test_image2feed_pipeline.py`, upload a test image to `image/01.jpg` or pass another image path with `--character-image`.
|
models/lora_bg_v1/pytorch_lora_weights.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b11ce0f1e8ebce873e6889598e2a6be5eb63d852ba468dbad921f22d4d4c9400
|
| 3 |
+
size 185963768
|
models/lora_v3_32bit/pytorch_lora_weights.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f935fa5276540033ab51f677a659c41624f26b093c21d91f1bea93fce408c60f
|
| 3 |
+
size 185963768
|
requirements.txt
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
--extra-index-url https://download.pytorch.org/whl/cu121
|
| 2 |
+
|
| 3 |
+
torch==2.5.1
|
| 4 |
+
torchvision==0.20.1
|
| 5 |
+
|
| 6 |
+
diffusers==0.31.0
|
| 7 |
+
transformers==4.46.3
|
| 8 |
+
accelerate==1.1.1
|
| 9 |
+
peft==0.13.2
|
| 10 |
+
huggingface_hub==0.26.5
|
| 11 |
+
safetensors
|
| 12 |
+
sentencepiece
|
| 13 |
+
protobuf
|
| 14 |
+
qwen-vl-utils
|
| 15 |
+
bitsandbytes
|
| 16 |
+
|
| 17 |
+
rembg
|
| 18 |
+
onnxruntime-gpu
|
| 19 |
+
opencv-python-headless
|
| 20 |
+
Pillow
|
| 21 |
+
matplotlib
|
| 22 |
+
numpy
|
| 23 |
+
runpod
|
src/feed/feed_generator_1.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
몽글마을 피드 이미지 생성기
|
| 3 |
+
입력: 캐릭터 외형(Florence-2 추출) + 퀘스트(영어)
|
| 4 |
+
출력: 피드 이미지 (캐릭터가 퀘스트를 수행하는 장면)
|
| 5 |
+
"""
|
| 6 |
+
import os as _os; _os.chdir(_os.path.dirname(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))))
|
| 7 |
+
import os, gc
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
|
| 10 |
+
|
| 11 |
+
from diffusers import StableDiffusionXLPipeline, DPMSolverMultistepScheduler
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
CHARACTER_LORA = "models/lora_v3_32bit/pytorch_lora_weights.safetensors"
|
| 15 |
+
BG_LORA = "models/lora_bg_v1/pytorch_lora_weights.safetensors"
|
| 16 |
+
|
| 17 |
+
NEGATIVE = (
|
| 18 |
+
"realistic, 3d render, photograph, blurry, dark, gloomy, "
|
| 19 |
+
"watercolor, sketch, painterly, smooth illustration, "
|
| 20 |
+
"modern city, urban, scary, violence, text, watermark, "
|
| 21 |
+
"multiple characters, tiling, repeated"
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _to_appearance_str(appearance) -> str:
|
| 26 |
+
"""dict 또는 string → 외형 텍스트 변환"""
|
| 27 |
+
if isinstance(appearance, str):
|
| 28 |
+
return appearance.strip()
|
| 29 |
+
if not isinstance(appearance, dict):
|
| 30 |
+
return str(appearance).strip()
|
| 31 |
+
|
| 32 |
+
animal = appearance.get("animal_type", "bear")
|
| 33 |
+
color = appearance.get("body_color", "")
|
| 34 |
+
shape = appearance.get("body_shape", "")
|
| 35 |
+
eye = appearance.get("eye_style", "")
|
| 36 |
+
nose = appearance.get("nose_mouth") or appearance.get("nose_shape", "")
|
| 37 |
+
cheeks = appearance.get("cheeks", "")
|
| 38 |
+
ears = appearance.get("ear_shape", "")
|
| 39 |
+
texture = appearance.get("texture", "")
|
| 40 |
+
acc = appearance.get("accessories", "")
|
| 41 |
+
features = appearance.get("distinctive_features", [])
|
| 42 |
+
secondary = appearance.get("secondary_colors", [])
|
| 43 |
+
|
| 44 |
+
parts = [f"a {color} pixel art stuffed {animal} character".strip()]
|
| 45 |
+
if shape: parts.append(shape)
|
| 46 |
+
if isinstance(secondary, list):
|
| 47 |
+
# 3단어 이하인 짧은 색상 값만 포함 (문장형 묘사 제외)
|
| 48 |
+
parts.extend(s for s in secondary[:2] if s and len(str(s).split()) <= 3)
|
| 49 |
+
if eye: parts.append(eye)
|
| 50 |
+
if cheeks and "no" not in cheeks.lower(): parts.append(cheeks)
|
| 51 |
+
if ears: parts.append(ears)
|
| 52 |
+
if nose: parts.append(nose)
|
| 53 |
+
if texture: parts.append(texture)
|
| 54 |
+
if isinstance(acc, list):
|
| 55 |
+
acc = ", ".join(str(a) for a in acc if a)
|
| 56 |
+
if acc and str(acc).lower() != "none": parts.append(acc)
|
| 57 |
+
# distinctive_features는 eye_style, cheeks 등과 중복되므로 제외
|
| 58 |
+
|
| 59 |
+
return ", ".join(p for p in parts if p)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def build_prompt(appearance, quest_en: str):
|
| 63 |
+
"""
|
| 64 |
+
prompt → 장면/스타일 (CLIP-L)
|
| 65 |
+
prompt_2 → 장면 + 캐릭터 외형 (OpenCLIP-bigG, 더 강한 인코더에 둘 다 전달)
|
| 66 |
+
"""
|
| 67 |
+
appearance_str = _to_appearance_str(appearance)
|
| 68 |
+
prompt = (
|
| 69 |
+
f"monglestyle, {quest_en}, "
|
| 70 |
+
f"cute chibi stuffed animal character in action, "
|
| 71 |
+
f"full body pose, 32-bit pixel art scene, pastel colors"
|
| 72 |
+
)
|
| 73 |
+
prompt_2 = f"monglestyle, {appearance_str}"
|
| 74 |
+
return prompt, prompt_2
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def load_pipeline():
|
| 78 |
+
print("SDXL 로드 중 (캐릭터 LoRA + 배경 LoRA)...")
|
| 79 |
+
pipe = StableDiffusionXLPipeline.from_pretrained(
|
| 80 |
+
"stabilityai/stable-diffusion-xl-base-1.0",
|
| 81 |
+
torch_dtype=torch.float16,
|
| 82 |
+
use_safetensors=True,
|
| 83 |
+
).to("cuda")
|
| 84 |
+
|
| 85 |
+
if not Path(CHARACTER_LORA).exists():
|
| 86 |
+
raise FileNotFoundError(f"Character LoRA not found: {CHARACTER_LORA}")
|
| 87 |
+
if not Path(BG_LORA).exists():
|
| 88 |
+
raise FileNotFoundError(f"Background LoRA not found: {BG_LORA}")
|
| 89 |
+
|
| 90 |
+
pipe.load_lora_weights(CHARACTER_LORA, adapter_name="character")
|
| 91 |
+
pipe.load_lora_weights(BG_LORA, adapter_name="bg")
|
| 92 |
+
pipe.set_adapters(
|
| 93 |
+
["character", "bg"],
|
| 94 |
+
adapter_weights=[0.8, 0.8],
|
| 95 |
+
)
|
| 96 |
+
pipe.scheduler = DPMSolverMultistepScheduler.from_config(
|
| 97 |
+
pipe.scheduler.config, use_karras_sigmas=True
|
| 98 |
+
)
|
| 99 |
+
pipe.enable_attention_slicing()
|
| 100 |
+
print("로드 완료!\n")
|
| 101 |
+
return pipe
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def generate(appearance, quest_en: str, pipe, seed: int = 42):
|
| 105 |
+
prompt, prompt_2 = build_prompt(appearance, quest_en)
|
| 106 |
+
print(f" prompt : {prompt}")
|
| 107 |
+
print(f" prompt_2: {prompt_2}")
|
| 108 |
+
|
| 109 |
+
return pipe(
|
| 110 |
+
prompt=prompt,
|
| 111 |
+
prompt_2=prompt_2,
|
| 112 |
+
negative_prompt=NEGATIVE,
|
| 113 |
+
num_inference_steps=20,
|
| 114 |
+
guidance_scale=7.5,
|
| 115 |
+
height=1024,
|
| 116 |
+
width=1024,
|
| 117 |
+
generator=torch.Generator("cuda").manual_seed(seed),
|
| 118 |
+
).images[0]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def unload_pipeline(pipe):
|
| 122 |
+
del pipe
|
| 123 |
+
gc.collect()
|
| 124 |
+
torch.cuda.empty_cache()
|
| 125 |
+
print("파이프라인 VRAM 해제 완료")
|
src/feed/feed_generator_2.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
몽글마을 피드 이미지 생성기 v2 (이미지 입력 파이프라인 전용)
|
| 3 |
+
|
| 4 |
+
feed_generator_1.py와의 차이점:
|
| 5 |
+
- 캐릭터 외형을 프롬프트 앞쪽에 배치 (SDXL 토큰 가중치 활용)
|
| 6 |
+
- appearance_str에 limbs 포함 (비전형 캐릭터 신체 반영)
|
| 7 |
+
- NEGATIVE에 human/person 추가
|
| 8 |
+
- char LoRA 0.9 / bg LoRA 1.1 (외형 강제력 상향)
|
| 9 |
+
- guidance_scale 8.5 (프롬프트 추종력 강화)
|
| 10 |
+
- 양쪽 인코더 모두 scene 정보 전달
|
| 11 |
+
"""
|
| 12 |
+
import os as _os; _os.chdir(_os.path.dirname(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))))
|
| 13 |
+
import os, gc
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
|
| 16 |
+
|
| 17 |
+
from diffusers import StableDiffusionXLPipeline, DPMSolverMultistepScheduler
|
| 18 |
+
import torch
|
| 19 |
+
|
| 20 |
+
CHARACTER_LORA = "models/lora_v3_32bit/pytorch_lora_weights.safetensors"
|
| 21 |
+
BG_LORA = "models/lora_bg_v1/pytorch_lora_weights.safetensors"
|
| 22 |
+
|
| 23 |
+
NEGATIVE = (
|
| 24 |
+
"realistic, 3d render, photograph, blurry, dark, gloomy, "
|
| 25 |
+
"watercolor, sketch, painterly, smooth illustration, "
|
| 26 |
+
"modern city, urban, scary, violence, text, watermark, "
|
| 27 |
+
"multiple characters, tiling, repeated, "
|
| 28 |
+
"human, person, girl, boy, man, woman, humanoid, human body, human face, "
|
| 29 |
+
"skin, hair, clothes, dress, shirt"
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _to_appearance_str(appearance) -> str:
|
| 34 |
+
if isinstance(appearance, str):
|
| 35 |
+
return appearance.strip()
|
| 36 |
+
if not isinstance(appearance, dict):
|
| 37 |
+
return str(appearance).strip()
|
| 38 |
+
|
| 39 |
+
animal = appearance.get("animal_type", "bear")
|
| 40 |
+
color = appearance.get("body_color", "")
|
| 41 |
+
shape = appearance.get("body_shape", "")
|
| 42 |
+
eye = appearance.get("eye_style", "")
|
| 43 |
+
nose = appearance.get("nose_mouth") or appearance.get("nose_shape", "")
|
| 44 |
+
cheeks = appearance.get("cheeks", "")
|
| 45 |
+
ears = appearance.get("ear_shape", "")
|
| 46 |
+
limbs = appearance.get("limbs", "")
|
| 47 |
+
texture = appearance.get("texture", "")
|
| 48 |
+
acc = appearance.get("accessories", "")
|
| 49 |
+
secondary = appearance.get("secondary_colors", [])
|
| 50 |
+
|
| 51 |
+
parts = [f"a {color} pixel art stuffed {animal} character".strip()]
|
| 52 |
+
if shape: parts.append(shape)
|
| 53 |
+
if isinstance(secondary, list):
|
| 54 |
+
parts.extend(s for s in secondary[:2] if s and len(str(s).split()) <= 3)
|
| 55 |
+
if eye: parts.append(eye)
|
| 56 |
+
if cheeks and "no" not in cheeks.lower(): parts.append(cheeks)
|
| 57 |
+
if ears and "not visible" not in ears.lower(): parts.append(ears)
|
| 58 |
+
if nose and "not visible" not in nose.lower(): parts.append(nose)
|
| 59 |
+
if limbs and "not visible" not in limbs.lower(): parts.append(limbs)
|
| 60 |
+
if texture: parts.append(texture)
|
| 61 |
+
if isinstance(acc, list):
|
| 62 |
+
acc = ", ".join(str(a) for a in acc if a)
|
| 63 |
+
if acc and str(acc).lower() not in ("none", ""): parts.append(acc)
|
| 64 |
+
|
| 65 |
+
return ", ".join(p for p in parts if p)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def build_prompt(appearance, quest_en: str):
|
| 69 |
+
"""
|
| 70 |
+
두 인코더 역할 분리:
|
| 71 |
+
prompt (CLIP-L) → 장면/배경 담당: scene 앞에 배치
|
| 72 |
+
prompt_2 (OpenCLIP-bigG) → 캐릭터 외형 담당: appearance 앞에 배치
|
| 73 |
+
"""
|
| 74 |
+
appearance_str = _to_appearance_str(appearance)
|
| 75 |
+
|
| 76 |
+
animal = "plush mascot"
|
| 77 |
+
if isinstance(appearance, dict):
|
| 78 |
+
animal = appearance.get("animal_type", "plush mascot")
|
| 79 |
+
|
| 80 |
+
# CLIP-L: 장면 먼저 → 배경 생성 담당
|
| 81 |
+
# 캐릭터 타입만 간단히 명시해 human 방지
|
| 82 |
+
prompt = (
|
| 83 |
+
f"monglestyle, {quest_en}, "
|
| 84 |
+
f"stuffed {animal} plush toy mascot, "
|
| 85 |
+
f"cozy pastel sky island village, "
|
| 86 |
+
f"32-bit pixel art scene, detailed background, outdoor scenery, pastel colors"
|
| 87 |
+
)
|
| 88 |
+
# OpenCLIP-bigG: 외형 먼저 → 캐릭터 정체성 담당
|
| 89 |
+
# 장면 힌트도 뒤에 포함해 배경 보조
|
| 90 |
+
prompt_2 = (
|
| 91 |
+
f"monglestyle, stuffed {animal} plush toy mascot, "
|
| 92 |
+
f"{appearance_str}, "
|
| 93 |
+
f"full body in action, {quest_en}, "
|
| 94 |
+
f"pixel art background scene"
|
| 95 |
+
)
|
| 96 |
+
return prompt, prompt_2
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def load_pipeline():
|
| 100 |
+
print("SDXL 로드 중 (캐릭터 LoRA + 배경 LoRA)...")
|
| 101 |
+
pipe = StableDiffusionXLPipeline.from_pretrained(
|
| 102 |
+
"stabilityai/stable-diffusion-xl-base-1.0",
|
| 103 |
+
torch_dtype=torch.float16,
|
| 104 |
+
use_safetensors=True,
|
| 105 |
+
).to("cuda")
|
| 106 |
+
|
| 107 |
+
if not Path(CHARACTER_LORA).exists():
|
| 108 |
+
raise FileNotFoundError(f"Character LoRA not found: {CHARACTER_LORA}")
|
| 109 |
+
if not Path(BG_LORA).exists():
|
| 110 |
+
raise FileNotFoundError(f"Background LoRA not found: {BG_LORA}")
|
| 111 |
+
|
| 112 |
+
pipe.load_lora_weights(CHARACTER_LORA, adapter_name="character")
|
| 113 |
+
pipe.load_lora_weights(BG_LORA, adapter_name="bg")
|
| 114 |
+
pipe.set_adapters(
|
| 115 |
+
["character", "bg"],
|
| 116 |
+
adapter_weights=[0.9, 1.1],
|
| 117 |
+
)
|
| 118 |
+
pipe.scheduler = DPMSolverMultistepScheduler.from_config(
|
| 119 |
+
pipe.scheduler.config, use_karras_sigmas=True
|
| 120 |
+
)
|
| 121 |
+
pipe.enable_attention_slicing()
|
| 122 |
+
print("로드 완료!\n")
|
| 123 |
+
return pipe
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def generate(appearance, quest_en: str, pipe, seed: int = 42):
|
| 127 |
+
prompt, prompt_2 = build_prompt(appearance, quest_en)
|
| 128 |
+
print(f" prompt : {prompt}")
|
| 129 |
+
print(f" prompt_2: {prompt_2}")
|
| 130 |
+
|
| 131 |
+
return pipe(
|
| 132 |
+
prompt=prompt,
|
| 133 |
+
prompt_2=prompt_2,
|
| 134 |
+
negative_prompt=NEGATIVE,
|
| 135 |
+
num_inference_steps=25,
|
| 136 |
+
guidance_scale=8.5,
|
| 137 |
+
height=1024,
|
| 138 |
+
width=1024,
|
| 139 |
+
generator=torch.Generator("cuda").manual_seed(seed),
|
| 140 |
+
).images[0]
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def unload_pipeline(pipe):
|
| 144 |
+
del pipe
|
| 145 |
+
gc.collect()
|
| 146 |
+
torch.cuda.empty_cache()
|
| 147 |
+
print("파이프라인 VRAM 해제 완료")
|
src/pipeline/persona2prompt.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
한국어 페르소나 텍스트 → 영어 시각 프롬프트 변환
|
| 3 |
+
모델: Qwen/Qwen2.5-7B-Instruct
|
| 4 |
+
"""
|
| 5 |
+
import os
|
| 6 |
+
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
|
| 7 |
+
|
| 8 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
MODEL_ID = "Qwen/Qwen2.5-7B-Instruct"
|
| 12 |
+
|
| 13 |
+
SYSTEM_PROMPT = """You are a visual prompt writer for 몽글마을 (Mongle Village), a cozy pixel art character game.
|
| 14 |
+
|
| 15 |
+
Your job: convert a Korean character description into a short English visual prompt for pixel art image generation.
|
| 16 |
+
|
| 17 |
+
Rules:
|
| 18 |
+
- Focus only on VISUAL traits: animal type, fur/body color, facial expression, body shape
|
| 19 |
+
- Personality keywords → visual expression (e.g. 용감한=confident bright eyes, 사랑스러운=rosy cheeks warm smile, 조용한=calm soft eyes, 활발한=sparkling cheerful eyes, 수줍은=blushing shy smile)
|
| 20 |
+
- If no animal mentioned → pick a soft round animal (bunny, bear, cat, hamster)
|
| 21 |
+
- If no color mentioned → pick a pastel color (cream, soft pink, light blue, lavender, mint)
|
| 22 |
+
- Output ONLY the English prompt. No explanations, no Korean, nothing else.
|
| 23 |
+
- Keep it under 60 words
|
| 24 |
+
- Always end with: single character, full body, centered, stuffed animal toy, 32-bit pixel art sprite, pure white background
|
| 25 |
+
|
| 26 |
+
Example:
|
| 27 |
+
Input: 이 친구의 이름은 최벨리입니다. 용감하고 사랑스러워요.
|
| 28 |
+
Output: white fluffy bunny, confident bright eyes, rosy cheeks, warm gentle smile, soft round chubby body, single character, full body, centered, stuffed animal toy, 32-bit pixel art sprite, pure white background"""
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def load_qwen():
|
| 32 |
+
print(f"Qwen2.5-7B 로드 중... ({MODEL_ID})")
|
| 33 |
+
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
| 34 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 35 |
+
MODEL_ID,
|
| 36 |
+
torch_dtype=torch.float16,
|
| 37 |
+
device_map="auto", # GPU/CPU 자동 분배 (12GB GPU에서 안전)
|
| 38 |
+
).eval()
|
| 39 |
+
print("Qwen 로드 완료!\n")
|
| 40 |
+
return model, tokenizer
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def persona_to_prompt(korean_text: str, model, tokenizer) -> str:
|
| 44 |
+
messages = [
|
| 45 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 46 |
+
{"role": "user", "content": korean_text},
|
| 47 |
+
]
|
| 48 |
+
text = tokenizer.apply_chat_template(
|
| 49 |
+
messages,
|
| 50 |
+
tokenize=False,
|
| 51 |
+
add_generation_prompt=True,
|
| 52 |
+
)
|
| 53 |
+
inputs = tokenizer([text], return_tensors="pt").to("cuda")
|
| 54 |
+
|
| 55 |
+
with torch.no_grad():
|
| 56 |
+
outputs = model.generate(
|
| 57 |
+
**inputs,
|
| 58 |
+
max_new_tokens=120,
|
| 59 |
+
do_sample=False,
|
| 60 |
+
temperature=None,
|
| 61 |
+
top_p=None,
|
| 62 |
+
pad_token_id=tokenizer.eos_token_id,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
generated = outputs[0][inputs.input_ids.shape[1]:]
|
| 66 |
+
result = tokenizer.decode(generated, skip_special_tokens=True).strip()
|
| 67 |
+
return result
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def unload_qwen(model, tokenizer):
|
| 71 |
+
# accelerate dispatch hook 먼저 제거 (device_map="auto" 사용 시 필수)
|
| 72 |
+
try:
|
| 73 |
+
from accelerate.hooks import remove_hook_from_module
|
| 74 |
+
remove_hook_from_module(model, recurse=True)
|
| 75 |
+
print("accelerate hook 제거 완료")
|
| 76 |
+
except Exception as e:
|
| 77 |
+
print(f"hook 제거 스킵: {e}")
|
| 78 |
+
model.cpu() # GPU → CPU로 이동
|
| 79 |
+
del model, tokenizer
|
| 80 |
+
import gc; gc.collect()
|
| 81 |
+
torch.cuda.empty_cache()
|
| 82 |
+
torch.cuda.synchronize()
|
| 83 |
+
print("Qwen VRAM 해제 완료\n")
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# ── 단독 테스트 ──────────────────────────────────────────────
|
| 87 |
+
if __name__ == "__main__":
|
| 88 |
+
samples = [
|
| 89 |
+
"이 친구의 이름은 최벨리입니다. 용감하고, 사랑스러워요. 밤마다 저의 이야기를 잘 들어주고, 포근하게 안아준답니다",
|
| 90 |
+
"몽실이는 조용하고 차분한 성격이에요. 항상 따뜻한 눈빛으로 바라봐주고, 힘든 날엔 옆에서 말없이 함께 있어줘요",
|
| 91 |
+
"두부는 엄청 활발하고 장난기가 넘쳐요! 매일 새로운 모험을 꿈꾸고, 친구들에게 먼저 다가가는 붙임성 좋은 친구예요",
|
| 92 |
+
"솜솜이는 수줍음이 많지만 일단 친해지면 엄청 다정해요. 작고 포근한 외모처럼 마음도 보드랍고 섬세한 친구랍니다",
|
| 93 |
+
]
|
| 94 |
+
|
| 95 |
+
model, tokenizer = load_qwen()
|
| 96 |
+
|
| 97 |
+
for i, text in enumerate(samples, 1):
|
| 98 |
+
prompt = persona_to_prompt(text, model, tokenizer)
|
| 99 |
+
print(f"[{i:02d}] 입력: {text[:40]}...")
|
| 100 |
+
print(f" 출력: {prompt}\n")
|
| 101 |
+
|
| 102 |
+
unload_qwen(model, tokenizer)
|
test_image2feed_pipeline.py
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Image-to-feed end-to-end test pipeline.
|
| 3 |
+
|
| 4 |
+
Flow:
|
| 5 |
+
Real character image (input)
|
| 6 |
+
-> rembg background removal
|
| 7 |
+
-> Qwen2.5-VL appearance JSON extraction (pixel_prompt_core)
|
| 8 |
+
-> SDXL + lora_v3_32bit pixel art character image
|
| 9 |
+
-> Korean quest to English scene text
|
| 10 |
+
-> SDXL + lora_v3_32bit + lora_bg_v1 feed image
|
| 11 |
+
|
| 12 |
+
Outputs:
|
| 13 |
+
outputs/image2feed/<name>/
|
| 14 |
+
input_nobg.png
|
| 15 |
+
appearance_raw.txt
|
| 16 |
+
appearance.json
|
| 17 |
+
character.png
|
| 18 |
+
quest.json
|
| 19 |
+
feed.png
|
| 20 |
+
results.json
|
| 21 |
+
"""
|
| 22 |
+
import argparse
|
| 23 |
+
import gc
|
| 24 |
+
import json
|
| 25 |
+
import os
|
| 26 |
+
import sys
|
| 27 |
+
import traceback
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
|
| 30 |
+
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
|
| 31 |
+
|
| 32 |
+
ROOT = Path(__file__).resolve().parent
|
| 33 |
+
os.chdir(ROOT)
|
| 34 |
+
sys.path.insert(0, str(ROOT))
|
| 35 |
+
|
| 36 |
+
OUTPUT_ROOT = Path("outputs/image2feed")
|
| 37 |
+
CHAR_LORA_DIR = "models/lora_v3_32bit"
|
| 38 |
+
|
| 39 |
+
DEFAULT_QUEST_KO = "공원에서 30분 달리기를 완료했어요!"
|
| 40 |
+
|
| 41 |
+
CHARACTER_NEGATIVE = (
|
| 42 |
+
"realistic, photograph, 3d render, human, person, anime, scary, "
|
| 43 |
+
"complex background, multiple characters, text, watermark, logo, "
|
| 44 |
+
"extra limbs, long limbs, harsh black outline, low quality, blurry"
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
QUEST_SYSTEM = """You are an action-pose scene prompt writer for Mongle Village, a cozy sky island pixel art village.
|
| 48 |
+
|
| 49 |
+
Convert a Korean quest completion message into a short English scene description for image generation.
|
| 50 |
+
|
| 51 |
+
Rules:
|
| 52 |
+
- The FIRST words must describe the character's visible action pose.
|
| 53 |
+
- Keep the activity from the quest very explicit.
|
| 54 |
+
- Include body movement cues such as arms swinging, legs moving, holding a book, stirring a pot, walking steps.
|
| 55 |
+
- Describe one simple matching environment after the action.
|
| 56 |
+
- Set it in a cozy pastel sky island village world.
|
| 57 |
+
- Mention only one character.
|
| 58 |
+
- Do not write only a landscape/background description.
|
| 59 |
+
- Do not make the character standing still unless the quest is resting.
|
| 60 |
+
- 16-28 words max.
|
| 61 |
+
- Output ONLY the English scene description.
|
| 62 |
+
|
| 63 |
+
Examples:
|
| 64 |
+
Input: 공원에서 30분 달리기를 완료했어요!
|
| 65 |
+
Output: running with arms swinging and legs in motion along a fluffy cloud meadow path
|
| 66 |
+
|
| 67 |
+
Input: 오늘 책 한 권을 다 읽었어요
|
| 68 |
+
Output: sitting and holding an open storybook under a blossoming cloud tree beside a cozy cottage
|
| 69 |
+
|
| 70 |
+
Input: 직접 요리해서 건강한 밥을 먹었어요
|
| 71 |
+
Output: cooking with both paws stirring a pot in a cozy cottage kitchen with fresh vegetables
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def parse_args():
|
| 76 |
+
parser = argparse.ArgumentParser()
|
| 77 |
+
parser.add_argument("--name", default="sample_01")
|
| 78 |
+
parser.add_argument("--character-image", required=True, help="Path to the input character image.")
|
| 79 |
+
parser.add_argument("--quest-ko", default=DEFAULT_QUEST_KO)
|
| 80 |
+
parser.add_argument("--quest-en", default="", help="If provided, skips Qwen text translation.")
|
| 81 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 82 |
+
parser.add_argument("--character-steps", type=int, default=30)
|
| 83 |
+
parser.add_argument("--feed-seed", type=int, default=123, help="Feed generation seed.")
|
| 84 |
+
parser.add_argument("--force", action="store_true")
|
| 85 |
+
parser.add_argument("--force-char", action="store_true")
|
| 86 |
+
parser.add_argument("--force-quest", action="store_true")
|
| 87 |
+
parser.add_argument("--force-feed", action="store_true")
|
| 88 |
+
parser.add_argument("--no-4bit-vlm", action="store_true")
|
| 89 |
+
return parser.parse_args()
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def remove_background(image_path: Path, out_path: Path):
|
| 93 |
+
from PIL import Image
|
| 94 |
+
from rembg import remove
|
| 95 |
+
|
| 96 |
+
print("Removing background...", flush=True)
|
| 97 |
+
image = Image.open(image_path).convert("RGBA")
|
| 98 |
+
removed = remove(image)
|
| 99 |
+
white_bg = Image.new("RGBA", removed.size, (255, 255, 255, 255))
|
| 100 |
+
white_bg.paste(removed, mask=removed.split()[3])
|
| 101 |
+
result = white_bg.convert("RGB")
|
| 102 |
+
result.save(out_path)
|
| 103 |
+
print(f"Background removed: {out_path}", flush=True)
|
| 104 |
+
return result
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def extract_appearance(nobg_image, out_dir: Path, use_4bit: bool):
|
| 108 |
+
from test_qwen25_vl_extract import (
|
| 109 |
+
extract_json_from_text,
|
| 110 |
+
load_model,
|
| 111 |
+
normalize_info,
|
| 112 |
+
run_extraction,
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
print("Loading Qwen2.5-VL appearance extractor...", flush=True)
|
| 116 |
+
model, processor = load_model("Qwen/Qwen2.5-VL-7B-Instruct", use_4bit=use_4bit)
|
| 117 |
+
raw = run_extraction(nobg_image, model, processor, max_new_tokens=900)
|
| 118 |
+
|
| 119 |
+
raw_path = out_dir / "appearance_raw.txt"
|
| 120 |
+
raw_path.write_text(raw, encoding="utf-8")
|
| 121 |
+
|
| 122 |
+
info = normalize_info(extract_json_from_text(raw))
|
| 123 |
+
appearance_path = out_dir / "appearance.json"
|
| 124 |
+
appearance_path.write_text(json.dumps(info, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 125 |
+
print(f"Appearance saved: {appearance_path}", flush=True)
|
| 126 |
+
print(f" pixel_prompt_core: {info.get('pixel_prompt_core', '')}", flush=True)
|
| 127 |
+
|
| 128 |
+
del model, processor
|
| 129 |
+
gc.collect()
|
| 130 |
+
try:
|
| 131 |
+
import torch
|
| 132 |
+
torch.cuda.empty_cache()
|
| 133 |
+
except Exception:
|
| 134 |
+
pass
|
| 135 |
+
return info
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def build_character_prompt(appearance: dict):
|
| 139 |
+
from src.feed.feed_generator_1 import _to_appearance_str
|
| 140 |
+
|
| 141 |
+
appearance_str = _to_appearance_str(appearance)
|
| 142 |
+
body_shape = appearance.get("body_shape", "") if isinstance(appearance, dict) else ""
|
| 143 |
+
pose = appearance.get("pose", "") if isinstance(appearance, dict) else ""
|
| 144 |
+
|
| 145 |
+
shape_hint = f"{body_shape}, " if body_shape else ""
|
| 146 |
+
pose_hint = f"{pose}, " if pose else ""
|
| 147 |
+
|
| 148 |
+
# CLIP-L: 스타일 + 원본의 체형/자세 (비율 강제 없음)
|
| 149 |
+
prompt = (
|
| 150 |
+
"monglestyle, "
|
| 151 |
+
f"{shape_hint}{pose_hint}"
|
| 152 |
+
"single stuffed animal toy mascot character, full body, centered, "
|
| 153 |
+
"32-bit pixel art sprite, "
|
| 154 |
+
"soft pixel shading, clean silhouette, soft brown outline, "
|
| 155 |
+
"pure white background"
|
| 156 |
+
)
|
| 157 |
+
# OpenCLIP-bigG: 캐릭터 외형 전체
|
| 158 |
+
prompt_2 = f"monglestyle, {appearance_str}"
|
| 159 |
+
return prompt, prompt_2
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def generate_character(appearance: dict, out_path: Path, seed: int, steps: int):
|
| 163 |
+
import torch
|
| 164 |
+
from diffusers import DPMSolverMultistepScheduler, StableDiffusionXLPipeline
|
| 165 |
+
|
| 166 |
+
print("Loading SDXL character pipeline...", flush=True)
|
| 167 |
+
pipe = StableDiffusionXLPipeline.from_pretrained(
|
| 168 |
+
"stabilityai/stable-diffusion-xl-base-1.0",
|
| 169 |
+
torch_dtype=torch.float16,
|
| 170 |
+
use_safetensors=True,
|
| 171 |
+
)
|
| 172 |
+
pipe.load_lora_weights(CHAR_LORA_DIR)
|
| 173 |
+
pipe.fuse_lora(lora_scale=0.9)
|
| 174 |
+
pipe.unload_lora_weights()
|
| 175 |
+
pipe.scheduler = DPMSolverMultistepScheduler.from_config(
|
| 176 |
+
pipe.scheduler.config, use_karras_sigmas=True
|
| 177 |
+
)
|
| 178 |
+
pipe.to("cuda")
|
| 179 |
+
pipe.enable_attention_slicing()
|
| 180 |
+
|
| 181 |
+
prompt, prompt_2 = build_character_prompt(appearance)
|
| 182 |
+
print(f"Character prompt : {prompt}", flush=True)
|
| 183 |
+
print(f"Character prompt_2: {prompt_2}", flush=True)
|
| 184 |
+
image = pipe(
|
| 185 |
+
prompt=prompt,
|
| 186 |
+
prompt_2=prompt_2,
|
| 187 |
+
negative_prompt=CHARACTER_NEGATIVE,
|
| 188 |
+
num_inference_steps=steps,
|
| 189 |
+
guidance_scale=7.5,
|
| 190 |
+
height=1024,
|
| 191 |
+
width=1024,
|
| 192 |
+
generator=torch.Generator("cuda").manual_seed(seed),
|
| 193 |
+
).images[0]
|
| 194 |
+
image.save(out_path)
|
| 195 |
+
print(f"Character saved: {out_path}", flush=True)
|
| 196 |
+
|
| 197 |
+
del pipe
|
| 198 |
+
gc.collect()
|
| 199 |
+
torch.cuda.empty_cache()
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def translate_quest(quest_ko: str):
|
| 203 |
+
print("[quest translation] importing load_qwen...", flush=True)
|
| 204 |
+
from src.pipeline.persona2prompt import load_qwen, unload_qwen
|
| 205 |
+
print("[quest translation] importing torch...", flush=True)
|
| 206 |
+
import torch
|
| 207 |
+
|
| 208 |
+
print("Loading Qwen text model for quest translation...", flush=True)
|
| 209 |
+
model, tokenizer = load_qwen()
|
| 210 |
+
messages = [
|
| 211 |
+
{"role": "system", "content": QUEST_SYSTEM},
|
| 212 |
+
{"role": "user", "content": quest_ko},
|
| 213 |
+
]
|
| 214 |
+
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 215 |
+
inputs = tokenizer([text], return_tensors="pt").to("cuda")
|
| 216 |
+
with torch.no_grad():
|
| 217 |
+
outputs = model.generate(
|
| 218 |
+
**inputs,
|
| 219 |
+
max_new_tokens=80,
|
| 220 |
+
do_sample=False,
|
| 221 |
+
temperature=None,
|
| 222 |
+
top_p=None,
|
| 223 |
+
pad_token_id=tokenizer.eos_token_id,
|
| 224 |
+
)
|
| 225 |
+
generated = outputs[0][inputs.input_ids.shape[1]:]
|
| 226 |
+
quest_en = tokenizer.decode(generated, skip_special_tokens=True).strip()
|
| 227 |
+
unload_qwen(model, tokenizer)
|
| 228 |
+
return quest_en
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def generate_feed(quest_en: str, appearance, out_path: Path, seed: int):
|
| 232 |
+
from src.feed.feed_generator_2 import generate, load_pipeline, unload_pipeline
|
| 233 |
+
|
| 234 |
+
print("Loading feed generation pipeline...", flush=True)
|
| 235 |
+
pipe = load_pipeline()
|
| 236 |
+
image = generate(appearance, quest_en, pipe, seed=seed)
|
| 237 |
+
image.save(out_path)
|
| 238 |
+
print(f"Feed saved: {out_path}", flush=True)
|
| 239 |
+
unload_pipeline(pipe)
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def main():
|
| 243 |
+
args = parse_args()
|
| 244 |
+
out_dir = OUTPUT_ROOT / args.name
|
| 245 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 246 |
+
status_path = out_dir / "status.log"
|
| 247 |
+
|
| 248 |
+
def status(message: str):
|
| 249 |
+
print(message, flush=True)
|
| 250 |
+
with status_path.open("a", encoding="utf-8") as f:
|
| 251 |
+
f.write(message + "\n")
|
| 252 |
+
|
| 253 |
+
status("=" * 60)
|
| 254 |
+
status("Image-to-feed pipeline started")
|
| 255 |
+
status(f"Output dir: {out_dir}")
|
| 256 |
+
|
| 257 |
+
src_image = Path(args.character_image).resolve()
|
| 258 |
+
if not src_image.exists():
|
| 259 |
+
raise FileNotFoundError(f"--character-image not found: {src_image}")
|
| 260 |
+
|
| 261 |
+
nobg_path = out_dir / "input_nobg.png"
|
| 262 |
+
appearance_path = out_dir / "appearance.json"
|
| 263 |
+
character_path = out_dir / "character.png"
|
| 264 |
+
character_nobg_path = out_dir / "character_nobg.png"
|
| 265 |
+
quest_path = out_dir / "quest.json"
|
| 266 |
+
feed_path = out_dir / "feed.png"
|
| 267 |
+
results_path = out_dir / "results.json"
|
| 268 |
+
|
| 269 |
+
# STEP 1: background removal
|
| 270 |
+
if args.force or not nobg_path.exists():
|
| 271 |
+
status("STEP 1: removing background")
|
| 272 |
+
nobg_image = remove_background(src_image, nobg_path)
|
| 273 |
+
else:
|
| 274 |
+
from PIL import Image
|
| 275 |
+
nobg_image = Image.open(nobg_path).convert("RGB")
|
| 276 |
+
status(f"STEP 1: using cached nobg image: {nobg_path}")
|
| 277 |
+
|
| 278 |
+
# STEP 2: extract appearance with Qwen2.5-VL
|
| 279 |
+
if args.force or not appearance_path.exists():
|
| 280 |
+
status("STEP 2: extracting appearance JSON with Qwen2.5-VL")
|
| 281 |
+
appearance = extract_appearance(nobg_image, out_dir, use_4bit=not args.no_4bit_vlm)
|
| 282 |
+
else:
|
| 283 |
+
appearance = json.loads(appearance_path.read_text(encoding="utf-8"))
|
| 284 |
+
status(f"STEP 2: using cached appearance: {appearance_path}")
|
| 285 |
+
|
| 286 |
+
# STEP 3: generate pixel art character from full appearance description
|
| 287 |
+
if args.force or args.force_char or not character_path.exists():
|
| 288 |
+
status("STEP 3: generating pixel art character with SDXL + LoRA")
|
| 289 |
+
generate_character(appearance, character_path, args.seed, args.character_steps)
|
| 290 |
+
else:
|
| 291 |
+
status(f"STEP 3: using cached character: {character_path}")
|
| 292 |
+
|
| 293 |
+
if args.force or args.force_char or not character_nobg_path.exists():
|
| 294 |
+
status("STEP 3.5: removing character background")
|
| 295 |
+
remove_background(character_path, character_nobg_path)
|
| 296 |
+
else:
|
| 297 |
+
status(f"STEP 3.5: using cached character nobg: {character_nobg_path}")
|
| 298 |
+
|
| 299 |
+
# STEP 4: translate Korean quest to English scene
|
| 300 |
+
if args.quest_en:
|
| 301 |
+
quest_en = args.quest_en
|
| 302 |
+
status("STEP 4: using provided English quest scene")
|
| 303 |
+
elif args.force or args.force_quest or not quest_path.exists():
|
| 304 |
+
status("STEP 4: translating Korean quest to English scene")
|
| 305 |
+
quest_en = translate_quest(args.quest_ko)
|
| 306 |
+
else:
|
| 307 |
+
quest_en = json.loads(quest_path.read_text(encoding="utf-8"))["quest_en"]
|
| 308 |
+
status(f"STEP 4: using cached quest translation: {quest_path}")
|
| 309 |
+
|
| 310 |
+
quest_data = {
|
| 311 |
+
"quest_ko": args.quest_ko,
|
| 312 |
+
"quest_en": quest_en,
|
| 313 |
+
}
|
| 314 |
+
quest_path.write_text(json.dumps(quest_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 315 |
+
status(f"Quest EN: {quest_en}")
|
| 316 |
+
|
| 317 |
+
# STEP 5: generate feed image
|
| 318 |
+
if args.force or args.force_feed or not feed_path.exists():
|
| 319 |
+
status("STEP 5: generating feed image")
|
| 320 |
+
generate_feed(quest_en, appearance, feed_path, args.feed_seed)
|
| 321 |
+
else:
|
| 322 |
+
status(f"STEP 5: using cached feed: {feed_path}")
|
| 323 |
+
|
| 324 |
+
results = {
|
| 325 |
+
"name": args.name,
|
| 326 |
+
"character_image_input": str(src_image),
|
| 327 |
+
"input_nobg": str(nobg_path).replace("\\", "/"),
|
| 328 |
+
"character_image": str(character_path).replace("\\", "/"),
|
| 329 |
+
"character_nobg": str(character_nobg_path).replace("\\", "/"),
|
| 330 |
+
"appearance_json": str(appearance_path).replace("\\", "/"),
|
| 331 |
+
"appearance": appearance,
|
| 332 |
+
"quest": quest_data,
|
| 333 |
+
"feed_image": str(feed_path).replace("\\", "/"),
|
| 334 |
+
}
|
| 335 |
+
results_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 336 |
+
status(f"Results saved: {results_path}")
|
| 337 |
+
status("Image-to-feed pipeline finished")
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
if __name__ == "__main__":
|
| 341 |
+
try:
|
| 342 |
+
main()
|
| 343 |
+
except Exception:
|
| 344 |
+
fallback_dir = OUTPUT_ROOT / "sample_01"
|
| 345 |
+
fallback_dir.mkdir(parents=True, exist_ok=True)
|
| 346 |
+
error_path = fallback_dir / "error.log"
|
| 347 |
+
error_text = traceback.format_exc()
|
| 348 |
+
error_path.write_text(error_text, encoding="utf-8")
|
| 349 |
+
print(error_text, flush=True)
|
| 350 |
+
print(f"Error saved: {error_path}", flush=True)
|
| 351 |
+
raise
|
test_qwen25_vl_extract.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Qwen2.5-VL detailed appearance extraction test.
|
| 3 |
+
|
| 4 |
+
Inputs:
|
| 5 |
+
image/01.jpg
|
| 6 |
+
image/10.jpg
|
| 7 |
+
image/19.jpg
|
| 8 |
+
|
| 9 |
+
Outputs:
|
| 10 |
+
outputs/vlm_qwen25_extract/
|
| 11 |
+
01_nobg.png
|
| 12 |
+
10_nobg.png
|
| 13 |
+
19_nobg.png
|
| 14 |
+
01_raw.txt
|
| 15 |
+
10_raw.txt
|
| 16 |
+
19_raw.txt
|
| 17 |
+
extraction_results.json
|
| 18 |
+
extraction_report.md
|
| 19 |
+
"""
|
| 20 |
+
import argparse
|
| 21 |
+
import gc
|
| 22 |
+
import json
|
| 23 |
+
import os
|
| 24 |
+
import re
|
| 25 |
+
import sys
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
|
| 28 |
+
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
|
| 29 |
+
|
| 30 |
+
ROOT = Path(__file__).resolve().parent
|
| 31 |
+
os.chdir(ROOT)
|
| 32 |
+
sys.path.insert(0, str(ROOT))
|
| 33 |
+
|
| 34 |
+
MODEL_ID = "Qwen/Qwen2.5-VL-7B-Instruct"
|
| 35 |
+
OUTPUT_DIR = Path("outputs/vlm_qwen25_extract")
|
| 36 |
+
|
| 37 |
+
TEST_IMAGES = [
|
| 38 |
+
{"name": "01", "path": "image/01.jpg"},
|
| 39 |
+
{"name": "10", "path": "image/10.jpg"},
|
| 40 |
+
{"name": "19", "path": "image/19.jpg"},
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
EXTRACT_PROMPT = """
|
| 44 |
+
You are a meticulous visual character analyst for a plush-to-pixel-art pipeline.
|
| 45 |
+
|
| 46 |
+
Analyze only the visible plush toy / mascot character in the image.
|
| 47 |
+
Return ONLY valid JSON. Do not include markdown, comments, or extra text.
|
| 48 |
+
|
| 49 |
+
Important rules:
|
| 50 |
+
- Be specific and concrete. Avoid generic words when visual evidence is available.
|
| 51 |
+
- If a detail is not visible, write "not visible" instead of inventing it.
|
| 52 |
+
- Do not describe the background except whether it is plain/removed.
|
| 53 |
+
- Preserve the character identity: color, silhouette, face, ears, body proportions,
|
| 54 |
+
markings, accessories, and texture are very important.
|
| 55 |
+
- Use short descriptive phrases, not long paragraphs.
|
| 56 |
+
|
| 57 |
+
JSON schema:
|
| 58 |
+
{
|
| 59 |
+
"animal_type": "specific animal or mascot type, e.g. bear, bunny, cat, dog, unknown plush mascot",
|
| 60 |
+
"body_color": "main body color with nuance, e.g. soft cherry pink, warm cream white",
|
| 61 |
+
"secondary_colors": ["all visible secondary colors and where they appear"],
|
| 62 |
+
"color_pattern": "how colors are arranged on the body",
|
| 63 |
+
"body_shape": "overall body silhouette and proportions",
|
| 64 |
+
"head_shape": "head shape and size relative to body",
|
| 65 |
+
"ear_shape": "ear shape, placement, and size",
|
| 66 |
+
"eye_style": "eye size, shape, color, material impression, highlights",
|
| 67 |
+
"nose_shape": "nose shape, color, size, and placement",
|
| 68 |
+
"mouth_shape": "mouth shape/expression, or not visible",
|
| 69 |
+
"cheeks": "blush or cheek detail, or not visible",
|
| 70 |
+
"limbs": "arms/legs/paws shape, length, pose, and color",
|
| 71 |
+
"pose": "front/side/standing/sitting/lying and any visible gesture",
|
| 72 |
+
"texture": "plush surface texture and softness",
|
| 73 |
+
"accessories": ["bows, tags, clothes, charms, strings, or none"],
|
| 74 |
+
"distinctive_features": ["identity-preserving details, each as one phrase"],
|
| 75 |
+
"pixel_prompt_core": "one concise English phrase for image generation preserving the character",
|
| 76 |
+
"overall_feel": "2-5 words describing visual impression",
|
| 77 |
+
"uncertain_details": ["details that may be ambiguous"]
|
| 78 |
+
}
|
| 79 |
+
""".strip()
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def parse_args():
|
| 83 |
+
parser = argparse.ArgumentParser()
|
| 84 |
+
parser.add_argument("--model", default=MODEL_ID)
|
| 85 |
+
parser.add_argument("--output", default=str(OUTPUT_DIR))
|
| 86 |
+
parser.add_argument("--max-new-tokens", type=int, default=900)
|
| 87 |
+
parser.add_argument("--no-4bit", action="store_true", help="Disable 4bit loading fallback attempt.")
|
| 88 |
+
parser.add_argument("--keep-original-bg", action="store_true", help="Do not run rembg before VLM.")
|
| 89 |
+
return parser.parse_args()
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def remove_bg_white(image_path: str):
|
| 93 |
+
from PIL import Image
|
| 94 |
+
|
| 95 |
+
image = Image.open(image_path).convert("RGB")
|
| 96 |
+
try:
|
| 97 |
+
from rembg import remove
|
| 98 |
+
|
| 99 |
+
removed = remove(image.convert("RGBA"))
|
| 100 |
+
white_bg = Image.new("RGBA", removed.size, (255, 255, 255, 255))
|
| 101 |
+
white_bg.paste(removed, mask=removed.split()[3])
|
| 102 |
+
return white_bg.convert("RGB"), True
|
| 103 |
+
except Exception as exc:
|
| 104 |
+
print(f" rembg failed, using original image: {exc}", flush=True)
|
| 105 |
+
return image, False
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def load_model(model_id: str, use_4bit: bool):
|
| 109 |
+
import torch
|
| 110 |
+
from transformers import AutoProcessor
|
| 111 |
+
|
| 112 |
+
try:
|
| 113 |
+
from transformers import Qwen2_5_VLForConditionalGeneration as QwenVLModel
|
| 114 |
+
except ImportError as exc:
|
| 115 |
+
raise ImportError(
|
| 116 |
+
"Your transformers version does not expose Qwen2_5_VLForConditionalGeneration. "
|
| 117 |
+
"Run: pip install -U transformers accelerate qwen-vl-utils"
|
| 118 |
+
) from exc
|
| 119 |
+
|
| 120 |
+
print(f"Loading {model_id}", flush=True)
|
| 121 |
+
|
| 122 |
+
if use_4bit:
|
| 123 |
+
try:
|
| 124 |
+
from transformers import BitsAndBytesConfig
|
| 125 |
+
|
| 126 |
+
bnb = BitsAndBytesConfig(
|
| 127 |
+
load_in_4bit=True,
|
| 128 |
+
bnb_4bit_compute_dtype=torch.float16,
|
| 129 |
+
bnb_4bit_use_double_quant=True,
|
| 130 |
+
bnb_4bit_quant_type="nf4",
|
| 131 |
+
)
|
| 132 |
+
model = QwenVLModel.from_pretrained(
|
| 133 |
+
model_id,
|
| 134 |
+
quantization_config=bnb,
|
| 135 |
+
device_map="auto",
|
| 136 |
+
trust_remote_code=True,
|
| 137 |
+
).eval()
|
| 138 |
+
print(" Loaded in 4bit.", flush=True)
|
| 139 |
+
except Exception as exc:
|
| 140 |
+
print(f" 4bit load failed: {exc}", flush=True)
|
| 141 |
+
print(" Falling back to float16 + device_map=auto.", flush=True)
|
| 142 |
+
model = QwenVLModel.from_pretrained(
|
| 143 |
+
model_id,
|
| 144 |
+
torch_dtype=torch.float16,
|
| 145 |
+
device_map="auto",
|
| 146 |
+
trust_remote_code=True,
|
| 147 |
+
).eval()
|
| 148 |
+
else:
|
| 149 |
+
model = QwenVLModel.from_pretrained(
|
| 150 |
+
model_id,
|
| 151 |
+
torch_dtype=torch.float16,
|
| 152 |
+
device_map="auto",
|
| 153 |
+
trust_remote_code=True,
|
| 154 |
+
).eval()
|
| 155 |
+
|
| 156 |
+
processor = AutoProcessor.from_pretrained(
|
| 157 |
+
model_id,
|
| 158 |
+
min_pixels=256 * 28 * 28,
|
| 159 |
+
max_pixels=768 * 28 * 28,
|
| 160 |
+
trust_remote_code=True,
|
| 161 |
+
)
|
| 162 |
+
return model, processor
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def extract_json_from_text(raw: str):
|
| 166 |
+
clean = raw.strip()
|
| 167 |
+
fence = re.search(r"```(?:json)?\s*(.*?)\s*```", clean, flags=re.DOTALL | re.IGNORECASE)
|
| 168 |
+
if fence:
|
| 169 |
+
clean = fence.group(1).strip()
|
| 170 |
+
|
| 171 |
+
start = clean.find("{")
|
| 172 |
+
end = clean.rfind("}")
|
| 173 |
+
if start != -1 and end != -1 and end > start:
|
| 174 |
+
clean = clean[start : end + 1]
|
| 175 |
+
|
| 176 |
+
return json.loads(clean)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def ensure_list(value):
|
| 180 |
+
if value is None:
|
| 181 |
+
return []
|
| 182 |
+
if isinstance(value, list):
|
| 183 |
+
return value
|
| 184 |
+
if isinstance(value, str):
|
| 185 |
+
if value.strip().lower() in {"", "none", "not visible"}:
|
| 186 |
+
return []
|
| 187 |
+
return [value]
|
| 188 |
+
return [str(value)]
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def normalize_info(info: dict):
|
| 192 |
+
list_fields = [
|
| 193 |
+
"secondary_colors",
|
| 194 |
+
"accessories",
|
| 195 |
+
"distinctive_features",
|
| 196 |
+
"uncertain_details",
|
| 197 |
+
]
|
| 198 |
+
for field in list_fields:
|
| 199 |
+
info[field] = ensure_list(info.get(field))
|
| 200 |
+
return info
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def run_extraction(image, model, processor, max_new_tokens: int):
|
| 204 |
+
import torch
|
| 205 |
+
from qwen_vl_utils import process_vision_info
|
| 206 |
+
|
| 207 |
+
messages = [
|
| 208 |
+
{
|
| 209 |
+
"role": "user",
|
| 210 |
+
"content": [
|
| 211 |
+
{"type": "image", "image": image},
|
| 212 |
+
{"type": "text", "text": EXTRACT_PROMPT},
|
| 213 |
+
],
|
| 214 |
+
}
|
| 215 |
+
]
|
| 216 |
+
|
| 217 |
+
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 218 |
+
image_inputs, video_inputs = process_vision_info(messages)
|
| 219 |
+
inputs = processor(
|
| 220 |
+
text=[text],
|
| 221 |
+
images=image_inputs,
|
| 222 |
+
videos=video_inputs,
|
| 223 |
+
padding=True,
|
| 224 |
+
return_tensors="pt",
|
| 225 |
+
)
|
| 226 |
+
|
| 227 |
+
if torch.cuda.is_available():
|
| 228 |
+
inputs = inputs.to("cuda")
|
| 229 |
+
|
| 230 |
+
with torch.no_grad():
|
| 231 |
+
generated_ids = model.generate(
|
| 232 |
+
**inputs,
|
| 233 |
+
max_new_tokens=max_new_tokens,
|
| 234 |
+
do_sample=False,
|
| 235 |
+
)
|
| 236 |
+
|
| 237 |
+
trimmed = generated_ids[0][inputs.input_ids.shape[1] :]
|
| 238 |
+
raw = processor.decode(trimmed, skip_special_tokens=True).strip()
|
| 239 |
+
|
| 240 |
+
del inputs, generated_ids, trimmed
|
| 241 |
+
gc.collect()
|
| 242 |
+
if torch.cuda.is_available():
|
| 243 |
+
torch.cuda.empty_cache()
|
| 244 |
+
|
| 245 |
+
return raw
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def write_report(results, output_dir: Path, model_id: str):
|
| 249 |
+
report_path = output_dir / "extraction_report.md"
|
| 250 |
+
with report_path.open("w", encoding="utf-8") as f:
|
| 251 |
+
f.write("# Qwen2.5-VL Appearance Extraction Report\n\n")
|
| 252 |
+
f.write(f"Model: `{model_id}`\n\n")
|
| 253 |
+
|
| 254 |
+
for result in results:
|
| 255 |
+
info = result["extracted"]
|
| 256 |
+
f.write(f"## Image {result['name']}\n\n")
|
| 257 |
+
f.write(f"- Source: `{result['source']}`\n")
|
| 258 |
+
f.write(f"- Processed: `{result['processed_image']}`\n")
|
| 259 |
+
f.write(f"- JSON parse: `{result['parse_ok']}`\n\n")
|
| 260 |
+
|
| 261 |
+
f.write("| Field | Value |\n")
|
| 262 |
+
f.write("|---|---|\n")
|
| 263 |
+
for key, value in info.items():
|
| 264 |
+
if isinstance(value, list):
|
| 265 |
+
value = ", ".join(str(v) for v in value) if value else "-"
|
| 266 |
+
f.write(f"| `{key}` | {value} |\n")
|
| 267 |
+
f.write("\n")
|
| 268 |
+
|
| 269 |
+
f.write("### Pixel Prompt Core\n\n")
|
| 270 |
+
f.write("```text\n")
|
| 271 |
+
f.write(str(info.get("pixel_prompt_core", "")))
|
| 272 |
+
f.write("\n```\n\n")
|
| 273 |
+
|
| 274 |
+
if result.get("raw_output"):
|
| 275 |
+
f.write("### Raw Output\n\n")
|
| 276 |
+
f.write("```json\n")
|
| 277 |
+
f.write(result["raw_output"])
|
| 278 |
+
f.write("\n```\n\n")
|
| 279 |
+
|
| 280 |
+
print(f"Report saved: {report_path}", flush=True)
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def main():
|
| 284 |
+
args = parse_args()
|
| 285 |
+
output_dir = Path(args.output)
|
| 286 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 287 |
+
|
| 288 |
+
model, processor = load_model(args.model, use_4bit=not args.no_4bit)
|
| 289 |
+
|
| 290 |
+
results = []
|
| 291 |
+
for case in TEST_IMAGES:
|
| 292 |
+
name = case["name"]
|
| 293 |
+
print("\n" + "=" * 60, flush=True)
|
| 294 |
+
print(f"[{name}] {case['path']}", flush=True)
|
| 295 |
+
|
| 296 |
+
if args.keep_original_bg:
|
| 297 |
+
from PIL import Image
|
| 298 |
+
|
| 299 |
+
image = Image.open(case["path"]).convert("RGB")
|
| 300 |
+
rembg_ok = False
|
| 301 |
+
else:
|
| 302 |
+
image, rembg_ok = remove_bg_white(case["path"])
|
| 303 |
+
|
| 304 |
+
processed_path = output_dir / f"{name}_nobg.png"
|
| 305 |
+
image.save(processed_path)
|
| 306 |
+
print(f" Processed image saved: {processed_path}", flush=True)
|
| 307 |
+
|
| 308 |
+
raw = run_extraction(image, model, processor, args.max_new_tokens)
|
| 309 |
+
raw_path = output_dir / f"{name}_raw.txt"
|
| 310 |
+
raw_path.write_text(raw, encoding="utf-8")
|
| 311 |
+
print(f" Raw output saved: {raw_path}", flush=True)
|
| 312 |
+
|
| 313 |
+
try:
|
| 314 |
+
info = normalize_info(extract_json_from_text(raw))
|
| 315 |
+
parse_ok = True
|
| 316 |
+
except Exception as exc:
|
| 317 |
+
print(f" JSON parse failed: {exc}", flush=True)
|
| 318 |
+
info = {
|
| 319 |
+
"animal_type": "parse_failed",
|
| 320 |
+
"body_color": "parse_failed",
|
| 321 |
+
"secondary_colors": [],
|
| 322 |
+
"color_pattern": "parse_failed",
|
| 323 |
+
"body_shape": "parse_failed",
|
| 324 |
+
"head_shape": "parse_failed",
|
| 325 |
+
"ear_shape": "parse_failed",
|
| 326 |
+
"eye_style": "parse_failed",
|
| 327 |
+
"nose_shape": "parse_failed",
|
| 328 |
+
"mouth_shape": "parse_failed",
|
| 329 |
+
"cheeks": "parse_failed",
|
| 330 |
+
"limbs": "parse_failed",
|
| 331 |
+
"pose": "parse_failed",
|
| 332 |
+
"texture": "parse_failed",
|
| 333 |
+
"accessories": [],
|
| 334 |
+
"distinctive_features": [],
|
| 335 |
+
"pixel_prompt_core": "",
|
| 336 |
+
"overall_feel": "parse_failed",
|
| 337 |
+
"uncertain_details": [str(exc)],
|
| 338 |
+
}
|
| 339 |
+
parse_ok = False
|
| 340 |
+
|
| 341 |
+
results.append(
|
| 342 |
+
{
|
| 343 |
+
"name": name,
|
| 344 |
+
"source": case["path"],
|
| 345 |
+
"processed_image": str(processed_path).replace("\\", "/"),
|
| 346 |
+
"rembg_ok": rembg_ok,
|
| 347 |
+
"parse_ok": parse_ok,
|
| 348 |
+
"extracted": info,
|
| 349 |
+
"raw_output": raw,
|
| 350 |
+
}
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
print(f" animal_type: {info.get('animal_type')}", flush=True)
|
| 354 |
+
print(f" body_color : {info.get('body_color')}", flush=True)
|
| 355 |
+
print(f" core : {info.get('pixel_prompt_core')}", flush=True)
|
| 356 |
+
|
| 357 |
+
json_path = output_dir / "extraction_results.json"
|
| 358 |
+
json_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 359 |
+
print("\n" + "=" * 60, flush=True)
|
| 360 |
+
print(f"JSON saved: {json_path}", flush=True)
|
| 361 |
+
write_report(results, output_dir, args.model)
|
| 362 |
+
|
| 363 |
+
del model, processor
|
| 364 |
+
gc.collect()
|
| 365 |
+
try:
|
| 366 |
+
import torch
|
| 367 |
+
|
| 368 |
+
if torch.cuda.is_available():
|
| 369 |
+
torch.cuda.empty_cache()
|
| 370 |
+
except Exception:
|
| 371 |
+
pass
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
if __name__ == "__main__":
|
| 375 |
+
main()
|
test_text2feed_pipeline.py
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Text-to-feed end-to-end test pipeline.
|
| 3 |
+
|
| 4 |
+
Flow:
|
| 5 |
+
Korean character text
|
| 6 |
+
-> English visual character prompt
|
| 7 |
+
-> SDXL + lora_v3_32bit character image
|
| 8 |
+
-> Qwen2.5-VL appearance JSON extraction
|
| 9 |
+
-> Korean quest to English scene text
|
| 10 |
+
-> SDXL + lora_v3_32bit + lora_bg_v1 feed image
|
| 11 |
+
|
| 12 |
+
Outputs:
|
| 13 |
+
outputs/text2feed/<name>/
|
| 14 |
+
character.png
|
| 15 |
+
appearance_raw.txt
|
| 16 |
+
appearance.json
|
| 17 |
+
quest.json
|
| 18 |
+
feed.png
|
| 19 |
+
results.json
|
| 20 |
+
"""
|
| 21 |
+
import argparse
|
| 22 |
+
import gc
|
| 23 |
+
import json
|
| 24 |
+
import os
|
| 25 |
+
import sys
|
| 26 |
+
import traceback
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
|
| 29 |
+
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
|
| 30 |
+
|
| 31 |
+
ROOT = Path(__file__).resolve().parent
|
| 32 |
+
os.chdir(ROOT)
|
| 33 |
+
sys.path.insert(0, str(ROOT))
|
| 34 |
+
|
| 35 |
+
OUTPUT_ROOT = Path("outputs/text2feed")
|
| 36 |
+
CHAR_LORA_DIR = "models/lora_v3_32bit"
|
| 37 |
+
|
| 38 |
+
DEFAULT_CHARACTER_KO = (
|
| 39 |
+
"이 친구는 부드러운 체리핑크색 곰 인형이에요. "
|
| 40 |
+
"크림색 배를 가지고 있고, 작은 둥근 귀와 분홍색 코, "
|
| 41 |
+
"짧고 통통한 팔다리와 포근한 표정을 가지고 있어요."
|
| 42 |
+
)
|
| 43 |
+
DEFAULT_CHARACTER_EN = (
|
| 44 |
+
"soft cherry pink bear plush mascot with a cream white belly, "
|
| 45 |
+
"small rounded ears, tiny pink oval nose, simple smiling mouth, "
|
| 46 |
+
"short rounded limbs, smooth plush texture"
|
| 47 |
+
)
|
| 48 |
+
DEFAULT_QUEST_KO = "공원에서 30분 달리기를 완료했어요!"
|
| 49 |
+
|
| 50 |
+
CHARACTER_NEGATIVE = (
|
| 51 |
+
"realistic, photograph, 3d render, human, person, anime, scary, "
|
| 52 |
+
"complex background, multiple characters, text, watermark, logo, "
|
| 53 |
+
"extra limbs, long limbs, harsh black outline, low quality, blurry"
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
QUEST_SYSTEM = """You are an action-pose scene prompt writer for Mongle Village, a cozy sky island pixel art village.
|
| 57 |
+
|
| 58 |
+
Convert a Korean quest completion message into a short English scene description for image generation.
|
| 59 |
+
|
| 60 |
+
Rules:
|
| 61 |
+
- The FIRST words must describe the character's visible action pose.
|
| 62 |
+
- Keep the activity from the quest very explicit.
|
| 63 |
+
- Include body movement cues such as arms swinging, legs moving, holding a book, stirring a pot, walking steps.
|
| 64 |
+
- Describe one simple matching environment after the action.
|
| 65 |
+
- Set it in a cozy pastel sky island village world.
|
| 66 |
+
- Mention only one character.
|
| 67 |
+
- Do not write only a landscape/background description.
|
| 68 |
+
- Do not make the character standing still unless the quest is resting.
|
| 69 |
+
- 16-28 words max.
|
| 70 |
+
- Output ONLY the English scene description.
|
| 71 |
+
|
| 72 |
+
Examples:
|
| 73 |
+
Input: 공원에서 30분 달리기를 완료했어요!
|
| 74 |
+
Output: running with arms swinging and legs in motion along a fluffy cloud meadow path
|
| 75 |
+
|
| 76 |
+
Input: 오늘 책 한 권을 다 읽었어요
|
| 77 |
+
Output: sitting and holding an open storybook under a blossoming cloud tree beside a cozy cottage
|
| 78 |
+
|
| 79 |
+
Input: 직접 요리해서 건강한 밥을 먹었어요
|
| 80 |
+
Output: cooking with both paws stirring a pot in a cozy cottage kitchen with fresh vegetables
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
CHARACTER_SYSTEM = """You are a visual prompt writer for a plush-to-pixel-art character generation pipeline.
|
| 84 |
+
|
| 85 |
+
Convert a Korean character description into a concise English visual prompt.
|
| 86 |
+
|
| 87 |
+
Rules:
|
| 88 |
+
- Focus only on visible appearance: animal type, body color, face, ears, body shape, limbs, texture, accessories.
|
| 89 |
+
- Convert personality words into visible expression only.
|
| 90 |
+
- If animal type is not specified, choose a soft plush animal that fits the description.
|
| 91 |
+
- If color is not specified, choose one pastel color.
|
| 92 |
+
- Do not include background, scene, action, story, name, or relationship.
|
| 93 |
+
- 25-45 words.
|
| 94 |
+
- Output ONLY the English visual prompt.
|
| 95 |
+
|
| 96 |
+
Example:
|
| 97 |
+
Input: 이 친구는 용감하고 사랑스러운 하얀 토끼 인형이에요. 볼이 발그레하고 포근해요.
|
| 98 |
+
Output: white bunny plush mascot, rosy cheeks, confident bright eyes, warm gentle smile, soft round chubby body, short rounded limbs, fluffy plush texture
|
| 99 |
+
"""
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def parse_args():
|
| 103 |
+
parser = argparse.ArgumentParser()
|
| 104 |
+
parser.add_argument("--name", default="sample_01")
|
| 105 |
+
parser.add_argument("--character-ko", default=DEFAULT_CHARACTER_KO)
|
| 106 |
+
parser.add_argument(
|
| 107 |
+
"--character-text",
|
| 108 |
+
default="",
|
| 109 |
+
help="English visual character prompt. If omitted, --character-ko is translated first.",
|
| 110 |
+
)
|
| 111 |
+
parser.add_argument("--quest-ko", default=DEFAULT_QUEST_KO)
|
| 112 |
+
parser.add_argument("--quest-en", default="", help="If provided, skips Qwen text translation.")
|
| 113 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 114 |
+
parser.add_argument("--force", action="store_true")
|
| 115 |
+
parser.add_argument("--force-quest", action="store_true")
|
| 116 |
+
parser.add_argument("--force-feed", action="store_true")
|
| 117 |
+
parser.add_argument("--no-4bit-vlm", action="store_true")
|
| 118 |
+
parser.add_argument("--character-steps", type=int, default=30)
|
| 119 |
+
parser.add_argument("--feed-seed", type=int, default=123)
|
| 120 |
+
return parser.parse_args()
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def character_prompt(character_text: str) -> str:
|
| 124 |
+
return (
|
| 125 |
+
"monglestyle, "
|
| 126 |
+
f"{character_text}, "
|
| 127 |
+
"single stuffed animal toy mascot character, full body, centered, "
|
| 128 |
+
"front view, cute chibi proportions, 32-bit pixel art sprite, "
|
| 129 |
+
"soft pixel shading, clean silhouette, soft brown outline, "
|
| 130 |
+
"pure white background"
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def translate_character(character_ko: str):
|
| 135 |
+
print("[character translation] importing load_qwen...", flush=True)
|
| 136 |
+
from src.pipeline.persona2prompt import load_qwen, unload_qwen
|
| 137 |
+
print("[character translation] importing torch...", flush=True)
|
| 138 |
+
import torch
|
| 139 |
+
|
| 140 |
+
print("Loading Qwen text model for character translation...", flush=True)
|
| 141 |
+
model, tokenizer = load_qwen()
|
| 142 |
+
messages = [
|
| 143 |
+
{"role": "system", "content": CHARACTER_SYSTEM},
|
| 144 |
+
{"role": "user", "content": character_ko},
|
| 145 |
+
]
|
| 146 |
+
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 147 |
+
inputs = tokenizer([text], return_tensors="pt").to("cuda")
|
| 148 |
+
with torch.no_grad():
|
| 149 |
+
outputs = model.generate(
|
| 150 |
+
**inputs,
|
| 151 |
+
max_new_tokens=120,
|
| 152 |
+
do_sample=False,
|
| 153 |
+
temperature=None,
|
| 154 |
+
top_p=None,
|
| 155 |
+
pad_token_id=tokenizer.eos_token_id,
|
| 156 |
+
)
|
| 157 |
+
generated = outputs[0][inputs.input_ids.shape[1] :]
|
| 158 |
+
character_en = tokenizer.decode(generated, skip_special_tokens=True).strip()
|
| 159 |
+
unload_qwen(model, tokenizer)
|
| 160 |
+
return character_en
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def generate_character(character_text: str, out_path: Path, seed: int, steps: int):
|
| 164 |
+
import torch
|
| 165 |
+
from diffusers import DPMSolverMultistepScheduler, StableDiffusionXLPipeline
|
| 166 |
+
|
| 167 |
+
print("Loading SDXL character pipeline...", flush=True)
|
| 168 |
+
pipe = StableDiffusionXLPipeline.from_pretrained(
|
| 169 |
+
"stabilityai/stable-diffusion-xl-base-1.0",
|
| 170 |
+
torch_dtype=torch.float16,
|
| 171 |
+
use_safetensors=True,
|
| 172 |
+
)
|
| 173 |
+
pipe.load_lora_weights(CHAR_LORA_DIR)
|
| 174 |
+
pipe.fuse_lora(lora_scale=0.9)
|
| 175 |
+
pipe.unload_lora_weights()
|
| 176 |
+
pipe.scheduler = DPMSolverMultistepScheduler.from_config(
|
| 177 |
+
pipe.scheduler.config, use_karras_sigmas=True
|
| 178 |
+
)
|
| 179 |
+
pipe.to("cuda")
|
| 180 |
+
pipe.enable_attention_slicing()
|
| 181 |
+
|
| 182 |
+
prompt = character_prompt(character_text)
|
| 183 |
+
print(f"Character prompt: {prompt}", flush=True)
|
| 184 |
+
image = pipe(
|
| 185 |
+
prompt=prompt,
|
| 186 |
+
negative_prompt=CHARACTER_NEGATIVE,
|
| 187 |
+
num_inference_steps=steps,
|
| 188 |
+
guidance_scale=7.5,
|
| 189 |
+
height=1024,
|
| 190 |
+
width=1024,
|
| 191 |
+
generator=torch.Generator("cuda").manual_seed(seed),
|
| 192 |
+
).images[0]
|
| 193 |
+
image.save(out_path)
|
| 194 |
+
print(f"Character saved: {out_path}", flush=True)
|
| 195 |
+
|
| 196 |
+
del pipe
|
| 197 |
+
gc.collect()
|
| 198 |
+
torch.cuda.empty_cache()
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def extract_appearance(character_path: Path, out_dir: Path, use_4bit: bool):
|
| 202 |
+
from PIL import Image
|
| 203 |
+
from test_qwen25_vl_extract import (
|
| 204 |
+
extract_json_from_text,
|
| 205 |
+
load_model,
|
| 206 |
+
normalize_info,
|
| 207 |
+
run_extraction,
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
print("Loading Qwen2.5-VL appearance extractor...", flush=True)
|
| 211 |
+
model, processor = load_model("Qwen/Qwen2.5-VL-7B-Instruct", use_4bit=use_4bit)
|
| 212 |
+
image = Image.open(character_path).convert("RGB")
|
| 213 |
+
raw = run_extraction(image, model, processor, max_new_tokens=900)
|
| 214 |
+
|
| 215 |
+
raw_path = out_dir / "appearance_raw.txt"
|
| 216 |
+
raw_path.write_text(raw, encoding="utf-8")
|
| 217 |
+
|
| 218 |
+
info = normalize_info(extract_json_from_text(raw))
|
| 219 |
+
appearance_path = out_dir / "appearance.json"
|
| 220 |
+
appearance_path.write_text(json.dumps(info, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 221 |
+
print(f"Appearance saved: {appearance_path}", flush=True)
|
| 222 |
+
|
| 223 |
+
del model, processor
|
| 224 |
+
gc.collect()
|
| 225 |
+
try:
|
| 226 |
+
import torch
|
| 227 |
+
|
| 228 |
+
torch.cuda.empty_cache()
|
| 229 |
+
except Exception:
|
| 230 |
+
pass
|
| 231 |
+
return info
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def translate_quest(quest_ko: str):
|
| 235 |
+
print("[quest translation] importing load_qwen...", flush=True)
|
| 236 |
+
from src.pipeline.persona2prompt import load_qwen, unload_qwen
|
| 237 |
+
print("[quest translation] importing torch...", flush=True)
|
| 238 |
+
import torch
|
| 239 |
+
|
| 240 |
+
print("Loading Qwen text model for quest translation...", flush=True)
|
| 241 |
+
model, tokenizer = load_qwen()
|
| 242 |
+
messages = [
|
| 243 |
+
{"role": "system", "content": QUEST_SYSTEM},
|
| 244 |
+
{"role": "user", "content": quest_ko},
|
| 245 |
+
]
|
| 246 |
+
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 247 |
+
inputs = tokenizer([text], return_tensors="pt").to("cuda")
|
| 248 |
+
with torch.no_grad():
|
| 249 |
+
outputs = model.generate(
|
| 250 |
+
**inputs,
|
| 251 |
+
max_new_tokens=80,
|
| 252 |
+
do_sample=False,
|
| 253 |
+
temperature=None,
|
| 254 |
+
top_p=None,
|
| 255 |
+
pad_token_id=tokenizer.eos_token_id,
|
| 256 |
+
)
|
| 257 |
+
generated = outputs[0][inputs.input_ids.shape[1] :]
|
| 258 |
+
quest_en = tokenizer.decode(generated, skip_special_tokens=True).strip()
|
| 259 |
+
unload_qwen(model, tokenizer)
|
| 260 |
+
return quest_en
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def remove_character_bg(character_path: Path, out_path: Path):
|
| 264 |
+
from PIL import Image
|
| 265 |
+
from rembg import remove
|
| 266 |
+
|
| 267 |
+
print("Removing character background...", flush=True)
|
| 268 |
+
image = Image.open(character_path).convert("RGBA")
|
| 269 |
+
result = remove(image)
|
| 270 |
+
result.save(out_path)
|
| 271 |
+
print(f"Character nobg saved: {out_path}", flush=True)
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def generate_feed(quest_en: str, appearance, out_path: Path, seed: int):
|
| 275 |
+
from src.feed.feed_generator_1 import generate, load_pipeline, unload_pipeline
|
| 276 |
+
|
| 277 |
+
print("Loading feed generation pipeline...", flush=True)
|
| 278 |
+
pipe = load_pipeline()
|
| 279 |
+
image = generate(appearance, quest_en, pipe, seed=seed)
|
| 280 |
+
image.save(out_path)
|
| 281 |
+
print(f"Feed saved: {out_path}", flush=True)
|
| 282 |
+
unload_pipeline(pipe)
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def main():
|
| 286 |
+
args = parse_args()
|
| 287 |
+
out_dir = OUTPUT_ROOT / args.name
|
| 288 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 289 |
+
status_path = out_dir / "status.log"
|
| 290 |
+
|
| 291 |
+
def status(message: str):
|
| 292 |
+
print(message, flush=True)
|
| 293 |
+
with status_path.open("a", encoding="utf-8") as f:
|
| 294 |
+
f.write(message + "\n")
|
| 295 |
+
|
| 296 |
+
status("=" * 60)
|
| 297 |
+
status("Text-to-feed pipeline started")
|
| 298 |
+
status(f"Output dir: {out_dir}")
|
| 299 |
+
|
| 300 |
+
character_path = out_dir / "character.png"
|
| 301 |
+
character_nobg_path = out_dir / "character_nobg.png"
|
| 302 |
+
character_prompt_path = out_dir / "character_prompt.json"
|
| 303 |
+
appearance_path = out_dir / "appearance.json"
|
| 304 |
+
quest_path = out_dir / "quest.json"
|
| 305 |
+
feed_path = out_dir / "feed.png"
|
| 306 |
+
results_path = out_dir / "results.json"
|
| 307 |
+
|
| 308 |
+
if args.character_text:
|
| 309 |
+
character_text = args.character_text
|
| 310 |
+
status("STEP 0: using provided English character prompt")
|
| 311 |
+
elif args.force or not character_prompt_path.exists():
|
| 312 |
+
status("STEP 0: translating Korean character description to English prompt")
|
| 313 |
+
character_text = translate_character(args.character_ko)
|
| 314 |
+
else:
|
| 315 |
+
character_text = json.loads(character_prompt_path.read_text(encoding="utf-8"))["character_text"]
|
| 316 |
+
status(f"STEP 0: using cached character prompt: {character_prompt_path}")
|
| 317 |
+
|
| 318 |
+
character_prompt_data = {
|
| 319 |
+
"character_ko": args.character_ko,
|
| 320 |
+
"character_text": character_text,
|
| 321 |
+
}
|
| 322 |
+
character_prompt_path.write_text(
|
| 323 |
+
json.dumps(character_prompt_data, ensure_ascii=False, indent=2),
|
| 324 |
+
encoding="utf-8",
|
| 325 |
+
)
|
| 326 |
+
status(f"Character EN: {character_text}")
|
| 327 |
+
|
| 328 |
+
if args.force or not character_path.exists():
|
| 329 |
+
status("STEP 1: generating character image")
|
| 330 |
+
generate_character(character_text, character_path, args.seed, args.character_steps)
|
| 331 |
+
else:
|
| 332 |
+
status(f"STEP 1: using cached character: {character_path}")
|
| 333 |
+
|
| 334 |
+
if args.force or not character_nobg_path.exists():
|
| 335 |
+
status("STEP 1.5: removing character background")
|
| 336 |
+
remove_character_bg(character_path, character_nobg_path)
|
| 337 |
+
else:
|
| 338 |
+
status(f"STEP 1.5: using cached character nobg: {character_nobg_path}")
|
| 339 |
+
|
| 340 |
+
if args.force or not appearance_path.exists():
|
| 341 |
+
status("STEP 2: extracting appearance JSON with Qwen2.5-VL")
|
| 342 |
+
appearance = extract_appearance(character_path, out_dir, use_4bit=not args.no_4bit_vlm)
|
| 343 |
+
else:
|
| 344 |
+
appearance = json.loads(appearance_path.read_text(encoding="utf-8"))
|
| 345 |
+
status(f"STEP 2: using cached appearance: {appearance_path}")
|
| 346 |
+
|
| 347 |
+
if args.quest_en:
|
| 348 |
+
quest_en = args.quest_en
|
| 349 |
+
status("STEP 3: using provided English quest scene")
|
| 350 |
+
elif args.force or args.force_quest or not quest_path.exists():
|
| 351 |
+
status("STEP 3: translating Korean quest to English scene")
|
| 352 |
+
quest_en = translate_quest(args.quest_ko)
|
| 353 |
+
else:
|
| 354 |
+
quest_en = json.loads(quest_path.read_text(encoding="utf-8"))["quest_en"]
|
| 355 |
+
status(f"STEP 3: using cached quest translation: {quest_path}")
|
| 356 |
+
|
| 357 |
+
quest_data = {
|
| 358 |
+
"quest_ko": args.quest_ko,
|
| 359 |
+
"quest_en": quest_en,
|
| 360 |
+
}
|
| 361 |
+
quest_path.write_text(json.dumps(quest_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 362 |
+
status(f"Quest EN: {quest_en}")
|
| 363 |
+
|
| 364 |
+
if args.force or args.force_feed or not feed_path.exists():
|
| 365 |
+
status("STEP 4: generating feed image")
|
| 366 |
+
generate_feed(quest_en, appearance, feed_path, args.feed_seed)
|
| 367 |
+
else:
|
| 368 |
+
status(f"STEP 4: using cached feed: {feed_path}")
|
| 369 |
+
|
| 370 |
+
results = {
|
| 371 |
+
"name": args.name,
|
| 372 |
+
"character_ko": args.character_ko,
|
| 373 |
+
"character_text": character_text,
|
| 374 |
+
"character_prompt_json": str(character_prompt_path).replace("\\", "/"),
|
| 375 |
+
"character_image": str(character_path).replace("\\", "/"),
|
| 376 |
+
"character_nobg": str(character_nobg_path).replace("\\", "/"),
|
| 377 |
+
"appearance_json": str(appearance_path).replace("\\", "/"),
|
| 378 |
+
"appearance": appearance,
|
| 379 |
+
"quest": quest_data,
|
| 380 |
+
"feed_image": str(feed_path).replace("\\", "/"),
|
| 381 |
+
}
|
| 382 |
+
results_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 383 |
+
status(f"Results saved: {results_path}")
|
| 384 |
+
status("Text-to-feed pipeline finished")
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
if __name__ == "__main__":
|
| 388 |
+
try:
|
| 389 |
+
main()
|
| 390 |
+
except Exception:
|
| 391 |
+
fallback_dir = OUTPUT_ROOT / "sample_01"
|
| 392 |
+
fallback_dir.mkdir(parents=True, exist_ok=True)
|
| 393 |
+
error_path = fallback_dir / "error.log"
|
| 394 |
+
error_text = traceback.format_exc()
|
| 395 |
+
error_path.write_text(error_text, encoding="utf-8")
|
| 396 |
+
print(error_text, flush=True)
|
| 397 |
+
print(f"Error saved: {error_path}", flush=True)
|
| 398 |
+
raise
|