Buckets:
| import json | |
| import re | |
| from math import gcd | |
| from collections import OrderedDict | |
| from typing import Optional | |
| from PIL import Image | |
| from .Qwen3VLCaptioner import Qwen3VLCaptioner | |
| from .prompts.ideogram4_caption_prompt import ideogram4_caption_prompt | |
| from toolkit.ideogram_caption import normalize_caption_dict, swap_bbox_xy_in_text | |
| import transformers | |
| import logging | |
| import warnings | |
| # transformers.logging.set_verbosity_error() | |
| warnings.filterwarnings("ignore") | |
| logging.disable(logging.WARNING) | |
| # The deconstruction JSON is long. 128 tokens (base default) truncates it badly, | |
| # so enforce a sane floor for this captioner unless the user asked for more. | |
| MIN_NEW_TOKENS = 3072 | |
| # Largest denominator allowed when snapping a real image's aspect ratio to a | |
| # clean W:H. Keeps captions in the same small-denominator ratio distribution the | |
| # generator was trained on, instead of ugly fractions like 1023:768. | |
| MAX_AR_DENOMINATOR = 16 | |
| class Ideogram4Captioner(Qwen3VLCaptioner): | |
| def __init__(self, process_id: int, job, config: OrderedDict, **kwargs): | |
| super(Ideogram4Captioner, self).__init__(process_id, job, config, **kwargs) | |
| if self.caption_config.max_new_tokens < MIN_NEW_TOKENS: | |
| print( | |
| f"[Ideogram4Captioner] Raising max_new_tokens " | |
| f"{self.caption_config.max_new_tokens} -> {MIN_NEW_TOKENS} " | |
| f"(the deconstruction JSON is long)." | |
| ) | |
| self.caption_config.max_new_tokens = MIN_NEW_TOKENS | |
| def compute_aspect_ratio(self, width: int, height: int) -> str: | |
| """Return a clean 'W:H' string for the image, snapped to a small | |
| denominator so it matches the generator's ratio distribution.""" | |
| if width <= 0 or height <= 0: | |
| return "1:1" | |
| g = gcd(width, height) | |
| rw, rh = width // g, height // g | |
| # Already clean enough. | |
| if rw <= MAX_AR_DENOMINATOR and rh <= MAX_AR_DENOMINATOR: | |
| return f"{rw}:{rh}" | |
| # Otherwise find the closest p:q (q <= MAX_AR_DENOMINATOR) to the true ratio. | |
| target = width / height | |
| best = None | |
| for q in range(1, MAX_AR_DENOMINATOR + 1): | |
| p = max(1, round(target * q)) | |
| err = abs(p / q - target) | |
| if best is None or err < best[0]: | |
| best = (err, p, q) | |
| return f"{best[1]}:{best[2]}" | |
| def build_prompt(self, aspect_ratio: str) -> str: | |
| # caption_prompt is the user-editable ADDITIONAL INSTRUCTIONS block, | |
| # injected into the fixed system prompt (not the whole prompt). | |
| user_instructions = (self.caption_config.caption_prompt or "").strip() | |
| if not user_instructions: | |
| user_instructions = "None." | |
| prompt = ideogram4_caption_prompt.replace("{{aspect_ratio}}", aspect_ratio) | |
| prompt = prompt.replace("{{user_instructions}}", user_instructions) | |
| return prompt | |
| def _extract_json(self, raw: str) -> Optional[dict]: | |
| """Pull the JSON object out of the model output, tolerating fences and | |
| stray preamble. Returns the parsed dict or None.""" | |
| text = raw.strip() | |
| # Strip ```json ... ``` fences if present. | |
| fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL) | |
| if fence: | |
| text = fence.group(1).strip() | |
| # Fall back to the outermost {...} span. | |
| start = text.find("{") | |
| end = text.rfind("}") | |
| if start == -1 or end == -1 or end <= start: | |
| return None | |
| candidate = text[start : end + 1] | |
| try: | |
| return json.loads(candidate) | |
| except json.JSONDecodeError: | |
| return None | |
| def _convert_bbox(self, bbox): | |
| """Qwen3-VL emits NORMALIZED 0-1000 boxes in [x1,y1,x2,y2] order (verified | |
| empirically: coords are stable across input resolution). Our stored | |
| format is also 0-1000 but in [y1,x1,y2,x2] order, so this only reorders | |
| and clamps -- no pixel scaling. Returns the box or None to drop it.""" | |
| if not isinstance(bbox, (list, tuple)) or len(bbox) != 4: | |
| return None | |
| try: | |
| x1, y1, x2, y2 = [float(v) for v in bbox] | |
| except (TypeError, ValueError): | |
| return None | |
| x1, x2 = sorted((max(0, min(1000, round(x1))), max(0, min(1000, round(x2))))) | |
| y1, y2 = sorted((max(0, min(1000, round(y1))), max(0, min(1000, round(y2))))) | |
| if y2 <= y1 or x2 <= x1: | |
| return None | |
| # stored order is [y1, x1, y2, x2] | |
| return [y1, x1, y2, x2] | |
| def _normalize_caption(self, data: dict) -> dict: | |
| """Cleanup the parsed caption before storage. The model emits bboxes in | |
| [x1,y1,x2,y2]; convert each to our stored [y1,x1,y2,x2] order, then hand off | |
| to the shared normalizer for the rest: drop aspect_ratio, enforce the | |
| photo/art_style branch and key order, canonicalize medium, and cap/uppercase | |
| color palettes (16 per image, 5 per element).""" | |
| decon = data.get("compositional_deconstruction", {}) | |
| elements = decon.get("elements", []) if isinstance(decon, dict) else [] | |
| if isinstance(elements, list): | |
| for el in elements: | |
| if isinstance(el, dict) and "bbox" in el: | |
| cleaned = self._convert_bbox(el["bbox"]) | |
| if cleaned is None: | |
| el.pop("bbox", None) | |
| else: | |
| el["bbox"] = cleaned | |
| return normalize_caption_dict(data) | |
| def get_caption_for_file(self, file_path: str) -> Optional[str]: | |
| try: | |
| # Read true dimensions before any resize so the aspect ratio is exact. | |
| with Image.open(file_path) as probe: | |
| width, height = probe.size | |
| aspect_ratio = self.compute_aspect_ratio(width, height) | |
| img = self.load_pil_image(file_path, max_res=self.caption_config.max_res) | |
| prompt = self.build_prompt(aspect_ratio) | |
| messages = [ | |
| { | |
| "role": "user", | |
| "content": [ | |
| {"type": "image", "image": img}, | |
| {"type": "text", "text": prompt}, | |
| ], | |
| } | |
| ] | |
| inputs = self.processor.apply_chat_template( | |
| messages, | |
| tokenize=True, | |
| add_generation_prompt=True, | |
| return_dict=True, | |
| return_tensors="pt", | |
| ) | |
| inputs = inputs.to(self.device_torch) | |
| generated_ids = self.model.generate( | |
| **inputs, max_new_tokens=self.caption_config.max_new_tokens | |
| ) | |
| generated_ids_trimmed = [ | |
| out_ids[len(in_ids) :] | |
| for in_ids, out_ids in zip(inputs.input_ids, generated_ids) | |
| ] | |
| output_text = self.processor.batch_decode( | |
| generated_ids_trimmed, | |
| skip_special_tokens=True, | |
| clean_up_tokenization_spaces=False, | |
| )[0].strip() | |
| data = self._extract_json(output_text) | |
| if data is None: | |
| print( | |
| f"[IdeogramCaptioner] Could not parse JSON for {file_path}; " | |
| f"saving raw output with regex-adapted bboxes." | |
| ) | |
| # JSON is malformed so we can't swap bboxes per-element. Adapt them | |
| # directly in the raw text instead, so the boxes still render right. | |
| return swap_bbox_xy_in_text(output_text) | |
| data = self._normalize_caption(data) | |
| # Store pretty JSON for QC/editing; the dataloader minifies at load. | |
| return json.dumps(data, ensure_ascii=False, indent=2) | |
| except Exception as e: | |
| print(f"Error processing {file_path}: {e}") | |
| return None | |
Xet Storage Details
- Size:
- 7.87 kB
- Xet hash:
- 0792c8eb282898143841b003b768f906c9a5589c600c555d5b9da1ab29e8a9dc
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.