Add RTX 4090 SDNQ vs NF4 follow-up benchmark

.gitattributes

@@ -42,3 +42,4 @@ assets/benchmarks/opensource.png filter=lfs diff=lfs merge=lfs -text
 assets/benchmarks/opensource2.png filter=lfs diff=lfs merge=lfs -text
 assets/samples/collage_landscape.jpg filter=lfs diff=lfs merge=lfs -text
 assets/comparison_matrix.webp filter=lfs diff=lfs merge=lfs -text
+assets/sdnq_vs_nf4_4090_side_by_side.webp filter=lfs diff=lfs merge=lfs -text

README.md

@@ -110,9 +110,19 @@ The matrix below contains the 10 original FP8 generations followed by the 10 SDN
 - `quantization_manifest.json`: component-level quantization timings, storage, and VRAM peaks.
 - `ideogram4_sdnq_pipeline.py`: loader helper for the SDNQ custom transformer components.
 
-## Follow-up
+## RTX 4090 Follow-up: SDNQ UInt4 vs Official NF4
+
+Hardware: RunPod NVIDIA GeForce RTX 4090, 24 GB VRAM, single process, concurrency 1. Both variants used the same 10 structured captions from `prompts.json`, 1024x1024, `V4_DEFAULT_20`, and no magic-prompt expansion. `nf4` uses the official `ideogram-ai/ideogram-4-nf4` checkpoint through the upstream `ideogram4` loader.
+
+| Variant | Cases | Load s | Load peak reserved MB | Load peak nvidia MB | Cold request s | Hot mean s | Hot max s | Gen peak reserved MB | Gen peak nvidia MB |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| sdnq | 10.00 | 211.61 | 14124.00 | 14466.00 | 59.65 | 37.05 | 37.57 | 19768.00 | 20521.00 |
+| nf4 | 10.00 | 269.31 | 15370.00 | 15766.00 | 36.57 | 36.31 | 36.77 | 21012.00 | 21801.00 |
+
+![SDNQ vs official NF4 on RTX 4090](assets/sdnq_vs_nf4_4090_side_by_side.webp)
+
+Raw follow-up metrics are in `benchmark/summary_4090_sdnq_vs_nf4.json`, `benchmark/sdnq_4090_metrics.*`, and `benchmark/nf4_4090_metrics.*`. The exact runner used for the follow-up is `benchmark/followup_runner.py`.
 
-A separate follow-up run will compare this SDNQ UInt4 checkpoint against the official `ideogram-ai/ideogram-4-nf4` checkpoint on an RTX 3090/4090-class pod and append the full-pipeline results here.
 
 ## License
 

benchmark/followup_runner.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+
+import argparse
+import csv
+import gc
+import json
+import os
+import shutil
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+import torch
+from huggingface_hub import hf_hub_download, snapshot_download
+from PIL import Image, ImageDraw, ImageFont
+
+from ideogram4 import Ideogram4Pipeline, Ideogram4PipelineConfig, PRESETS
+
+
+SDNQ_REPO = "WaveCut/ideogram-4-sdnq-uint4"
+NF4_REPO = "ideogram-ai/ideogram-4-nf4"
+DTYPE = torch.bfloat16
+
+
+def read_json(path: Path) -> Any:
+  with path.open("r", encoding="utf-8") as f:
+    return json.load(f)
+
+
+def write_json(path: Path, payload: Any) -> None:
+  path.parent.mkdir(parents=True, exist_ok=True)
+  with path.open("w", encoding="utf-8") as f:
+    json.dump(payload, f, ensure_ascii=False, indent=2)
+    f.write("\n")
+
+
+def prompt_to_string(prompt_case: dict[str, Any]) -> str:
+  return json.dumps(prompt_case["caption"], ensure_ascii=False, separators=(",", ":"))
+
+
+def current_gpu_mb() -> int | None:
+  try:
+    output = subprocess.check_output(
+      ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
+      text=True,
+      timeout=5,
+    )
+    return max(int(line.strip()) for line in output.splitlines() if line.strip())
+  except Exception:
+    return None
+
+
+class GpuPeakMonitor:
+  def __init__(self, interval: float = 0.05) -> None:
+    self.interval = interval
+    self.samples: list[int] = []
+    self._stop = threading.Event()
+    self._thread: threading.Thread | None = None
+
+  def start(self) -> None:
+    self.samples = []
+    self._stop.clear()
+    self._thread = threading.Thread(target=self._run, daemon=True)
+    self._thread.start()
+
+  def stop(self) -> int | None:
+    self._stop.set()
+    if self._thread is not None:
+      self._thread.join(timeout=2)
+    return max(self.samples) if self.samples else None
+
+  def _run(self) -> None:
+    while not self._stop.is_set():
+      value = current_gpu_mb()
+      if value is not None:
+        self.samples.append(value)
+      time.sleep(self.interval)
+
+
+def cuda_cleanup() -> None:
+  gc.collect()
+  if torch.cuda.is_available():
+    torch.cuda.empty_cache()
+    torch.cuda.reset_peak_memory_stats()
+    torch.cuda.synchronize()
+
+
+def measure(name: str, fn: Callable[[], Any], extra: dict[str, Any] | None = None) -> tuple[Any, dict[str, Any]]:
+  cuda_cleanup()
+  before = current_gpu_mb()
+  monitor = GpuPeakMonitor()
+  monitor.start()
+  start = time.perf_counter()
+  result = fn()
+  if torch.cuda.is_available():
+    torch.cuda.synchronize()
+  elapsed = time.perf_counter() - start
+  nvidia_peak = monitor.stop()
+  after = current_gpu_mb()
+  row = {
+    "name": name,
+    "elapsed_seconds": elapsed,
+    "gpu_before_mb": before,
+    "gpu_after_mb": after,
+    "gpu_peak_mb": nvidia_peak,
+    "torch_peak_allocated_mb": (
+      torch.cuda.max_memory_allocated() / 1024 / 1024 if torch.cuda.is_available() else None
+    ),
+    "torch_peak_reserved_mb": (
+      torch.cuda.max_memory_reserved() / 1024 / 1024 if torch.cuda.is_available() else None
+    ),
+  }
+  if extra:
+    row.update(extra)
+  return result, row
+
+
+def append_jsonl(path: Path, row: dict[str, Any]) -> None:
+  path.parent.mkdir(parents=True, exist_ok=True)
+  with path.open("a", encoding="utf-8") as f:
+    f.write(json.dumps(row, ensure_ascii=False, default=str) + "\n")
+
+
+def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
+  if not rows:
+    return
+  path.parent.mkdir(parents=True, exist_ok=True)
+  keys: list[str] = []
+  for row in rows:
+    for key in row:
+      if key not in keys:
+        keys.append(key)
+  with path.open("w", encoding="utf-8", newline="") as f:
+    writer = csv.DictWriter(f, fieldnames=keys)
+    writer.writeheader()
+    writer.writerows(rows)
+
+
+def load_prompts(path: Path) -> list[dict[str, Any]]:
+  if path.exists():
+    return read_json(path)
+  downloaded = Path(hf_hub_download(SDNQ_REPO, filename="prompts.json"))
+  return read_json(downloaded)
+
+
+def ensure_sdnq_helper() -> None:
+  helper = Path(hf_hub_download(SDNQ_REPO, filename="ideogram4_sdnq_pipeline.py"))
+  sys.path.insert(0, str(helper.parent))
+
+
+def load_pipeline(variant: str, device: str):
+  if variant == "sdnq":
+    ensure_sdnq_helper()
+    from ideogram4_sdnq_pipeline import Ideogram4SDNQPipeline
+
+    return Ideogram4SDNQPipeline.from_pretrained(
+      SDNQ_REPO,
+      device=device,
+      dtype=DTYPE,
+      use_quantized_matmul=False,
+      dequantize_fp32=False,
+    )
+  if variant == "nf4":
+    return Ideogram4Pipeline.from_pretrained(
+      config=Ideogram4PipelineConfig(weights_repo=NF4_REPO),
+      device=device,
+      dtype=DTYPE,
+    )
+  raise ValueError(f"unknown variant: {variant}")
+
+
+def command_generate(args: argparse.Namespace) -> None:
+  output_dir = Path(args.output_dir)
+  image_dir = output_dir / "images"
+  image_dir.mkdir(parents=True, exist_ok=True)
+  metrics_path = output_dir / f"{args.variant}_metrics.jsonl"
+  if metrics_path.exists():
+    metrics_path.unlink()
+  prompts = load_prompts(Path(args.prompts))
+  preset = PRESETS[args.preset]
+
+  pipe, load_row = measure(
+    f"{args.variant}_load",
+    lambda: load_pipeline(args.variant, args.device),
+    {"variant": args.variant, "hardware": args.hardware, "preset": args.preset},
+  )
+  append_jsonl(metrics_path, load_row)
+  rows = [load_row]
+
+  for idx, case in enumerate(prompts):
+    prompt = prompt_to_string(case)
+    seed = int(case.get("seed", idx))
+    height = int(case.get("height", args.height))
+    width = int(case.get("width", args.width))
+
+    def run_case() -> Image.Image:
+      return pipe(
+        prompt,
+        height=height,
+        width=width,
+        num_steps=preset.num_steps,
+        guidance_schedule=preset.guidance_schedule,
+        mu=preset.mu,
+        std=preset.std,
+        seed=seed,
+        raise_on_caption_issues=False,
+      )[0]
+
+    image, row = measure(
+      f"{args.variant}_generate",
+      run_case,
+      {
+        "variant": args.variant,
+        "hardware": args.hardware,
+        "case_id": case["id"],
+        "case_index": idx,
+        "seed": seed,
+        "height": height,
+        "width": width,
+        "preset": args.preset,
+        "request_temperature": "cold" if idx == 0 else "hot",
+      },
+    )
+    out_path = image_dir / f"{idx + 1:02d}_{case['id']}_{args.variant}.png"
+    image.save(out_path)
+    row["image"] = str(out_path)
+    append_jsonl(metrics_path, row)
+    rows.append(row)
+    print(json.dumps(row, ensure_ascii=False, default=str), flush=True)
+
+  write_csv(output_dir / f"{args.variant}_metrics.csv", rows)
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+  if not path.exists():
+    return []
+  return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def summarize_variant(rows: list[dict[str, Any]], variant: str) -> dict[str, Any]:
+  load = next((r for r in rows if r.get("name") == f"{variant}_load"), {})
+  gens = [r for r in rows if r.get("name") == f"{variant}_generate"]
+  cold = next((r for r in gens if r.get("request_temperature") == "cold"), {})
+  hot = [r for r in gens if r.get("request_temperature") == "hot"]
+
+  def mean(key: str, items: list[dict[str, Any]]) -> float | None:
+    vals = [float(x[key]) for x in items if x.get(key) not in (None, "")]
+    return sum(vals) / len(vals) if vals else None
+
+  def maxv(key: str, items: list[dict[str, Any]]) -> float | None:
+    vals = [float(x[key]) for x in items if x.get(key) not in (None, "")]
+    return max(vals) if vals else None
+
+  return {
+    "variant": variant,
+    "load_seconds": load.get("elapsed_seconds"),
+    "load_peak_reserved_mb": load.get("torch_peak_reserved_mb"),
+    "load_peak_nvidia_mb": load.get("gpu_peak_mb"),
+    "cold_request_seconds": cold.get("elapsed_seconds"),
+    "cold_request_peak_reserved_mb": cold.get("torch_peak_reserved_mb"),
+    "cold_request_peak_nvidia_mb": cold.get("gpu_peak_mb"),
+    "hot_request_mean_seconds": mean("elapsed_seconds", hot),
+    "hot_request_max_seconds": maxv("elapsed_seconds", hot),
+    "generation_peak_reserved_mb": maxv("torch_peak_reserved_mb", gens),
+    "generation_peak_nvidia_mb": maxv("gpu_peak_mb", gens),
+    "cases": len(gens),
+  }
+
+
+def fmt(value: Any) -> str:
+  if value is None or value == "":
+    return ""
+  if isinstance(value, str):
+    return value
+  return f"{float(value):.2f}"
+
+
+def markdown_table(rows: list[dict[str, Any]], keys: list[tuple[str, str]]) -> str:
+  header = "| " + " | ".join(label for label, _ in keys) + " |"
+  sep = "| " + " | ".join("---" for _ in keys) + " |"
+  body = ["| " + " | ".join(fmt(row.get(key)) for _, key in keys) + " |" for row in rows]
+  return "\n".join([header, sep, *body])
+
+
+def load_font(size: int) -> ImageFont.ImageFont:
+  for path in [
+    "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
+    "/usr/share/fonts/truetype/liberation2/LiberationSans-Regular.ttf",
+  ]:
+    try:
+      return ImageFont.truetype(path, size)
+    except Exception:
+      pass
+  return ImageFont.load_default()
+
+
+def draw_centered(draw: ImageDraw.ImageDraw, xy: tuple[int, int, int, int], text: str, font: ImageFont.ImageFont, fill: tuple[int, int, int]) -> None:
+  left, top, right, bottom = xy
+  bbox = draw.textbbox((0, 0), text, font=font)
+  x = left + (right - left - (bbox[2] - bbox[0])) // 2
+  y = top + (bottom - top - (bbox[3] - bbox[1])) // 2
+  draw.text((x, y), text, font=font, fill=fill)
+
+
+def make_side_by_side_matrix(
+  left_images: list[Path],
+  right_images: list[Path],
+  left_label: str,
+  right_label: str,
+  output_path: Path,
+) -> None:
+  if len(left_images) != len(right_images):
+    raise ValueError("left and right image counts differ")
+  count = len(left_images)
+  canvas_size = 8192
+  header_h = 160
+  row_h = (canvas_size - header_h) // count
+  col_w = canvas_size // 2
+  tile = min(col_w, row_h) - 18
+  bg = (18, 18, 18)
+  line = (58, 58, 58)
+  canvas = Image.new("RGB", (canvas_size, canvas_size), bg)
+  draw = ImageDraw.Draw(canvas)
+  header_font = load_font(82)
+  label_font = load_font(36)
+  draw.rectangle((0, 0, canvas_size, header_h), fill=(28, 28, 28))
+  draw_centered(draw, (0, 0, col_w, header_h), left_label, header_font, (245, 245, 245))
+  draw_centered(draw, (col_w, 0, canvas_size, header_h), right_label, header_font, (245, 245, 245))
+  draw.line((col_w, 0, col_w, canvas_size), fill=line, width=3)
+
+  for idx, (left_path, right_path) in enumerate(zip(left_images, right_images)):
+    y = header_h + idx * row_h
+    draw.line((0, y, canvas_size, y), fill=line, width=1)
+    for col, path in enumerate([left_path, right_path]):
+      with Image.open(path) as img:
+        img = img.convert("RGB")
+        img.thumbnail((tile, tile), Image.Resampling.LANCZOS)
+        x0 = col * col_w
+        px = x0 + (col_w - img.width) // 2
+        py = y + (row_h - img.height) // 2
+        canvas.paste(img, (px, py))
+      label = path.stem.split("_", 1)[-1].rsplit("_", 1)[0]
+      draw.text((col * col_w + 28, y + 16), f"{idx + 1:02d} {label}", font=label_font, fill=(230, 230, 230))
+
+  output_path.parent.mkdir(parents=True, exist_ok=True)
+  canvas.save(output_path, "WEBP", quality=95, method=6)
+
+
+def command_collect(args: argparse.Namespace) -> None:
+  results_dir = Path(args.results_dir)
+  publish_dir = Path(args.publish_dir)
+  publish_dir.mkdir(parents=True, exist_ok=True)
+  sdnq_rows = read_jsonl(results_dir / "sdnq" / "sdnq_metrics.jsonl")
+  nf4_rows = read_jsonl(results_dir / "nf4" / "nf4_metrics.jsonl")
+  summaries = [summarize_variant(sdnq_rows, "sdnq"), summarize_variant(nf4_rows, "nf4")]
+  write_json(publish_dir / "summary_4090_sdnq_vs_nf4.json", summaries)
+
+  sdnq_images = sorted((results_dir / "sdnq" / "images").glob("*_sdnq.png"))
+  nf4_images = sorted((results_dir / "nf4" / "images").glob("*_nf4.png"))
+  matrix_path = publish_dir / "sdnq_vs_nf4_4090_side_by_side.webp"
+  make_side_by_side_matrix(sdnq_images, nf4_images, "SDNQ UInt4", "Official NF4", matrix_path)
+
+  for rel in [
+    "sdnq/sdnq_metrics.jsonl",
+    "sdnq/sdnq_metrics.csv",
+    "nf4/nf4_metrics.jsonl",
+    "nf4/nf4_metrics.csv",
+  ]:
+    src = results_dir / rel
+    if src.exists():
+      shutil.copy2(src, publish_dir / src.name.replace("_metrics", "_4090_metrics"))
+
+  table = markdown_table(
+    summaries,
+    [
+      ("Variant", "variant"),
+      ("Cases", "cases"),
+      ("Load s", "load_seconds"),
+      ("Load peak reserved MB", "load_peak_reserved_mb"),
+      ("Load peak nvidia MB", "load_peak_nvidia_mb"),
+      ("Cold request s", "cold_request_seconds"),
+      ("Hot mean s", "hot_request_mean_seconds"),
+      ("Hot max s", "hot_request_max_seconds"),
+      ("Gen peak reserved MB", "generation_peak_reserved_mb"),
+      ("Gen peak nvidia MB", "generation_peak_nvidia_mb"),
+    ],
+  )
+  (publish_dir / "README_APPEND.md").write_text(
+    f"""## RTX 4090 Follow-up: SDNQ UInt4 vs Official NF4
+
+Hardware: RunPod NVIDIA GeForce RTX 4090, 24 GB VRAM, single process, concurrency 1. Both variants used the same 10 structured captions from `prompts.json`, 1024x1024, `V4_DEFAULT_20`, and no magic-prompt expansion. `nf4` uses the official `ideogram-ai/ideogram-4-nf4` checkpoint through the upstream `ideogram4` loader.
+
+{table}
+
+![SDNQ vs official NF4 on RTX 4090](assets/sdnq_vs_nf4_4090_side_by_side.webp)
+""",
+    encoding="utf-8",
+  )
+  print(table)
+  print(matrix_path)
+
+
+def main() -> None:
+  parser = argparse.ArgumentParser()
+  sub = parser.add_subparsers(dest="command", required=True)
+
+  gen = sub.add_parser("generate")
+  gen.add_argument("--variant", choices=["sdnq", "nf4"], required=True)
+  gen.add_argument("--prompts", default="/workspace/ideogram4_followup/prompts.json")
+  gen.add_argument("--output-dir", required=True)
+  gen.add_argument("--device", default="cuda")
+  gen.add_argument("--height", type=int, default=1024)
+  gen.add_argument("--width", type=int, default=1024)
+  gen.add_argument("--preset", default="V4_DEFAULT_20", choices=sorted(PRESETS))
+  gen.add_argument("--hardware", default="NVIDIA GeForce RTX 4090")
+  gen.set_defaults(func=command_generate)
+
+  collect = sub.add_parser("collect")
+  collect.add_argument("--results-dir", default="/workspace/ideogram4_followup/results")
+  collect.add_argument("--publish-dir", default="/workspace/ideogram4_followup/publish")
+  collect.set_defaults(func=command_collect)
+
+  args = parser.parse_args()
+  os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1")
+  args.func(args)
+
+
+if __name__ == "__main__":
+  main()

benchmark/nf4_4090_metrics.csv

@@ -0,0 +1,12 @@
+name,elapsed_seconds,gpu_before_mb,gpu_after_mb,gpu_peak_mb,torch_peak_allocated_mb,torch_peak_reserved_mb,variant,hardware,preset,case_id,case_index,seed,height,width,request_temperature,image
+nf4_load,269.30941787501797,396,15766,15766,15349.59521484375,15370.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,,,,,,,
+nf4_generate,36.57375315600075,15766,21430,21430,20293.41552734375,20952.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,editorial_watch_photo,0,4101,1024,1024,cold,/workspace/ideogram4_followup/results/nf4/images/01_editorial_watch_photo_nf4.png
+nf4_generate,36.26296863902826,15888,21370,21761,20256.4404296875,20892.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,risograph_botanical_poster,1,4102,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/02_risograph_botanical_poster_nf4.png
+nf4_generate,36.768314866989385,15888,21490,21490,20344.2578125,21012.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,cyrillic_cafe_menu,2,4103,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/03_cyrillic_cafe_menu_nf4.png
+nf4_generate,36.241010975965764,15888,21410,21410,20290.642578125,20932.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,brutalist_architecture,3,4104,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/04_brutalist_architecture_nf4.png
+nf4_generate,36.19913812598679,15888,21370,21370,20256.4404296875,20892.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,ink_manga_rain,4,4105,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/05_ink_manga_rain_nf4.png
+nf4_generate,36.216044905013405,15888,21390,21390,20269.380859375,20912.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,museum_clay_render,5,4106,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/06_museum_clay_render_nf4.png
+nf4_generate,36.23377947497647,15888,21370,21370,20262.91064453125,20892.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,food_packaging_label,6,4107,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/07_food_packaging_label_nf4.png
+nf4_generate,36.32639682298759,15888,21430,21430,20303.583984375,20952.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,fantasy_map_typography,7,4108,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/08_fantasy_map_typography_nf4.png
+nf4_generate,36.178082400991116,15888,21350,21350,20248.12060546875,20872.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,streetwear_lookbook,8,4109,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/09_streetwear_lookbook_nf4.png
+nf4_generate,36.34287546604173,15888,21410,21801,20292.4912109375,20932.0,nf4,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,scientific_cutaway,9,4110,1024,1024,hot,/workspace/ideogram4_followup/results/nf4/images/10_scientific_cutaway_nf4.png

benchmark/nf4_4090_metrics.jsonl

@@ -0,0 +1,11 @@
+{"name": "nf4_load", "elapsed_seconds": 269.30941787501797, "gpu_before_mb": 396, "gpu_after_mb": 15766, "gpu_peak_mb": 15766, "torch_peak_allocated_mb": 15349.59521484375, "torch_peak_reserved_mb": 15370.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "preset": "V4_DEFAULT_20"}
+{"name": "nf4_generate", "elapsed_seconds": 36.57375315600075, "gpu_before_mb": 15766, "gpu_after_mb": 21430, "gpu_peak_mb": 21430, "torch_peak_allocated_mb": 20293.41552734375, "torch_peak_reserved_mb": 20952.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "editorial_watch_photo", "case_index": 0, "seed": 4101, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "cold", "image": "/workspace/ideogram4_followup/results/nf4/images/01_editorial_watch_photo_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.26296863902826, "gpu_before_mb": 15888, "gpu_after_mb": 21370, "gpu_peak_mb": 21761, "torch_peak_allocated_mb": 20256.4404296875, "torch_peak_reserved_mb": 20892.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "risograph_botanical_poster", "case_index": 1, "seed": 4102, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/02_risograph_botanical_poster_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.768314866989385, "gpu_before_mb": 15888, "gpu_after_mb": 21490, "gpu_peak_mb": 21490, "torch_peak_allocated_mb": 20344.2578125, "torch_peak_reserved_mb": 21012.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "cyrillic_cafe_menu", "case_index": 2, "seed": 4103, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/03_cyrillic_cafe_menu_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.241010975965764, "gpu_before_mb": 15888, "gpu_after_mb": 21410, "gpu_peak_mb": 21410, "torch_peak_allocated_mb": 20290.642578125, "torch_peak_reserved_mb": 20932.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "brutalist_architecture", "case_index": 3, "seed": 4104, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/04_brutalist_architecture_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.19913812598679, "gpu_before_mb": 15888, "gpu_after_mb": 21370, "gpu_peak_mb": 21370, "torch_peak_allocated_mb": 20256.4404296875, "torch_peak_reserved_mb": 20892.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "ink_manga_rain", "case_index": 4, "seed": 4105, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/05_ink_manga_rain_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.216044905013405, "gpu_before_mb": 15888, "gpu_after_mb": 21390, "gpu_peak_mb": 21390, "torch_peak_allocated_mb": 20269.380859375, "torch_peak_reserved_mb": 20912.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "museum_clay_render", "case_index": 5, "seed": 4106, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/06_museum_clay_render_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.23377947497647, "gpu_before_mb": 15888, "gpu_after_mb": 21370, "gpu_peak_mb": 21370, "torch_peak_allocated_mb": 20262.91064453125, "torch_peak_reserved_mb": 20892.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "food_packaging_label", "case_index": 6, "seed": 4107, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/07_food_packaging_label_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.32639682298759, "gpu_before_mb": 15888, "gpu_after_mb": 21430, "gpu_peak_mb": 21430, "torch_peak_allocated_mb": 20303.583984375, "torch_peak_reserved_mb": 20952.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "fantasy_map_typography", "case_index": 7, "seed": 4108, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/08_fantasy_map_typography_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.178082400991116, "gpu_before_mb": 15888, "gpu_after_mb": 21350, "gpu_peak_mb": 21350, "torch_peak_allocated_mb": 20248.12060546875, "torch_peak_reserved_mb": 20872.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "streetwear_lookbook", "case_index": 8, "seed": 4109, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/09_streetwear_lookbook_nf4.png"}
+{"name": "nf4_generate", "elapsed_seconds": 36.34287546604173, "gpu_before_mb": 15888, "gpu_after_mb": 21410, "gpu_peak_mb": 21801, "torch_peak_allocated_mb": 20292.4912109375, "torch_peak_reserved_mb": 20932.0, "variant": "nf4", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "scientific_cutaway", "case_index": 9, "seed": 4110, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/nf4/images/10_scientific_cutaway_nf4.png"}

benchmark/sdnq_4090_metrics.csv

@@ -0,0 +1,12 @@
+name,elapsed_seconds,gpu_before_mb,gpu_after_mb,gpu_peak_mb,torch_peak_allocated_mb,torch_peak_reserved_mb,variant,hardware,preset,case_id,case_index,seed,height,width,request_temperature,image
+sdnq_load,211.60528413800057,396,14522,14466,14107.06298828125,14124.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,,,,,,,
+sdnq_generate,59.65369569603354,14522,20186,20186,19050.88330078125,19708.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,editorial_watch_photo,0,4101,1024,1024,cold,/workspace/ideogram4_followup/results/sdnq/images/01_editorial_watch_photo_sdnq.png
+sdnq_generate,36.95279458502773,14622,20126,20126,19013.908203125,19648.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,risograph_botanical_poster,1,4102,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/02_risograph_botanical_poster_sdnq.png
+sdnq_generate,37.568486024974845,14622,20246,20246,19101.7255859375,19768.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,cyrillic_cafe_menu,2,4103,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/03_cyrillic_cafe_menu_sdnq.png
+sdnq_generate,37.06334384600632,14622,20186,20186,19048.1103515625,19708.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,brutalist_architecture,3,4104,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/04_brutalist_architecture_sdnq.png
+sdnq_generate,36.373742469004355,14622,20126,20521,19013.908203125,19648.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,ink_manga_rain,4,4105,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/05_ink_manga_rain_sdnq.png
+sdnq_generate,37.08211989700794,14622,20146,20146,19026.8486328125,19668.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,museum_clay_render,5,4106,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/06_museum_clay_render_sdnq.png
+sdnq_generate,37.078365966968704,14622,20146,20146,19020.37841796875,19668.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,food_packaging_label,6,4107,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/07_food_packaging_label_sdnq.png
+sdnq_generate,37.32429828296881,14622,20186,20186,19061.0517578125,19708.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,fantasy_map_typography,7,4108,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/08_fantasy_map_typography_sdnq.png
+sdnq_generate,36.95170207798947,14622,20126,20126,19005.58837890625,19648.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,streetwear_lookbook,8,4109,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/09_streetwear_lookbook_sdnq.png
+sdnq_generate,37.0877975319745,14622,20186,20186,19049.958984375,19708.0,sdnq,NVIDIA GeForce RTX 4090,V4_DEFAULT_20,scientific_cutaway,9,4110,1024,1024,hot,/workspace/ideogram4_followup/results/sdnq/images/10_scientific_cutaway_sdnq.png

benchmark/sdnq_4090_metrics.jsonl

@@ -0,0 +1,11 @@
+{"name": "sdnq_load", "elapsed_seconds": 211.60528413800057, "gpu_before_mb": 396, "gpu_after_mb": 14522, "gpu_peak_mb": 14466, "torch_peak_allocated_mb": 14107.06298828125, "torch_peak_reserved_mb": 14124.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "preset": "V4_DEFAULT_20"}
+{"name": "sdnq_generate", "elapsed_seconds": 59.65369569603354, "gpu_before_mb": 14522, "gpu_after_mb": 20186, "gpu_peak_mb": 20186, "torch_peak_allocated_mb": 19050.88330078125, "torch_peak_reserved_mb": 19708.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "editorial_watch_photo", "case_index": 0, "seed": 4101, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "cold", "image": "/workspace/ideogram4_followup/results/sdnq/images/01_editorial_watch_photo_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 36.95279458502773, "gpu_before_mb": 14622, "gpu_after_mb": 20126, "gpu_peak_mb": 20126, "torch_peak_allocated_mb": 19013.908203125, "torch_peak_reserved_mb": 19648.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "risograph_botanical_poster", "case_index": 1, "seed": 4102, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/02_risograph_botanical_poster_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 37.568486024974845, "gpu_before_mb": 14622, "gpu_after_mb": 20246, "gpu_peak_mb": 20246, "torch_peak_allocated_mb": 19101.7255859375, "torch_peak_reserved_mb": 19768.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "cyrillic_cafe_menu", "case_index": 2, "seed": 4103, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/03_cyrillic_cafe_menu_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 37.06334384600632, "gpu_before_mb": 14622, "gpu_after_mb": 20186, "gpu_peak_mb": 20186, "torch_peak_allocated_mb": 19048.1103515625, "torch_peak_reserved_mb": 19708.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "brutalist_architecture", "case_index": 3, "seed": 4104, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/04_brutalist_architecture_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 36.373742469004355, "gpu_before_mb": 14622, "gpu_after_mb": 20126, "gpu_peak_mb": 20521, "torch_peak_allocated_mb": 19013.908203125, "torch_peak_reserved_mb": 19648.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "ink_manga_rain", "case_index": 4, "seed": 4105, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/05_ink_manga_rain_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 37.08211989700794, "gpu_before_mb": 14622, "gpu_after_mb": 20146, "gpu_peak_mb": 20146, "torch_peak_allocated_mb": 19026.8486328125, "torch_peak_reserved_mb": 19668.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "museum_clay_render", "case_index": 5, "seed": 4106, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/06_museum_clay_render_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 37.078365966968704, "gpu_before_mb": 14622, "gpu_after_mb": 20146, "gpu_peak_mb": 20146, "torch_peak_allocated_mb": 19020.37841796875, "torch_peak_reserved_mb": 19668.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "food_packaging_label", "case_index": 6, "seed": 4107, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/07_food_packaging_label_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 37.32429828296881, "gpu_before_mb": 14622, "gpu_after_mb": 20186, "gpu_peak_mb": 20186, "torch_peak_allocated_mb": 19061.0517578125, "torch_peak_reserved_mb": 19708.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "fantasy_map_typography", "case_index": 7, "seed": 4108, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/08_fantasy_map_typography_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 36.95170207798947, "gpu_before_mb": 14622, "gpu_after_mb": 20126, "gpu_peak_mb": 20126, "torch_peak_allocated_mb": 19005.58837890625, "torch_peak_reserved_mb": 19648.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "streetwear_lookbook", "case_index": 8, "seed": 4109, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/09_streetwear_lookbook_sdnq.png"}
+{"name": "sdnq_generate", "elapsed_seconds": 37.0877975319745, "gpu_before_mb": 14622, "gpu_after_mb": 20186, "gpu_peak_mb": 20186, "torch_peak_allocated_mb": 19049.958984375, "torch_peak_reserved_mb": 19708.0, "variant": "sdnq", "hardware": "NVIDIA GeForce RTX 4090", "case_id": "scientific_cutaway", "case_index": 9, "seed": 4110, "height": 1024, "width": 1024, "preset": "V4_DEFAULT_20", "request_temperature": "hot", "image": "/workspace/ideogram4_followup/results/sdnq/images/10_scientific_cutaway_sdnq.png"}

benchmark/summary_4090_sdnq_vs_nf4.json

@@ -0,0 +1,30 @@
+[
+  {
+    "variant": "sdnq",
+    "load_seconds": 211.60528413800057,
+    "load_peak_reserved_mb": 14124.0,
+    "load_peak_nvidia_mb": 14466,
+    "cold_request_seconds": 59.65369569603354,
+    "cold_request_peak_reserved_mb": 19708.0,
+    "cold_request_peak_nvidia_mb": 20186,
+    "hot_request_mean_seconds": 37.05362785354696,
+    "hot_request_max_seconds": 37.568486024974845,
+    "generation_peak_reserved_mb": 19768.0,
+    "generation_peak_nvidia_mb": 20521.0,
+    "cases": 10
+  },
+  {
+    "variant": "nf4",
+    "load_seconds": 269.30941787501797,
+    "load_peak_reserved_mb": 15370.0,
+    "load_peak_nvidia_mb": 15766,
+    "cold_request_seconds": 36.57375315600075,
+    "cold_request_peak_reserved_mb": 20952.0,
+    "cold_request_peak_nvidia_mb": 21430,
+    "hot_request_mean_seconds": 36.30762351977561,
+    "hot_request_max_seconds": 36.768314866989385,
+    "generation_peak_reserved_mb": 21012.0,
+    "generation_peak_nvidia_mb": 21801.0,
+    "cases": 10
+  }
+]