#!/usr/bin/env python3 """Visualise PANORAMA's masks: one panel per phrase, plus a combined overlay. python visualize.py --image test.png --pred ./out # after predict_panocaps.py python visualize.py --image ./my_images/ --pred ./out # a whole folder For each image it writes a folder _viz/ containing: 00_.png ... the image with ONLY that phrase's mask(s) lit; everything else dimmed overlay.png all masks tinted per phrase + a colour->phrase legend caption.txt the caption verbatim, with the

...

[SEG] markers caption_clean.txt the same caption with the markers stripped Per-phrase panels dim the background instead of tinting the mask, so you judge the mask against the object's real colours (a 50% tint turns a red mug green). The overlay is for coverage; the panels are for correctness. Pillow + numpy only — no GPU, no extra libraries. """ import argparse import json import os import re import numpy as np from PIL import Image, ImageDraw, ImageFont # Colour is per PHRASE, not per mask: one phrase can own several masks ("three chairs"), and they # belong together. masks_per_phrase in the json gives the split. # Okabe-Ito (colour-blind friendly), extended; cycles if a caption names more regions than colours. # Names are the canonical Okabe-Ito ones for the first 8; the rest are our tints/extensions. # Index == the phrase number printed in the per-phrase panels, so "phrase 03" is always yellow. PALETTE_NAMES = [ "orange", # 0 (230, 159, 0) Okabe-Ito "sky-blue", # 1 ( 86, 180, 233) Okabe-Ito "bluish-green", # 2 ( 0, 158, 115) Okabe-Ito "yellow", # 3 (240, 228, 66) Okabe-Ito "blue", # 4 ( 0, 114, 178) Okabe-Ito "vermillion", # 5 (213, 94, 0) Okabe-Ito "reddish-purple", # 6 (204, 121, 167) Okabe-Ito "grey", # 7 (153, 153, 153) Okabe-Ito (black slot, lightened) "salmon", # 8 (255, 128, 128) extension "mint", # 9 (128, 255, 128) extension "periwinkle", # 10 (128, 128, 255) extension "apricot", # 11 (255, 200, 100) extension "amethyst", # 12 (170, 110, 200) extension "teal", # 13 (100, 200, 190) extension "raspberry", # 14 (200, 90, 140) extension ] PALETTE = [ (230, 159, 0), (86, 180, 233), (0, 158, 115), (240, 228, 66), (0, 114, 178), (213, 94, 0), (204, 121, 167), (153, 153, 153), (255, 128, 128), (128, 255, 128), (128, 128, 255), (255, 200, 100), (170, 110, 200), (100, 200, 190), (200, 90, 140), ] assert len(PALETTE_NAMES) == len(PALETTE), "palette names and colours out of sync" def colour_name(i): """Name of the colour phrase `i` is drawn in (wraps like PALETTE does).""" return PALETTE_NAMES[i % len(PALETTE_NAMES)] OVERLAY_ALPHA = 0.45 # combined view: strong enough to read coverage DIM = 0.28 # per-phrase panels: how much of the image survives outside the mask def phrase_of_mask(counts, n_masks): """mask index -> phrase index, from masks_per_phrase (falls back to 1:1).""" if not counts: return list(range(n_masks)) out = [] for pi, c in enumerate(counts): out.extend([pi] * int(c)) if len(out) < n_masks: # trust the masks, not the counts out.extend([len(counts) - 1] * (n_masks - len(out))) return out[:n_masks] def outline(mask): """1px boundary of a boolean mask.""" e = np.zeros_like(mask) e[:-1, :] |= mask[:-1, :] ^ mask[1:, :] e[:, :-1] |= mask[:, :-1] ^ mask[:, 1:] return e def _font(size=15): for p in ('DejaVuSans.ttf', '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf'): try: return ImageFont.truetype(p, size) except Exception: pass return ImageFont.load_default() def panel(img_arr, masks, colour, label): """One phrase: its mask(s) at full brightness, everything else dimmed.""" m = np.zeros(img_arr.shape[:2], bool) for k in masks: m |= k out = (img_arr * DIM).astype(np.float32) # dim everything out[m] = img_arr[m] # restore the mask region out[outline(m)] = colour # trace it im = Image.fromarray(out.astype(np.uint8)) d = ImageDraw.Draw(im) f = _font(16) px = int(m.sum()) txt = f'{label} — {px:,} px' + (f' ({len(masks)} masks)' if len(masks) > 1 else '') x0, y0, x1, y1 = d.textbbox((0, 0), txt, font=f) d.rectangle([6, 6, 14 + (x1 - x0), 14 + (y1 - y0)], fill=(0, 0, 0)) d.rectangle([6, 6, 12, 14 + (y1 - y0)], fill=colour) # colour tab d.text((16, 10), txt, fill=(255, 255, 255), font=f) return im def _wrap(draw, text, font, width): """Greedy word-wrap to `width` px.""" lines, cur = [], '' for w in text.split(): t = f'{cur} {w}'.strip() if draw.textlength(t, font=font) <= width or not cur: cur = t else: lines.append(cur) cur = w if cur: lines.append(cur) return lines def overlay(img_arr, masks, phrases, m2p, caption=''): arr = img_arr.astype(np.float32) for i, m in enumerate(masks): if not m.any(): continue c = np.array(PALETTE[m2p[i] % len(PALETTE)], np.float32) arr[m] = (1 - OVERLAY_ALPHA) * arr[m] + OVERLAY_ALPHA * c for i, m in enumerate(masks): if m.any(): arr[outline(m)] = PALETTE[m2p[i] % len(PALETTE)] im = Image.fromarray(arr.astype(np.uint8)) f, fc = _font(14), _font(15) probe = ImageDraw.Draw(im) # caption block: the [SEG] markers are noise on a picture that already shows the masks cap = re.sub(r'\s*\[SEG\]', '', caption).replace('

', '').replace('

', '') cap = re.sub(r'\s+', ' ', cap).strip() cap_lines = _wrap(probe, cap, fc, im.width - 24) if cap else [] cap_h = (len(cap_lines) * 20 + 16) if cap_lines else 0 lh = 20 strip_h = cap_h + max(lh * len(phrases) + 12, 24) strip = Image.new('RGB', (im.width, strip_h), (255, 255, 255)) sd = ImageDraw.Draw(strip) y = 8 for ln in cap_lines: sd.text((12, y), ln, fill=(0, 0, 0), font=fc) y += 20 if cap_lines: y += 8 sd.line([12, y - 5, im.width - 12, y - 5], fill=(210, 210, 210)) for pi, ph in enumerate(phrases): sd.rectangle([6, y + 3, 20, y + 15], fill=PALETTE[pi % len(PALETTE)], outline=(0, 0, 0)) n = m2p.count(pi) sd.text((28, y), f'{pi:02d} {ph.strip()}' + (f' ({n} masks)' if n > 1 else ''), fill=(0, 0, 0), font=f) y += lh out = Image.new('RGB', (im.width, im.height + strip_h), (255, 255, 255)) out.paste(im, (0, 0)) out.paste(strip, (0, im.height)) return out def slug(s, n=40): s = re.sub(r'[^a-z0-9]+', '_', s.strip().lower()).strip('_') return (s[:n] or 'phrase') def run(image_path, pred_dir): stem = os.path.splitext(os.path.basename(image_path))[0] jf, mf = os.path.join(pred_dir, f'{stem}.json'), os.path.join(pred_dir, f'{stem}_masks.npz') if not (os.path.isfile(jf) and os.path.isfile(mf)): miss = [os.path.basename(f) for f in (jf, mf) if not os.path.isfile(f)] print(f' skip {stem}: missing {", ".join(miss)} in {pred_dir} — run predict_panocaps.py ' f'with --out {pred_dir} first') return d = json.load(open(jf)) masks = np.load(mf)['masks'].astype(bool) phrases = d['phrases'] img = np.asarray(Image.open(image_path).convert('RGB')) if masks.size and masks.shape[1:] != img.shape[:2]: print(f' !! {stem}: masks are {masks.shape[1:]} but the image is {img.shape[:2]}') return m2p = phrase_of_mask(d.get('masks_per_phrase'), len(masks)) outdir = os.path.join(pred_dir, f'{stem}_viz') os.makedirs(outdir, exist_ok=True) # The caption, next to the panels, in both forms: # caption.txt — verbatim, with the

...

[SEG] markers. This is the alignment # record: the i-th

...

is the i-th phrase, hence panel 00, 01, ... # caption_clean.txt — the markers stripped, i.e. the prose on its own, for reading or for # feeding somewhere that does not want the markup. raw = d['caption'] clean = re.sub(r'\s*\[SEG\]', '', raw).replace('

', '').replace('

', '') clean = re.sub(r'\s+', ' ', clean) # the model writes "

red mug

[SEG]." — the spaces live INSIDE the tags, so removing them # strands a space before the punctuation ("a red mug ."). Close it up. clean = re.sub(r'\s+([,.;:!?%)\]])', r'\1', clean) clean = re.sub(r'([(\[])\s+', r'\1', clean).strip() open(os.path.join(outdir, 'caption.txt'), 'w').write(raw.strip() + '\n') open(os.path.join(outdir, 'caption_clean.txt'), 'w').write(clean + '\n') for pi, ph in enumerate(phrases): mine = [masks[i] for i in range(len(masks)) if m2p[i] == pi] if not mine: continue panel(img, mine, PALETTE[pi % len(PALETTE)], f'{pi:02d} {ph.strip()}').save( os.path.join(outdir, f'{pi:02d}_{slug(ph)}.png')) overlay(img, masks, phrases, m2p).save(os.path.join(outdir, 'overlay.png')) print(f'{stem}: {len(masks)} mask(s), {len(phrases)} phrase(s) -> {outdir}/ ' f'({len(phrases)} panels + overlay.png + caption.txt/caption_clean.txt)') def main(): ap = argparse.ArgumentParser() ap.add_argument('--image', required=True, help='the image file, or the directory you predicted on') ap.add_argument('--pred', default='./out', help='the --out DIRECTORY predict_panocaps.py wrote (not a file)') a = ap.parse_args() if os.path.isfile(a.pred): # easy slip: passing _masks.npz a.pred = os.path.dirname(os.path.abspath(a.pred)) print(f'--pred is a file; using its directory: {a.pred}') elif not os.path.isdir(a.pred): raise SystemExit(f'--pred: no such directory: {a.pred}') imgs = ([os.path.join(a.image, f) for f in sorted(os.listdir(a.image)) if f.lower().endswith(('.jpg', '.jpeg', '.png', '.webp', '.bmp'))] if os.path.isdir(a.image) else [a.image]) for p in imgs: run(p, a.pred) if __name__ == '__main__': main()