File size: 8,304 Bytes
e8fa066 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | """Unified pool of ALL H&E image-text pairs in PathGen-1.6M/ (every text-image source of PAIR_SUMMARY.md; all TCGA
patches). Images are de-duplicated by patch key <wsi>_<x>_<y> (the same patch recurs across sources); one image can
carry several texts. Text types:
caption : PathGen-1.6M captions, figure_caption_summary / summary_testing / subsample_summary, slide_report desc,
sample_data/output txt, output_subsample txt, path_reason_data caption
summary : path_summary_train(.json, _5w.json) one-paragraph trainee summaries
qa_open : open-ended Q/A -> assistant answers joined (PathGen-Instruct OPEN, output_subsample json,
qa_pathinstruct_withreason*.csv answers, path_reason_data question+answer)
qa_close : multiple-choice Q/A -> question stem + text of the correct option (PathGen-Instruct CLOSE,
qa_instruct_merged_onlyqaforrl, output_reason json, multiple-choice csv rows)
Outputs
cache/allimg_meta.csv : unique images (idx, key, wsi_id, patient, image_path) -> image features
cache/alltxt_meta.csv : unique (image, text) pairs (idx, img_idx, caption, ttype, source) -> text features
(the `caption` column holds the text so the extractors' --set option works unchanged)
Not used: pathor1_output*.csv (hashed non-TCGA image names, images absent).
"""
import ast
import glob
import json
import os
import re
import pandas as pd
from common import CACHE, ROOT
PG = os.path.join(ROOT, "..", "PathGen-1.6M")
img, txt = {}, [] # key -> image path ; list of (key, text, ttype, source)
KEY = re.compile(r"(TCGA-[A-Z0-9-]+\.[A-Za-z0-9-]+)_(\d+)_(\d+)")
def key_of(s):
m = KEY.search(os.path.basename(str(s)))
return f"{m.group(1)}_{m.group(2)}_{m.group(3)}" if m else None
def add_img(key, path):
if key and key not in img and os.path.exists(path):
img[key] = path
def add_txt(key, text, ttype, source):
text = re.sub(r"\s+", " ", str(text).replace("<image>", " ")).strip()
if key and text and text.lower() != "nan":
txt.append((key, text, ttype, source))
def close_text(question, answer):
"""multiple-choice -> 'question stem + correct option text'"""
q = question.replace("<image>", " ")
opts = dict(re.findall(r"(?:^|\n)\s*\(?([A-F])[\).:]\s*([^\n]+)", q))
stem = re.split(r"\n\s*\(?A[\).:]", q)[0].strip()
a = answer.strip()
letter = re.match(r"^\(?([A-F])\)?(?:[\).:\s]|$)", a)
if letter and letter.group(1) in opts:
a = opts[letter.group(1)]
elif letter and len(a) <= 3:
return None
else:
a = re.sub(r"^\(?[A-F]\)\s*", "", a)
return f"{stem} {a.strip()}"
def conv_text(conv):
"""returns (ttype, text) for a LLaVA-style conversation list"""
gpt = [c["value"].strip() for c in conv if c["from"] == "gpt"]
if len(gpt) == 1 and re.fullmatch(r"\(?[A-F]\)?\.?", gpt[0]):
human = [c["value"] for c in conv if c["from"] == "human"][0]
t = close_text(human, gpt[0])
return ("qa_close", t) if t else (None, None)
return "qa_open", " ".join(gpt)
# 1. PathGen-1.6M captions (Data/images, 672 px)
for x in json.load(open(os.path.join(PG, "Data", "PathGen-1.6M.json"))):
k = f"{x['wsi_id']}_{x['position'][0]}_{x['position'][1]}"
add_img(k, os.path.join(PG, "Data", "images", k + ".png"))
add_txt(k, x["caption"], "caption", "pathgen1.6m")
# 2. PathGen-Instruct (OPEN + CLOSE)
idx = pd.read_csv(os.path.join(PG, "PathGen-Instruct.json.image_index.csv")).drop_duplicates("key").set_index("key")
for x in json.load(open(os.path.join(PG, "PathGen-Instruct.json"))):
k = f"{x['wsi_id']}_{x['position'][0]}_{x['position'][1]}"
if k in idx.index:
add_img(k, os.path.join(PG, idx.loc[k, "local_image"]))
tt, t = conv_text(x["conversations"])
if tt:
add_txt(k, t, tt, "pathgen-instruct")
# 3. qa_instruct_merged_onlyqaforrl
for x in json.load(open(os.path.join(PG, "qa_instruct_merged_onlyqaforrl.json"))):
k = x.get("path_list") or f"{x['wsi_id']}_{x['position'][0]}_{x['position'][1]}"
for c in glob.glob(os.path.join(PG, "qa_instruct_merged_images", k + ".*")):
if c.endswith((".png", ".jpg")):
add_img(k, c)
tt, t = conv_text(x["conversations"])
if tt:
add_txt(k, t, tt, "qa_instruct_merged")
# 4. summaries
for f in ["path_summary_train.json", "path_summary_train_5w.json"]:
for x in json.load(open(os.path.join(PG, f))):
k = key_of(x["images"][0])
add_img(k, os.path.join(PG, x["images"][0]))
add_txt(k, [m["content"] for m in x["messages"] if m["role"] == "assistant"][0], "summary", f.replace(".json", ""))
# 5. output_reason (png + json conversation), output_subsample (png + txt caption / json conversation), sample_data
for d, src in [("output_reason", "output_reason"), ("output_subsample", "output_subsample"), ("sample_data/output", "sample_data")]:
for pth in glob.glob(os.path.join(PG, d, "*.png")):
k = key_of(pth)
add_img(k, pth)
base = pth[:-4]
if os.path.exists(base + ".txt"):
add_txt(k, open(base + ".txt").read(), "caption", src)
if os.path.exists(base + ".json"):
tt, t = conv_text(json.load(open(base + ".json")))
if tt:
add_txt(k, t, tt, src)
# 6. qa_pathinstruct_withreason*.csv (images in output_reason/)
for f in sorted(glob.glob(os.path.join(PG, "qa_pathinstruct_withreason*.csv"))):
for _, r in pd.read_csv(f).iterrows():
k = key_of(r.get("path_list", r.get("imagep")))
add_img(k, os.path.join(PG, str(r["imagep"]).lstrip("./")))
t = close_text(r["question"], str(r["answer"])) if re.search(r"\n\s*\(?A[\).:]", str(r["question"])) else r["answer"]
if t:
add_txt(k, t, "qa_close" if t is not r["answer"] else "qa_open", "qa_pathinstruct_withreason")
# 7. path_reason_data (caption + diagnostic Q/A)
for f in sorted(glob.glob(os.path.join(PG, "path_reason_data_3600*.csv"))):
for _, r in pd.read_csv(f).iterrows():
k = key_of(r["patch_id"])
for c in [os.path.join(PG, str(r["patch_id"]).lstrip("./")), os.path.join(PG, "output_subsample", os.path.basename(str(r["patch_id"])))]:
add_img(k, c)
add_txt(k, r["caption"], "caption", "path_reason_data")
add_txt(k, f"{r['question']} {r['answer']}", "qa_open", "path_reason_data")
# 8. figure_caption_summary (summary_testing_data / subsample_summary_sentence), slide_report
for f in ["summary_testing_data.csv", "subsample_summary_sentence.csv"]:
for _, r in pd.read_csv(os.path.join(PG, f)).iterrows():
x, y = ast.literal_eval(r["position"])
k = f"{r['wsi_id']}_{x}_{y}"
add_img(k, os.path.join(PG, "figure_caption_summary", k + ".png"))
add_txt(k, r["caption"], "caption", "figure_caption_summary")
for f in glob.glob(os.path.join(PG, "slide_report", "*", "*_desc.csv")):
k = key_of(f.replace("_desc.csv", ""))
add_img(k, f.replace("_desc.csv", ".png"))
for _, r in pd.read_csv(f).iterrows():
add_txt(k, r["caption"], "caption", "slide_report")
T = pd.DataFrame(txt, columns=["key", "caption", "ttype", "source"])
n_raw = len(T)
T = T[T.key.isin(img)]
n_noimg = n_raw - len(T)
T = T.groupby(["key", "caption"], as_index=False).agg(ttype=("ttype", "first"), source=("source", lambda s: "|".join(sorted(set(s)))))
I = pd.DataFrame({"key": sorted(T.key.unique())})
I["image_path"] = I.key.map(img)
I["wsi_id"] = I.key.str.rsplit("_", n=2).str[0]
I["patient"] = I.wsi_id.str[:12]
I["caption"] = ""
T["img_idx"] = T.key.map(dict(zip(I.key, I.index)))
I.to_csv(os.path.join(CACHE, "allimg_meta.csv"), index_label="idx")
T[["img_idx", "key", "caption", "ttype", "source"]].to_csv(os.path.join(CACHE, "alltxt_meta.csv"), index_label="idx")
print(f"texts: {n_raw} raw, {n_noimg} without an image on disk, {len(T)} unique (image, text) pairs")
print(f"images: {len(I)} unique patches, {I.wsi_id.nunique()} slides, {I.patient.nunique()} patients")
print(T.ttype.value_counts().to_string())
print(T.source.str.split("|").explode().value_counts().to_string())
print("texts per image:", T.groupby("img_idx").size().describe().round(2).to_dict())
print("words per text by type:", T.assign(w=T.caption.str.split().str.len()).groupby("ttype").w.median().to_dict())
|