File size: 1,581 Bytes
9a5439c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | """Remove junk texts from the TCGA image-text pairs (cache/alltxt_meta.csv) and list the images left without real text.
Junk = bare multiple-choice answer letters ("A A B B"), "error error" strings, or < 5 words (answers without question).
Output cache/alltxt_junk.csv (idx = row of alltxt_meta.csv, junk, reason) -> 33_ trains / tests on non-junk only
cache/notext_meta.csv (img_idx, image_path, patient) images whose texts are all junk -> 27_ --sets tcga_notext
(few-shot GPT captions; VALIDATION ONLY, never trained on)
"""
import os
import pandas as pd
from common import CACHE
T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
c = T.caption.astype(str)
w = c.str.split().str.len()
reason = pd.Series("", index=T.index)
reason[w < 5] = "fewer than 5 words"
reason[c.str.lower().str.fullmatch(r"(\s*error\s*)+")] = "error string"
reason[c.str.fullmatch(r"\s*([A-Ea-e][\s,.;]*)+")] = "answer letters"
J = pd.DataFrame({"idx": T.index, "junk": reason != "", "reason": reason})
J.to_csv(os.path.join(CACHE, "alltxt_junk.csv"), index=False)
ok_img = set(T.img_idx[~J.junk])
N = I[~I.index.isin(ok_img)]
pd.DataFrame({"img_idx": N.index, "image_path": N.image_path, "patient": N.patient}).to_csv(
os.path.join(CACHE, "notext_meta.csv"), index=False)
print(f"texts {len(T)}: junk {J.junk.sum()} {J[J.junk].reason.value_counts().to_dict()}; clean {(~J.junk).sum()}")
print(f"images {len(I)}: with clean real text {len(ok_img)}, without -> notext_meta.csv {len(N)}")
|