| """Remove junk texts from the TCGA image-text pairs (cache/alltxt_meta.csv) and list the images left without real text. |
| Junk = bare multiple-choice answer letters ("A A B B"), "error error" strings, or < 5 words (answers without question). |
| Output cache/alltxt_junk.csv (idx = row of alltxt_meta.csv, junk, reason) -> 33_ trains / tests on non-junk only |
| cache/notext_meta.csv (img_idx, image_path, patient) images whose texts are all junk -> 27_ --sets tcga_notext |
| (few-shot GPT captions; VALIDATION ONLY, never trained on) |
| """ |
| import os |
|
|
| import pandas as pd |
|
|
| from common import CACHE |
|
|
| T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv")) |
| I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv")) |
| c = T.caption.astype(str) |
| w = c.str.split().str.len() |
| reason = pd.Series("", index=T.index) |
| reason[w < 5] = "fewer than 5 words" |
| reason[c.str.lower().str.fullmatch(r"(\s*error\s*)+")] = "error string" |
| reason[c.str.fullmatch(r"\s*([A-Ea-e][\s,.;]*)+")] = "answer letters" |
| J = pd.DataFrame({"idx": T.index, "junk": reason != "", "reason": reason}) |
| J.to_csv(os.path.join(CACHE, "alltxt_junk.csv"), index=False) |
| ok_img = set(T.img_idx[~J.junk]) |
| N = I[~I.index.isin(ok_img)] |
| pd.DataFrame({"img_idx": N.index, "image_path": N.image_path, "patient": N.patient}).to_csv( |
| os.path.join(CACHE, "notext_meta.csv"), index=False) |
| print(f"texts {len(T)}: junk {J.junk.sum()} {J[J.junk].reason.value_counts().to_dict()}; clean {(~J.junk).sum()}") |
| print(f"images {len(I)}: with clean real text {len(ok_img)}, without -> notext_meta.csv {len(N)}") |
|
|
