{ "cells": [ { "cell_type": "markdown", "id": "26283604", "metadata": {}, "source": [ "# Export tập test MIMIC-CXR_resized → thư mục theo 14 nhãn bệnh lý\n", "\n", "Chỉ cần điền **CONFIG** bên dưới rồi **Run All**.\n", "\n", "- Tải `manifest_test.csv` + tar shards từ HF (`hieu3636/cxr-vlm-data/MIMIC-CXR_resized/`).\n", "- Rút ảnh test, đổ vào `OUT//`. Ảnh multi-label → copy vào nhiều thư mục.\n", "- **Kèm report**: mỗi ảnh `.jpg` có file `.txt` (nội dung report của study đó) đặt ngay cạnh.\n", "- Repo **private** → cần token HF (điền vào `HF_TOKEN`, hoặc đã `huggingface-cli login` thì để trống)." ] }, { "cell_type": "markdown", "id": "c41dd18e", "metadata": {}, "source": [ "## 1. CONFIG — chỉnh ở đây" ] }, { "cell_type": "code", "execution_count": 1, "id": "8fd81f44", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CONFIG ok | split = test | out = D:\\USTH\\KLTN\\test_by_pathology | with_report = True\n" ] } ], "source": [ "# ==== CHỈNH CÁC BIẾN NÀY ====\n", "HF_TOKEN = \"\" # token HF (Read là đủ). Để \"\" nếu đã huggingface-cli login.\n", "REPO_ID = \"hieu3636/cxr-vlm-data\"\n", "SPLIT = \"test\" # \"train\" | \"val\" | \"test\"\n", "\n", "OUT = r\"D:\\USTH\\KLTN\\test_by_pathology\" # thư mục output\n", "WORK = r\"D:\\USTH\\KLTN\\_hf_resized_dl\" # nơi cache tải từ HF\n", "\n", "# Nếu ĐÃ có shards giải nén/tar sẵn ở máy thì trỏ vào đây để KHỎI tải lại,\n", "# ví dụ r\"D:\\USTH\\KLTN\\_hf_resized_dl\\MIMIC-CXR_resized\". Để None = tải từ HF.\n", "EXTRACTED_ROOT = None\n", "\n", "WITH_REPORT = True # True = ghi kèm .txt (report) cạnh mỗi ảnh\n", "UNCERTAIN = \"separate\" # \"separate\" (_uncertain/

) | \"merge\" | \"skip\"\n", "LINK = \"copy\" # \"copy\" | \"hardlink\" | \"symlink\" (hardlink đỡ tốn ổ)\n", "# ============================\n", "print(\"CONFIG ok | split =\", SPLIT, \"| out =\", OUT, \"| with_report =\", WITH_REPORT)" ] }, { "cell_type": "markdown", "id": "8085defe", "metadata": {}, "source": [ "## 2. Cài thư viện (chạy 1 lần)" ] }, { "cell_type": "code", "execution_count": 2, "id": "11afbe51", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "huggingface_hub đã có: 1.11.0\n" ] } ], "source": [ "# Chỉ cần huggingface_hub; tarfile/csv là built-in.\n", "try:\n", " import huggingface_hub # noqa\n", " print(\"huggingface_hub đã có:\", huggingface_hub.__version__)\n", "except ImportError:\n", " import sys, subprocess\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"huggingface_hub\"])\n", " print(\"đã cài huggingface_hub\")" ] }, { "cell_type": "markdown", "id": "ac53032e", "metadata": {}, "source": [ "## 3. Logic (không cần sửa)" ] }, { "cell_type": "code", "execution_count": 3, "id": "9ac60a59", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "logic loaded\n" ] } ], "source": [ "import os, csv, tarfile\n", "from collections import defaultdict\n", "from pathlib import Path\n", "\n", "# 14 nhãn CheXpert — đúng thứ tự dùng trong project.\n", "PATHOLOGIES = [\n", " \"No Finding\", \"Enlarged Cardiomediastinum\", \"Cardiomegaly\", \"Lung Opacity\",\n", " \"Lung Lesion\", \"Edema\", \"Consolidation\", \"Pneumonia\", \"Atelectasis\",\n", " \"Pneumothorax\", \"Pleural Effusion\", \"Pleural Other\", \"Fracture\",\n", " \"Support Devices\",\n", "]\n", "_POS = {\"1\", \"1.0\"}\n", "_UNC = {\"-1\", \"-1.0\"}\n", "_MANIFEST = {\"train\": \"manifest_train.csv\", \"val\": \"manifest_val.csv\",\n", " \"validate\": \"manifest_val.csv\", \"test\": \"manifest_test.csv\"}\n", "\n", "def _safe(n): return n.replace(\" \", \"_\")\n", "def _norm(p): return p.replace(\"\\\\\", \"/\").lstrip(\"/\")\n", "\n", "def download_from_hf(repo_id, split, work):\n", " from huggingface_hub import snapshot_download\n", " mname = _MANIFEST[split]\n", " print(f\"[download] {repo_id}:MIMIC-CXR_resized (manifest + shards) -> {work}\")\n", " snapshot_download(\n", " repo_id=repo_id, repo_type=\"dataset\", local_dir=str(work),\n", " allow_patterns=[f\"MIMIC-CXR_resized/{mname}\", \"MIMIC-CXR_resized/shards/*.tar\"],\n", " )\n", " mr = Path(work) / \"MIMIC-CXR_resized\"\n", " manifest = mr / mname\n", " shards = sorted((mr / \"shards\").glob(\"*.tar\"))\n", " assert manifest.is_file(), f\"không thấy manifest: {manifest}\"\n", " assert shards, f\"không thấy tar shard dưới {mr/'shards'}\"\n", " print(f\"[download] manifest={manifest.name} shards={len(shards)}\")\n", " return manifest, shards\n", "\n", "def load_label_map(manifest):\n", " label_map = {}\n", " with open(manifest, encoding=\"utf-8\", newline=\"\") as f:\n", " reader = csv.DictReader(f); cols = reader.fieldnames or []\n", " chex_cols = {p: f\"chex_{p}\" for p in PATHOLOGIES if f\"chex_{p}\" in cols}\n", " miss = [p for p in PATHOLOGIES if f\"chex_{p}\" not in cols]\n", " assert \"image_relpath\" in cols, f\"manifest thiếu image_relpath. Có: {cols}\"\n", " has_report = \"report_relpath\" in cols\n", " for row in reader:\n", " rel = _norm(str(row[\"image_relpath\"]).strip())\n", " pos, unc = set(), set()\n", " for p, c in chex_cols.items():\n", " v = str(row.get(c, \"\")).strip()\n", " if v in _POS: pos.add(p)\n", " elif v in _UNC: unc.add(p)\n", " rep = _norm(str(row[\"report_relpath\"]).strip()) if has_report else None\n", " label_map[rel] = {\"pos\": pos, \"unc\": unc, \"report\": rep or None}\n", " if miss: print(f\"[labels] CẢNH BÁO thiếu cột: {miss}\")\n", " if not has_report: print(\"[labels] CẢNH BÁO: manifest không có report_relpath → bỏ qua report\")\n", " print(f\"[labels] {len(label_map):,} ảnh trong manifest\")\n", " return label_map\n", "\n", "def gather_reports(shards, report_set):\n", " \"\"\"Gom text các report cần dùng (1 pass qua tar). Report nhỏ → giữ RAM.\"\"\"\n", " reports = {}\n", " if not report_set: return reports\n", " for shard in shards:\n", " with tarfile.open(shard, \"r\") as tf:\n", " for m in tf:\n", " if not m.isfile(): continue\n", " name = _norm(m.name)\n", " if name in report_set and name not in reports:\n", " reports[name] = tf.extractfile(m).read()\n", " print(f\"[reports] rút được {len(reports):,} / {len(report_set):,} report\")\n", " return reports\n", "\n", "def _place(data, dicom, paths, base, counts, link, report=None):\n", " txt = Path(dicom).stem + \".txt\"\n", " first = None\n", " for lab in paths:\n", " d = base / _safe(lab); d.mkdir(parents=True, exist_ok=True)\n", " dst = d / dicom; counts[lab] += 1\n", " if report is not None: (d / txt).write_bytes(report)\n", " if dst.exists(): continue\n", " if link == \"copy\" or first is None:\n", " dst.write_bytes(data); first = dst\n", " else:\n", " try:\n", " os.link(first, dst) if link == \"hardlink\" else os.symlink(os.path.abspath(first), dst)\n", " except OSError:\n", " dst.write_bytes(data)\n", "\n", "def export(shards, label_map, out, uncertain, link, with_report=True):\n", " out = Path(out); out.mkdir(parents=True, exist_ok=True)\n", " unc_base = out / \"_uncertain\"\n", " test_set = set(label_map)\n", " reports = {}\n", " if with_report:\n", " rset = {label_map[k][\"report\"] for k in test_set if label_map[k].get(\"report\")}\n", " reports = gather_reports(shards, rset)\n", " cpos, cunc = defaultdict(int), defaultdict(int)\n", " n_imgs = 0; n_no_rep = 0; seen = set()\n", " for si, shard in enumerate(shards, 1):\n", " print(f\"[extract] [{si}/{len(shards)}] {shard.name}\")\n", " with tarfile.open(shard, \"r\") as tf:\n", " for m in tf:\n", " if not m.isfile(): continue\n", " name = _norm(m.name)\n", " if name not in test_set: continue\n", " seen.add(name)\n", " ent = label_map[name]; pos, unc = ent[\"pos\"], ent[\"unc\"]\n", " if not pos and not (uncertain != \"skip\" and unc): continue\n", " data = tf.extractfile(m).read(); dicom = Path(name).name; n_imgs += 1\n", " rep = reports.get(ent.get(\"report\")) if with_report else None\n", " if with_report and rep is None: n_no_rep += 1\n", " if pos: _place(data, dicom, pos, out, cpos, link, rep)\n", " if unc and uncertain != \"skip\":\n", " _place(data, dicom, unc, (out if uncertain == \"merge\" else unc_base), cunc, link, rep)\n", " if with_report and n_no_rep:\n", " print(f\"[reports] CẢNH BÁO: {n_no_rep:,} ảnh không thấy report → chỉ có .jpg\")\n", " missing = test_set - seen\n", " print(f\"\\n[done] ảnh rút được: {n_imgs:,} / {len(test_set):,} trong manifest\")\n", " if missing:\n", " print(f\"[done] CẢNH BÁO: {len(missing):,} ảnh manifest không có trong shard (vd: {list(missing)[:2]})\")\n", " with open(out / \"_summary.csv\", \"w\", encoding=\"utf-8\", newline=\"\") as f:\n", " w = csv.writer(f); w.writerow([\"pathology\", \"positive_images\", \"uncertain_images\"])\n", " for p in PATHOLOGIES: w.writerow([p, cpos.get(p, 0), cunc.get(p, 0)])\n", " print(\"\\n Nhãn positive uncertain\")\n", " for p in PATHOLOGIES:\n", " print(f\" {p:28s} {cpos.get(p,0):8d} {cunc.get(p,0):8d}\")\n", " return cpos, cunc\n", "\n", "print(\"logic loaded\")" ] }, { "cell_type": "markdown", "id": "f23b1e30", "metadata": {}, "source": [ "## 4. Run" ] }, { "cell_type": "code", "execution_count": 4, "id": "df4a1338", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[download] hieu3636/cxr-vlm-data:MIMIC-CXR_resized (manifest + shards) -> D:\\USTH\\KLTN\\_hf_resized_dl\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4dfc7af1d4194177a7d67f83df163309", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Downloading (incomplete total...): 0.00B [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b1626e7949cf45a3b33fd2d1c7416aa9", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Fetching ... files: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[download] manifest=manifest_test.csv shards=2\n", "[labels] 5,000 ảnh trong manifest\n", "[reports] rút được 5,000 / 5,000 report\n", "[extract] [1/2] cxr-0000.tar\n", "[extract] [2/2] cxr-0001.tar\n", "\n", "[done] ảnh rút được: 4,883 / 5,000 trong manifest\n", "\n", " Nhãn positive uncertain\n", " No Finding 2375 0\n", " Enlarged Cardiomediastinum 29 13\n", " Cardiomegaly 387 58\n", " Lung Opacity 802 65\n", " Lung Lesion 100 22\n", " Edema 506 265\n", " Consolidation 116 73\n", " Pneumonia 313 465\n", " Atelectasis 540 198\n", " Pneumothorax 92 12\n", " Pleural Effusion 670 93\n", " Pleural Other 27 14\n", " Fracture 71 16\n", " Support Devices 449 4\n", "\n", "Xong! Output: D:\\USTH\\KLTN\\test_by_pathology\n" ] } ], "source": [ "# token\n", "if HF_TOKEN.strip():\n", " os.environ[\"HF_TOKEN\"] = HF_TOKEN.strip()\n", " os.environ[\"HUGGING_FACE_HUB_TOKEN\"] = HF_TOKEN.strip()\n", "\n", "# 1) manifest + shards\n", "if EXTRACTED_ROOT:\n", " mr = Path(EXTRACTED_ROOT)\n", " shards = sorted((mr / \"shards\").glob(\"*.tar\")) or sorted(mr.glob(\"*.tar\"))\n", " manifest = mr / _MANIFEST[SPLIT]\n", " assert shards, f\"không thấy *.tar dưới {mr}\"\n", " assert manifest.is_file(), f\"không thấy manifest: {manifest}\"\n", " print(f\"[local] manifest={manifest} shards={len(shards)}\")\n", "else:\n", " manifest, shards = download_from_hf(REPO_ID, SPLIT, WORK)\n", "\n", "# 2) đọc nhãn 3) rút ảnh (+ report)\n", "label_map = load_label_map(manifest)\n", "cpos, cunc = export(shards, label_map, OUT, UNCERTAIN, LINK, with_report=WITH_REPORT)\n", "print(f\"\\nXong! Output: {Path(OUT).resolve()}\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.7" } }, "nbformat": 4, "nbformat_minor": 5 }