{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "code", "source": [ "%%capture\n", "\n", "!pip install -q tabulate fpdf2" ], "metadata": { "id": "cdvLOdcZnzaX" }, "execution_count": 16, "outputs": [] }, { "cell_type": "markdown", "source": [ "GPU check" ], "metadata": { "id": "nfy6rq76HdiO" } }, { "cell_type": "code", "execution_count": 17, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "nxjR8SCn3L5U", "outputId": "f39a6f53-6781-4204-ab89-7309af30031c" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "python : 3.12.13\n", "torch : 2.11.0+cu128\n", "transformers: 5.13.1\n", "datasets : 4.0.0\n", "cuda avail : True\n", "gpu : Tesla T4 | 15.6 GB | compute 7.5\n", "dtype : torch.float16\n" ] } ], "source": [ "import os, warnings, logging, sys\n", "os.environ[\"HF_HUB_DISABLE_PROGRESS_BARS\"] = \"1\"\n", "os.environ[\"TRANSFORMERS_VERBOSITY\"] = \"error\"\n", "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", "warnings.filterwarnings(\"ignore\")\n", "logging.getLogger(\"huggingface_hub\").setLevel(logging.ERROR)\n", "\n", "import torch, transformers\n", "transformers.utils.logging.disable_progress_bar()\n", "transformers.utils.logging.set_verbosity_error()\n", "\n", "\n", "try:\n", " import datasets\n", " datasets.logging.set_verbosity_error()\n", " DATASETS_VERSION = datasets.__version__\n", "except Exception as e:\n", " DATASETS_VERSION = f\"unavailable ({type(e).__name__})\"\n", "\n", "from packaging.version import parse as V\n", "assert V(transformers.__version__) >= V(\"4.45\"), \\\n", " f\"transformers too old: {transformers.__version__}\"\n", "\n", "print(\"python :\", sys.version.split()[0])\n", "print(\"torch :\", torch.__version__)\n", "print(\"transformers:\", transformers.__version__)\n", "print(\"datasets :\", DATASETS_VERSION)\n", "print(\"cuda avail :\", torch.cuda.is_available())\n", "\n", "if not torch.cuda.is_available():\n", " raise RuntimeError(\"No GPU. Runtime > Change runtime type > T4 GPU, then re-run.\")\n", "\n", "gpu = torch.cuda.get_device_name(0)\n", "vram = torch.cuda.get_device_properties(0).total_memory / 1e9\n", "cc = torch.cuda.get_device_capability(0)\n", "print(f\"gpu : {gpu} | {vram:.1f} GB | compute {cc[0]}.{cc[1]}\")\n", "\n", "\n", "DTYPE = torch.bfloat16 if cc[0] >= 8 else torch.float16\n", "print(\"dtype :\", DTYPE)" ] }, { "cell_type": "markdown", "source": [ "Data layer" ], "metadata": { "id": "K_4lZRqR4UdV" } }, { "cell_type": "code", "source": [ "import pandas as pd\n", "from huggingface_hub import HfApi, hf_hub_download\n", "\n", "TABULAR_EXT = (\".csv\", \".tsv\", \".parquet\", \".json\", \".jsonl\")\n", "\n", "\n", "def _read_any(path: str) -> pd.DataFrame:\n", " \"\"\"Dispatch to the right pandas reader based on extension.\"\"\"\n", " if path.endswith(\".parquet\"):\n", " return pd.read_parquet(path)\n", " if path.endswith(\".tsv\"):\n", " return pd.read_csv(path, sep=\"\\t\")\n", " if path.endswith((\".json\", \".jsonl\")):\n", " return pd.read_json(path, lines=path.endswith(\".jsonl\"))\n", " return pd.read_csv(path)\n", "\n", "\n", "def load_hf_dataframe(hf_dataset: str) -> pd.DataFrame:\n", " \"\"\"Hub dataset id -> DataFrame. Tries load_dataset, falls back to raw files.\n", "\n", " The fallback matters: datasets>=3.0 dropped script-based repos, so datasets\n", " such as mstz/titanic can no longer be loaded the standard way.\n", " \"\"\"\n", "\n", " try:\n", " from datasets import load_dataset\n", " ds = load_dataset(hf_dataset)\n", " split = \"train\" if \"train\" in ds else list(ds.keys())[0]\n", " print(f\"[loader] load_dataset OK split='{split}'\")\n", " return ds[split].to_pandas()\n", " except Exception as e:\n", " print(f\"[loader] load_dataset failed -> {type(e).__name__}: {str(e)[:120]}\")\n", "\n", "\n", " files = HfApi().list_repo_files(hf_dataset, repo_type=\"dataset\")\n", " cands = [f for f in files if f.lower().endswith(TABULAR_EXT)]\n", " if not cands:\n", " raise RuntimeError(f\"No tabular file found in {hf_dataset}. Files: {files}\")\n", "\n", "\n", " cands.sort(key=lambda f: ((\"train\" not in f.lower()), len(f)))\n", " pick = cands[0]\n", " print(f\"[loader] fallback -> downloading '{pick}'\")\n", " df = _read_any(hf_hub_download(hf_dataset, pick, repo_type=\"dataset\"))\n", " print(\"[loader] fallback OK\")\n", " return df\n", "\n", "\n", "\n", "hf_dataset = \"mstz/titanic\"\n", "df = load_hf_dataframe(hf_dataset)\n", "print(\"\\nshape:\", df.shape)\n", "display(df.head())" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 323 }, "id": "Yk6csvF_4TpC", "outputId": "ad114027-a69e-4d0f-9cc3-6c4cb9fbd46a" }, "execution_count": 18, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "[loader] load_dataset failed -> RuntimeError: Dataset scripts are no longer supported, but found titanic.py\n", "[loader] fallback -> downloading 'titanic.csv'\n", "[loader] fallback OK\n", "\n", "shape: (891, 12)\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ " has_survived passenger_class surname \\\n", "0 0 3 'Braund' \n", "1 1 1 'Cumings' \n", "2 1 3 'Heikkinen' \n", "3 1 1 'Futrelle' \n", "4 0 3 'Allen' \n", "\n", " name sex age sibsp parch \\\n", "0 'Mr. Owen Harris' male 22 1 0 \n", "1 'Mrs. John Bradley (Florence Briggs Thayer)' female 38 1 0 \n", "2 'Miss. Laina' female 26 0 0 \n", "3 'Mrs. Jacques Heath (Lily May Peel)' female 35 1 0 \n", "4 'Mr. William Henry' male 35 0 0 \n", "\n", " ticket fare cabin embarked \n", "0 'A/5 21171' 7.2500 '' S \n", "1 'PC 17599' 71.2833 C85 C \n", "2 'STON/O2. 3101282' 7.9250 '' S \n", "3 113803 53.1000 C123 S \n", "4 373450 8.0500 '' S " ], "text/html": [ "\n", "
| \n", " | has_survived | \n", "passenger_class | \n", "surname | \n", "name | \n", "sex | \n", "age | \n", "sibsp | \n", "parch | \n", "ticket | \n", "fare | \n", "cabin | \n", "embarked | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "0 | \n", "3 | \n", "'Braund' | \n", "'Mr. Owen Harris' | \n", "male | \n", "22 | \n", "1 | \n", "0 | \n", "'A/5 21171' | \n", "7.2500 | \n", "'' | \n", "S | \n", "
| 1 | \n", "1 | \n", "1 | \n", "'Cumings' | \n", "'Mrs. John Bradley (Florence Briggs Thayer)' | \n", "female | \n", "38 | \n", "1 | \n", "0 | \n", "'PC 17599' | \n", "71.2833 | \n", "C85 | \n", "C | \n", "
| 2 | \n", "1 | \n", "3 | \n", "'Heikkinen' | \n", "'Miss. Laina' | \n", "female | \n", "26 | \n", "0 | \n", "0 | \n", "'STON/O2. 3101282' | \n", "7.9250 | \n", "'' | \n", "S | \n", "
| 3 | \n", "1 | \n", "1 | \n", "'Futrelle' | \n", "'Mrs. Jacques Heath (Lily May Peel)' | \n", "female | \n", "35 | \n", "1 | \n", "0 | \n", "113803 | \n", "53.1000 | \n", "C123 | \n", "S | \n", "
| 4 | \n", "0 | \n", "3 | \n", "'Allen' | \n", "'Mr. William Henry' | \n", "male | \n", "35 | \n", "0 | \n", "0 | \n", "373450 | \n", "8.0500 | \n", "'' | \n", "S | \n", "