{ "cells": [ { "cell_type": "code", "id": "5b178466-559f-47ed-bcd1-a171641d47b5", "metadata": {}, "source": [ "import os\n", "\n", "import hydra\n", "import numpy as np\n", "import omegaconf\n", "import torch\n", "import transformers\n", "from sklearn.metrics import f1_score, matthews_corrcoef, precision_score, recall_score\n", "from tqdm.auto import tqdm\n", "\n", "import classifier\n", "import dataloader" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "08301e02-d279-426f-8aad-c23eea8fb120", "metadata": {}, "source": [ "omegaconf.OmegaConf.register_new_resolver(\n", " 'cwd', os.getcwd)\n", "omegaconf.OmegaConf.register_new_resolver(\n", " 'device_count', torch.cuda.device_count)\n", "omegaconf.OmegaConf.register_new_resolver(\n", " 'eval', eval)\n", "omegaconf.OmegaConf.register_new_resolver(\n", " 'div_up', lambda x, y: (x + y - 1) // y)\n", "omegaconf.OmegaConf.register_new_resolver(\n", " 'if_then_else',\n", " lambda condition, x, y: x if condition else y\n", ")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "4685c167-63c8-4912-81e0-4ecd635fcc24", "metadata": {}, "source": [ "# Load classifier\n", "with hydra.initialize(version_base=None, config_path='../configs/'):\n", " classifier_config = hydra.compose(\n", " config_name='config',\n", " overrides=[\n", " 'hydra.output_subdir=null',\n", " f\"hydra.run.dir={os.path.dirname(os.getcwd())}/outputs/ten_species/eval_classifier/hyenadna-small-32k_from-scratch_nlayer-8\",\n", " 'hydra/job_logging=disabled',\n", " 'hydra/hydra_logging=disabled',\n", " '+is_eval_classifier=True',\n", " 'mode=train_classifier',\n", " 'loader.global_batch_size=32',\n", " 'loader.eval_global_batch_size=64',\n", " 'loader.batch_size=1',\n", " 'loader.eval_batch_size=1',\n", " 'data=ten_species',\n", " 'data.label_col=species_label',\n", " 'data.num_classes=10',\n", " 'classifier_model=hyenadna-classifier',\n", " 'classifier_model.hyena_model_name_or_path=LongSafari/hyenadna-small-32k-seqlen-hf',\n", " 'classifier_model.n_layer=8',\n", " 'classifier_backbone=hyenadna',\n", " 'model.length=32768',\n", " 'diffusion=null',\n", " 'T=null',\n", " f\"eval.checkpoint_path={os.path.dirname(os.getcwd())}/outputs/ten_species/eval_classifier/hyenadna-small-32k_from-scratch_nlayer-8/checkpoints/best.ckpt\",\n", " ]\n", " )\n", "classifier_config = omegaconf.OmegaConf.create(classifier_config)\n", "tokenizer = transformers.AutoTokenizer.from_pretrained(classifier_config.data.tokenizer_name_or_path, trust_remote_code=True)\n", "pretrained_classifier = classifier.Classifier.load_from_checkpoint(\n", " classifier_config.eval.checkpoint_path,\n", " tokenizer=tokenizer,\n", " config=classifier_config, logger=False)\n", "pretrained_classifier.eval();" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "bf18720b-64a9-4e9e-9e1e-2aa1c12dc6f0", "metadata": {}, "source": [ "tokenizer = dataloader.get_tokenizer(classifier_config)\n", "_, val_dl = dataloader.get_dataloaders(\n", " classifier_config, tokenizer, skip_train=True, valid_seed=classifier_config.seed)" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "bdcd3ba7-e26a-4e36-a5fb-ff1fb747cc3c", "metadata": {}, "source": [ "labels = []\n", "preds = []\n", "for batch in tqdm(val_dl):\n", " preds.append(\n", " pretrained_classifier(batch['input_ids'].to(pretrained_classifier.device)).argmax(dim=-1).detach().to(\n", " 'cpu', non_blocking=True).numpy()\n", " )\n", " labels.append(batch['species_label'].numpy())" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "110ed75e-613c-4b6a-bb79-15517988735c", "metadata": {}, "source": [ "labels = np.concatenate(labels)\n", "preds = np.concatenate(preds)" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "1558ca2e-6454-4c8c-b141-fca77f0025c5", "metadata": {}, "source": [ "overall_accuracy_score = (preds == labels).sum() / preds.size\n", "overall_f1_score = f1_score(y_pred=preds, y_true=labels, average=\"macro\", labels=list(range(classifier_config.data.num_classes)))\n", "overall_mcc_score = matthews_corrcoef(y_pred=preds, y_true=labels)\n", "\n", "print(f\"Overall Acc: {overall_accuracy_score:0.3f}\")\n", "print(f\"Overall F1: {overall_f1_score:0.3f}\")\n", "print(f\"Overall MCC: {overall_mcc_score:0.3f}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "df8ce828-f6e1-4167-bae2-db4f13900758", "metadata": {}, "source": [ "f1_scores = f1_score(y_pred=preds, y_true=labels, average=None , labels=list(range(classifier_config.data.num_classes)))\n", "precision_scores = precision_score(y_pred=preds, y_true=labels, average=None , labels=list(range(classifier_config.data.num_classes)))\n", "recall_scores = recall_score(y_pred=preds, y_true=labels, average=None , labels=list(range(classifier_config.data.num_classes)))\n", "\n", "species_list = ['Homo_sapiens', 'Mus_musculus', 'Drosophila_melanogaster', 'Danio_rerio',\n", " 'Caenorhabditis_elegans', 'Gallus_gallus', 'Gorilla_gorilla', 'Felis_catus',\n", " 'Salmo_trutta', 'Arabidopsis_thaliana']\n", "for s in range(classifier_config.data.num_classes):\n", " print(f\"Class {s} - {species_list[s]}:\")\n", " print(f\" F1: {f1_scores[s]:0.3f}\")\n", " print(f\" Precision: {precision_scores[s]:0.3f}\")\n", " print(f\" Recall: {recall_scores[s]:0.3f}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "d18ca7cc-4fe6-4ba9-9175-1eac9ebca7b1", "metadata": {}, "source": [], "outputs": [], "execution_count": null } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.18" } }, "nbformat": 4, "nbformat_minor": 5 }