Included project files

S0_PrepareDataset.py

@@ -0,0 +1,109 @@
+# Download files
+from zipfile import ZipFile
+import tarfile
+import requests
+from tqdm import tqdm
+import os
+from joblib import Parallel, delayed
+
+def listdir_full(path: str) -> list[str]:
+    return [f"{path}/{p}" for p in os.listdir(path)]
+
+def download_file(url, pos):
+    local_filename = f"data/{url.split('/')[-1]}"
+    if not os.path.exists(local_filename):
+        with requests.get(url, stream=True) as r:
+            r.raise_for_status()
+
+            total_size = int(r.headers.get("content-length", 0))
+            block_size = 8192
+            with tqdm(total=total_size,unit="B", unit_scale=True, position=pos, desc=f"#{pos} {local_filename}", ncols=100) as p_bar:
+                with open(local_filename, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=block_size): 
+                        p_bar.update(len(chunk))
+                        f.write(chunk)
+    return local_filename
+
+IMAGE_DIR = "data/image"
+if not os.path.isdir(IMAGE_DIR):
+    links = ["http://www.inf.ufpr.br/lesoliveira/download/macroscopic0.zip",
+             "http://www.inf.ufpr.br/lesoliveira/download/macroscopic1.tar.gz",
+             "http://www.inf.ufpr.br/lesoliveira/download/macroscopic2.tar.gz",
+             "http://www.inf.ufpr.br/lesoliveira/download/macroscopic3.tar.gz",
+             "http://www.inf.ufpr.br/lesoliveira/download/macroscopic4.tar.gz",
+    ]
+    archives : list[str] = Parallel(-1)(delayed(download_file)(l, i) for i , l in enumerate(links))
+    
+    def unzip(file: str):
+        if file.endswith(".zip"):
+            with ZipFile(file) as zip_file: zip_file.extractall(IMAGE_DIR)
+        if file.endswith(".tar.gz"):
+            with tarfile.open(file, "r:gz") as tar: tar.extractall(IMAGE_DIR)
+    Parallel(-1)(delayed(unzip)(file) for file in archives)
+    # delete faulty images
+    [os.remove(f"{IMAGE_DIR}/{i}") for i in os.listdir(IMAGE_DIR) if i.startswith("._")]
+
+from pathlib import Path
+import shutil
+
+images = os.listdir(IMAGE_DIR)
+# Group by species
+labels = ["Aspidosperma polyneuron", "Araucaria angustifolia", "Tabebuia sp.", "Cordia goeldiana", "Cordia sp.", "Hura crepitans", "Acrocarpus fraxinifolius", "Hymenaea sp.", "Peltogyne sp.", "Hymenolobium petraeum", "Myroxylon balsamum", "Dipteryx sp.", "Machaerium sp.", "Bowdichia sp.", "Mimosa scabrella", "Cedrelinga catenaeformis", "Goupia glabra", "Ocotea porosa", "Mezilaurus itauba", "Laurus nobilis", "Bertholethia excelsa", "Cariniana estrellensis", "Couratari sp.", "Carapa guianensis", "Cedrela fissilis", "Melia azedarach", "Swietenia macrophylla", "Brosimum paraense", "Bagassa guianensis", "Virola surinamensis", "Eucalyptus sp.", "Pinus sp.", "Podocarpus lambertii", "Grevilea robusta", "Balfourodendron riedelianum", "Euxylophora paraensis", "Micropholis venulosa", "Pouteria pachycarpa", "Manilkara huberi", "Erisma uncinatum", "Vochysia sp."]
+def group_label(id:int, label:str):
+    id = f"{id+1:02d}"
+    class_dir = f"{IMAGE_DIR}/{label}"
+    Path(class_dir).mkdir(parents=True, exist_ok=True)
+    
+    imgs = [im for im in images if im.startswith(id)]
+    [shutil.move(f"{IMAGE_DIR}/{im}",f"{class_dir}/{im}") for im in imgs]
+
+Parallel(-1)(delayed(group_label)(i,l) for i,l in enumerate(labels))
+
+# Train, Test Split
+train_dir = f"{IMAGE_DIR}/train"
+test_dir = f"{IMAGE_DIR}/test"
+test_full_dir = f"{IMAGE_DIR}/test_full"
+
+Path(train_dir).mkdir(parents=True, exist_ok=True)
+Path(test_dir).mkdir(parents=True, exist_ok=True)
+Path(test_full_dir).mkdir(parents=True, exist_ok=True)
+
+train_ratio = 0.9
+dirs = os.listdir(IMAGE_DIR)
+
+def train_test_split(dir:str):
+    imgs = os.listdir(f"{IMAGE_DIR}/{dir}")
+    split_index = int(len(imgs) * train_ratio)
+    train, test = imgs[:split_index], imgs[split_index:]
+    Path(f"{train_dir}/{dir}").mkdir(parents=True, exist_ok=True)
+    Path(f"{test_dir}/{dir}").mkdir(parents=True, exist_ok=True)
+    Path(f"{test_full_dir}/{dir}").mkdir(parents=True, exist_ok=True)
+
+    [shutil.move(f"{IMAGE_DIR}/{dir}/{t}",f"{train_dir}/{dir}/{t}") for t in train]
+    [shutil.copy(f"{IMAGE_DIR}/{dir}/{t}",f"{test_full_dir}/{dir}/{t}") for t in test]
+    [shutil.move(f"{IMAGE_DIR}/{dir}/{t}",f"{test_dir}/{dir}/{t}") for t in test]
+    shutil.rmtree(f"{IMAGE_DIR}/{dir}")
+
+[train_test_split(d) for d in dirs if d not in ["test", "train", "test_full"]]
+
+# Split to patches
+import cv2
+import patchify
+
+L = 816
+# tiles = [im[x:x+M,y:y+N] for x in range(0,im.shape[0],M) for y in range(0,im.shape[1],N)]
+dirs = os.listdir(train_dir)
+def patch_dir(dir):
+    for img in os.listdir(dir):
+        path = f"{dir}/{img}"
+        img = cv2.imread(f"{dir}/{img}")
+        os.remove(path)
+        patches = patchify.patchify(img,(L,L,3),L)
+        w,h,_ = patches.shape[:3]
+        patches = patches.reshape(w*h,*patches.shape[3:])
+        path, ext = os.path.splitext(path)
+        for i, p in enumerate(patches):
+            cv2.imwrite(f"{path}_{i}{ext}",p)
+
+Parallel(-1)(delayed(patch_dir)(d) for d in listdir_full(train_dir))
+Parallel(-1)(delayed(patch_dir)(d) for d in listdir_full(test_dir ))

S2_TimberDataset.py

@@ -0,0 +1,78 @@
+import torch
+from torch import Tensor
+from torchvision import transforms
+import cv2
+from torch.utils.data import Dataset, DataLoader
+import os
+from PIL import Image
+import pandas as pd
+# 3264 x 2448
+
+DATA_DIR = "data/image/train"
+labels = ["Aspidosperma polyneuron", "Araucaria angustifolia", "Tabebuia sp.", "Cordia goeldiana", "Cordia sp.", "Hura crepitans", "Acrocarpus fraxinifolius", "Hymenaea sp.", "Peltogyne sp.", "Hymenolobium petraeum", "Myroxylon balsamum", "Dipteryx sp.", "Machaerium sp.", "Bowdichia sp.", "Mimosa scabrella", "Cedrelinga catenaeformis", "Goupia glabra", "Ocotea porosa", "Mezilaurus itauba", "Laurus nobilis", "Bertholethia excelsa", "Cariniana estrellensis", "Couratari sp.", "Carapa guianensis", "Cedrela fissilis", "Melia azedarach", "Swietenia macrophylla", "Brosimum paraense", "Bagassa guianensis", "Virola surinamensis", "Eucalyptus sp.", "Pinus sp.", "Podocarpus lambertii", "Grevilea robusta", "Balfourodendron riedelianum", "Euxylophora paraensis", "Micropholis venulosa", "Pouteria pachycarpa", "Manilkara huberi", "Erisma uncinatum", "Vochysia sp."]
+label2id = {label:id for id, label in enumerate(labels)}
+
+def compile_image_df(data_dir:str, split_at = 0.9)-> pd.DataFrame:
+    dirs = os.listdir(data_dir)
+    columns=['Image_ID','Species']
+    train = pd.DataFrame(columns=columns)
+    val = pd.DataFrame(columns=columns)
+    for dir in dirs:
+        imgs = [(f"{data_dir}/{dir}/{img}", dir) for img in list(os.listdir(f"{data_dir}/{dir}"))]
+        length = len(imgs)
+        train_count = int(length * split_at)
+        train = pd.concat([train, pd.DataFrame(imgs[:train_count],columns=columns)])
+        val = pd.concat([val, pd.DataFrame(imgs[train_count:],columns=columns)])
+
+    return train, val
+
+class TimberDataset(Dataset):
+    def __init__(self, 
+                 dataframe: pd.DataFrame, 
+                 is_train=False, 
+                 transform=None,
+                 device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')) -> None:
+        super().__init__()
+        self.dataframe = dataframe
+        self.is_train = is_train
+        self.transform = transform
+        self.device = device
+
+    def __len__(self) -> int:
+        return len(self.dataframe)
+
+    def __getitem__(self, idx: list[int]|Tensor):
+        if torch.is_tensor(idx): 
+            idx = idx.tolist()
+        
+        img_name = os.path.join(self.dataframe.iloc[idx,0])
+        image = cv2.imread(img_name)
+        image = Image.fromarray(image)
+
+        label = self.dataframe.iloc[idx,1]
+        label = label2id[label]
+        label = torch.tensor(int(label))
+
+        if self.transform:
+            image = self.transform(image)
+        return image.to(self.device), label.to(self.device)
+
+def build_dataloader(
+        train_ratio = 0.9,
+        img_size = (640,640),
+        batch_size = 12,
+    ) -> tuple[DataLoader,DataLoader]:
+    train_df, val_df = compile_image_df(DATA_DIR, split_at=train_ratio)
+
+    transform = transforms.Compose([
+        transforms.Resize(img_size),
+        transforms.ToTensor(),
+    ])
+
+    train_loader = DataLoader(TimberDataset(train_df, is_train=True,transform=transform),
+                              shuffle=True,
+                              batch_size=batch_size)
+    val_loader = DataLoader(TimberDataset(val_df, is_train=True,transform=transform),
+                            batch_size=batch_size)
+    
+    return train_loader,val_loader

S3_intermediateDataset.py

@@ -0,0 +1,85 @@
+import os
+import torch
+from time import time
+from torch import Tensor
+from tqdm import tqdm
+from torch.utils.data import Dataset, DataLoader
+from S2_TimberDataset import build_dataloader
+from typing import Callable
+import pandas as pd
+import numpy as np
+# 3264 x 2448
+
+def write_random_lowercase(n):
+    min_lc = ord(b'a')
+    len_lc = 26
+    ba = bytearray(os.urandom(n))
+    for i, b in enumerate(ba):
+        ba[i] = min_lc + b % len_lc # convert 0..255 to 97..122
+    return ba.decode("utf-8")
+
+INTERMEDIATE_DIR = "data/intermediate"
+
+class IntermediateDataset(Dataset):
+    def __init__(self,
+                 name,
+                 device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')) -> None:
+        super().__init__()
+        self.name = name
+        self.device = device
+        files = os.listdir(f"{INTERMEDIATE_DIR}/{self.name}")
+        self.tensors = np.array([f for f in files if os.path.splitext(f)[-1] == ".pt"])
+        self.labels  = np.array([f for f in files if os.path.splitext(f)[-1] == ".txt"])
+
+    def __len__(self) -> int:
+        return len(self.tensors)
+
+    def __getitem__(self, idx: list[int]|Tensor):
+        tensor = self.tensors[idx]
+        labels = self.labels[idx]
+        images = torch.load(f"{INTERMEDIATE_DIR}/{self.name}/{tensor}")
+        
+        with open(f"{INTERMEDIATE_DIR}/{self.name}/{labels}", 'r') as f: 
+            labels = f.readline().split("-")
+            labels = Tensor(list(map(int, labels)))
+
+        images = images.to(self.device) 
+        labels = labels.to(device=self.device, dtype=torch.int64)
+
+        return images, labels
+
+    @staticmethod
+    def prepare_intermediate_dataset(pred: Callable, name: str, dataset: DataLoader, iterations = 1) -> None:
+        with torch.no_grad():
+            for _ in range(iterations):
+                for images, labels in tqdm(dataset):
+                    out = pred(images)
+                    labels = np.char.mod('%d', labels.cpu().numpy())
+                    labels = '-'.join(labels)
+
+                    file_name = f"{INTERMEDIATE_DIR}/{name}/{int(time())}_{write_random_lowercase(10)}"
+                    torch.save(out,f"{file_name}.pt")
+                    with open(f"{file_name}.txt", 'w') as f:
+                        f.write(labels)
+
+def build_intermediate_dataset_if_not_exists(pred_:Callable, name:str, dataset:DataLoader) -> None:
+    try: os.mkdir(INTERMEDIATE_DIR)
+    except: pass
+    try: os.mkdir(f"{INTERMEDIATE_DIR}/{name}")
+    except: pass
+
+    if os.listdir(f"{INTERMEDIATE_DIR}/{name}") == []:
+        IntermediateDataset.prepare_intermediate_dataset(pred_, name, dataset)
+
+def intermediate_dataset(name:str) -> DataLoader:
+    return DataLoader(IntermediateDataset(name=name),batch_size=1)
+
+if __name__ == '__main__':
+    train, val, test = build_dataloader(train_ratio= 0.01)
+    
+    build_intermediate_dataset_if_not_exists(lambda x:x, "testing", train)
+
+    train_loader = DataLoader(IntermediateDataset("testing"),batch_size=1)
+    (i1,i2,i3), val = next(iter(train_loader))
+    "a"
+    

S4_Training.py

@@ -0,0 +1,89 @@
+from collections import Counter
+from tqdm import tqdm
+from time import time
+import torch
+from torch import nn, Tensor
+from S3_intermediateDataset import build_intermediate_dataset_if_not_exists, intermediate_dataset
+from S2_TimberDataset import build_dataloader
+from S1_CNN_Model import build_model
+
+if __name__ == '__main__':
+    img_size = (320,320)
+    train_loader, val_loader = build_dataloader(
+        # train_ratio= 0.005,
+        img_size=img_size,
+        batch_size=16,
+    )
+    
+    build_intermediate_dataset_if_not_exists(lambda x:x, "train", train_loader)
+    build_intermediate_dataset_if_not_exists(lambda x:x, "val", val_loader)
+    
+    train_loader = intermediate_dataset("train") 
+    val_loader = intermediate_dataset("val") 
+
+    model = build_model(img_size=img_size)
+    model.train()
+
+    criterion = nn.CrossEntropyLoss()
+    optimizer = torch.optim.Adam(model.parameters(), lr=5e-5)
+
+    accuracies = []
+
+    n_epoch = 40
+    timer = time()
+    pbar_0 = tqdm(range(n_epoch), position=0, ncols=100)
+    pbar_0.set_description(f"epoch 1/{n_epoch}")
+    img = None
+    for epoch in pbar_0:
+        pbar_1 = tqdm(enumerate(train_loader), total=len(train_loader), position=1, ncols=100, leave=False)
+        for i, (images, labels) in pbar_1:
+            # # Reshape
+            images = images.reshape(images.shape[1:])
+            labels = labels.reshape(labels.shape[1:])
+            # Forward
+            out = model.forward(images)
+            loss = criterion.forward(out, labels)
+
+            # Backward
+            optimizer.zero_grad()
+            loss.backward()
+            
+            optimizer.step()
+
+            if i % 10 == 0:
+                pbar_1.set_description(f"  loss = {loss.item():.4f} ({(time() - timer)*1000:.4f} ms)")
+                timer = time()
+        
+        n_correct = 0
+        n_samples = 0
+        pbar_0.set_description(f"epoch {epoch+1}/{n_epoch}, Validating . . .")
+
+        with torch.no_grad(): 
+            with tqdm(bar_format='{desc}{postfix}', position=1, leave=False) as val_desc:
+                tally = Counter()
+                for images, labels in tqdm(val_loader, position=2, ncols=100, leave=False):
+                    # # Reshape
+                    images = images.reshape(images.shape[1:])
+                    labels = labels.reshape(labels.shape[1:])
+
+                    x = model.forward(images)
+
+                    _, predictions = torch.max(x,1)
+                    tally += Counter(predictions.tolist())
+                    n_samples += labels.shape[0]
+                    n_correct += (predictions == labels).sum().item()
+                    tally_desc = ' '.join([f"{n}:{c}" for n,c in tally.most_common()])[:80] + "..."
+                    val_desc.set_description(f"{n_correct}/{n_samples} correct")
+                    val_desc.set_postfix_str(tally_desc)
+                
+                accuracy = f"{n_correct/n_samples * 100:.2f}%"
+                pbar_0.set_description(f"epoch {epoch+2}/{n_epoch}, accuracy: {accuracy}")
+                
+                if len(accuracies) >= 3 and accuracy > max(accuracies):
+                    torch.save(model,f"model_{epoch}.pt")
+
+                accuracies.append(accuracy)
+
+    torch.save(model,"model.pt")
+    
+    print(accuracies)

S5_Evaluation.ipynb

@@ -0,0 +1,209 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import os\n",
+    "from tqdm import tqdm\n",
+    "from S1_CNN_Model import CNN_Model\n",
+    "from S2_TimberDataset import TimberDataset, compile_image_df\n",
+    "from S3_intermediateDataset import build_intermediate_dataset_if_not_exists, intermediate_dataset\n",
+    "\n",
+    "from torch.utils.data import DataLoader\n",
+    "from torchvision import transforms"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_df, _ = compile_image_df(\"data/image/test\", split_at=1.0)\n",
+    "\n",
+    "def listdir_full(path: str) -> list[str]:\n",
+    "    return [f\"{path}/{p}\" for p in os.listdir(path)]\n",
+    "\n",
+    "transform = transforms.Compose([\n",
+    "    transforms.Resize((320,320)),\n",
+    "    transforms.ToTensor(),\n",
+    "])\n",
+    "\n",
+    "test_loader = DataLoader(TimberDataset(test_df, is_train=True,transform=transform),\n",
+    "                            shuffle=True,\n",
+    "                            batch_size=12)\n",
+    "\n",
+    "build_intermediate_dataset_if_not_exists(lambda x:x, \"test\", test_loader)\n",
+    "test_loader = intermediate_dataset(\"test\") "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 17/17 [09:00<00:00, 31.82s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('ckpt/model_37.pt', 92.36111111111111), ('ckpt/model_36.pt', 91.39957264957265), ('ckpt/model.pt', 90.11752136752136), ('ckpt/model_23.pt', 90.11752136752136), ('ckpt/model_27.pt', 90.03739316239316), ('ckpt/model_19.pt', 89.95726495726495), ('ckpt/model_17.pt', 88.56837606837607), ('ckpt/model_16.pt', 87.79380341880342), ('ckpt/model_15.pt', 85.79059829059828), ('ckpt/model_13.pt', 85.04273504273505), ('ckpt/model_11.pt', 85.01602564102564), ('ckpt/model_14.pt', 84.58867521367522), ('ckpt/model_7.pt', 79.8076923076923), ('ckpt/model_6.pt', 79.27350427350427), ('ckpt/model_5.pt', 75.58760683760684), ('ckpt/model_4.pt', 70.86004273504274), ('ckpt/model_3.pt', 63.67521367521367)]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "def evaluate_model(model: CNN_Model):\n",
+    "    n_correct = 0\n",
+    "    n_samples = 0\n",
+    "\n",
+    "    with torch.no_grad(): \n",
+    "        with tqdm(test_loader, position=1, leave=False) as pb:\n",
+    "            for images, labels in pb:\n",
+    "                images = images.reshape(images.shape[1:])\n",
+    "                labels = labels.reshape(labels.shape[1:])\n",
+    "\n",
+    "                x = model.forward(images)\n",
+    "                _, predictions = torch.max(x,1)\n",
+    "                \n",
+    "                n_samples += labels.shape[0]\n",
+    "                n_correct += (predictions == labels).sum().item()\n",
+    "\n",
+    "                pb.set_description(f\"{n_correct}/{n_samples} correct predictions ({n_correct/n_samples*100 :.2f}%)\")\n",
+    "\n",
+    "    return n_correct/n_samples*100\n",
+    "\n",
+    "accuracies = [(model_path, evaluate_model(torch.load(model_path))) for model_path in tqdm(listdir_full(\"ckpt\"), position=0)]\n",
+    "accuracies.sort(key = lambda x : x[1], reverse=True)\n",
+    "accuracies"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_loader = DataLoader(TimberDataset(test_df, is_train=True,transform=transform),\n",
+    "                            batch_size=12)\n",
+    "\n",
+    "build_intermediate_dataset_if_not_exists(lambda x:x, \"test_full\", test_loader)\n",
+    "test_loader = intermediate_dataset(\"test_full\") "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 17/17 [08:32<00:00, 30.14s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('ckpt/model_37.pt', 97.11538461538461), ('ckpt/model_23.pt', 96.15384615384616), ('ckpt/model_36.pt', 96.15384615384616), ('ckpt/model_27.pt', 95.51282051282051), ('ckpt/model_19.pt', 95.1923076923077), ('ckpt/model.pt', 94.23076923076923), ('ckpt/model_17.pt', 94.23076923076923), ('ckpt/model_16.pt', 93.26923076923077), ('ckpt/model_15.pt', 92.94871794871796), ('ckpt/model_11.pt', 91.98717948717949), ('ckpt/model_13.pt', 91.34615384615384), ('ckpt/model_14.pt', 91.02564102564102), ('ckpt/model_7.pt', 88.78205128205127), ('ckpt/model_6.pt', 86.85897435897436), ('ckpt/model_5.pt', 82.05128205128204), ('ckpt/model_4.pt', 78.84615384615384), ('ckpt/model_3.pt', 70.83333333333334)]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "def full_images_evaluation(model: CNN_Model):\n",
+    "    model.image_size = (320,320)\n",
+    "    n_correct = 0\n",
+    "    n_samples = 0\n",
+    "\n",
+    "    with torch.no_grad(): \n",
+    "        with tqdm(test_loader, position=1, leave = False) as pb:\n",
+    "            for images, labels in pb:\n",
+    "                images = images.reshape(images.shape[1:])\n",
+    "                labels = labels.reshape(labels.shape[1:])\n",
+    "\n",
+    "                assert torch.all(labels == labels[0]).item()\n",
+    "                label = labels[0]\n",
+    "\n",
+    "                x = model.forward(images)\n",
+    "                _, preds = torch.max(x,1)\n",
+    "                pred = torch.mode(preds,0).values\n",
+    "                \n",
+    "                n_samples += 1\n",
+    "                n_correct += (pred == label).item()\n",
+    "\n",
+    "                pb.set_description(f\"{n_correct}/{n_samples} correct predictions ({n_correct/n_samples*100 :.2f}%)\")\n",
+    "\n",
+    "    return n_correct/n_samples*100\n",
+    "\n",
+    "full_accuracies = [(model_path, full_images_evaluation(torch.load(model_path)))\n",
+    "                   for model_path in tqdm(listdir_full(\"ckpt\"), position=0)]\n",
+    "full_accuracies.sort(key = lambda x : x[1], reverse=True)\n",
+    "print(full_accuracies)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<zip at 0x286ec807c00>"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "fyp",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -9,3 +9,6 @@ patchify
 pillow
 torch
 torchvision
+tqdm
+requests
+joblib