import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet18, resnet50  # Example models

# Define the teacher (weak model) and student (strong model)
teacher_model = resnet18(pretrained=True)
student_model = resnet50(pretrained=True)

# Custom Adaptive Confidence Distillation Loss
class AdaptiveConfidenceDistillationLoss(nn.Module):
    def __init__(self, temperature=1.0):
        super().__init__()
        self.temperature = temperature
        self.ce_loss = nn.CrossEntropyLoss()

    def forward(self, student_logits, teacher_logits, targets):
        # Soften probabilities
        soft_teacher_probs = F.softmax(teacher_logits / self.temperature, dim=1)
        soft_student_probs = F.log_softmax(student_logits / self.temperature, dim=1)

        # Calculate the KL Divergence for the soft targets
        distillation_loss = F.kl_div(
            soft_student_probs, soft_teacher_probs, reduction="batchmean"
        )

        # Calculate the standard loss with hard targets
        hard_loss = self.ce_loss(student_logits, targets)

        # Adaptive weighting could be implemented here based on confidence
        # For simplicity, this example uses a fixed ratio
        return distillation_loss + hard_loss

# Example Training Loop Skeleton
def train_model(student_model, teacher_model, dataloader, optimizer, device):
    student_model.train()
    teacher_model.eval()  # Teacher model should be in eval mode

    loss_fn = AdaptiveConfidenceDistillationLoss(temperature=2.0)

    for inputs, labels in dataloader:
        inputs, labels = inputs.to(device), labels.to(device)

        optimizer.zero_grad()

        with torch.no_grad():
            teacher_outputs = teacher_model(inputs)

        student_outputs = student_model(inputs)

        loss = loss_fn(student_outputs, teacher_outputs, labels)
        loss.backward()
        optimizer.step()

# Example usage
# Assuming `dataloader`, `optimizer`, and `device` are defined
# train_model(student_model, teacher_model, dataloader, optimizer, device)


AdaptiveConfidenceDistillationLoss

import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet18, resnet50  # Example models

# Define the teacher (weak model) and student (strong model)
teacher_model = resnet18(pretrained=True)
student_model = resnet50(pretrained=True)

# Custom Adaptive Confidence Distillation Loss
class AdaptiveConfidenceDistillationLoss(nn.Module):
    def __init__(self, temperature=1.0):
        super().__init__()
        self.temperature = temperature
        self.ce_loss = nn.CrossEntropyLoss()

    def forward(self, student_logits, teacher_logits, targets):
        # Soften probabilities
        soft_teacher_probs = F.softmax(teacher_logits / self.temperature, dim=1)
        soft_student_probs = F.log_softmax(student_logits / self.temperature, dim=1)

        # Calculate the KL Divergence for the soft targets
        distillation_loss = F.kl_div(
            soft_student_probs, soft_teacher_probs, reduction="batchmean"
        )

        # Calculate the standard loss with hard targets
        hard_loss = self.ce_loss(student_logits, targets)

        # Adaptive weighting could be implemented here based on confidence
        # For simplicity, this example uses a fixed ratio
        return distillation_loss + hard_loss

# Example Training Loop Skeleton
def train_model(student_model, teacher_model, dataloader, optimizer, device):
    student_model.train()
    teacher_model.eval()  # Teacher model should be in eval mode

    loss_fn = AdaptiveConfidenceDistillationLoss(temperature=2.0)

    for inputs, labels in dataloader:
        inputs, labels = inputs.to(device), labels.to(device)

        optimizer.zero_grad()

        with torch.no_grad():
            teacher_outputs = teacher_model(inputs)

        student_outputs = student_model(inputs)

        loss = loss_fn(student_outputs, teacher_outputs, labels)


    def forward(self, student_logits, teacher_logits, targets):
        # Soften probabilities
        soft_teacher_probs = F.softmax(teacher_logits / self.temperature, dim=1)
        soft_student_probs = F.log_softmax(student_logits / self.temperature, dim=1)

        # Calculate the KL Divergence for the soft targets
        distillation_loss = F.kl_div(
            soft_student_probs, soft_teacher_probs, reduction="batchmean"
        )

        # Calculate the standard loss with hard targets
        hard_loss = self.ce_loss(student_logits, targets)

        # Adaptive weighting could be implemented here based on confidence
        # For simplicity, this example uses a fixed ratio
        return distillation_loss + hard_loss

# Example Training Loop Skeleton
def train_model(student_model, teacher_model, dataloader, optimizer, device):
    student_model.train()
    teacher_model.eval()  # Teacher model should be in eval mode

    loss_fn = AdaptiveConfidenceDistillationLoss(temperature=2.0)

    for inputs, labels in dataloader:
        inputs, labels = inputs.to(device), labels.to(device)

        optimizer.zero_grad()

        with torch.no_grad():
            teacher_outputs = teacher_model(inputs)

        student_outputs = student_model(inputs)

        loss = loss_fn(student_outputs, teacher_outputs, labels)
        loss.backward()
        optimizer.step()

# Example usage
# Assuming `dataloader`, `optimizer`, and `device` are defined
# train_model(student_model, teacher_model, dataloader, optimizer, device)


import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet18, resnet50  # Example models

# Define the teacher (weak model) and student (strong model)
teacher_model = resnet18(pretrained=True)
student_model = resnet50(pretrained=True)

# Custom Adaptive Confidence Distillation Loss
class AdaptiveConfidenceDistillationLoss(nn.Module):
    def __init__(self, temperature=1.0):
        super().__init__()
        self.temperature = temperature
        self.ce_loss = nn.CrossEntropyLoss()

    def forward(self, student_logits, teacher_logits, targets):
        # Soften probabilities
        soft_teacher_probs = F.softmax(teacher_logits / self.temperature, dim=1)
        soft_student_probs = F.log_softmax(student_logits / self.temperature, dim=1)

        # Calculate the KL Divergence for the soft targets
        distillation_loss = F.kl_div(
            soft_student_probs, soft_teacher_probs, reduction="batchmean"
        )

        # Calculate the standard loss with hard targets
        hard_loss = self.ce_loss(student_logits, targets)

        # Adaptive weighting could be implemented here based on confidence
        # For simplicity, this example uses a fixed ratio
        return distillation_loss + hard_loss

# Example Training Loop Skeleton


AdaptiveConfidenceDistillation

import torch.nn as nn
import torch.nn.functional as F

# Custom Adaptive Confidence Distillation Loss
class AdaptiveConfidenceDistillationLoss(nn.Module):
    def __init__(self, temperature=1.0):
        super().__init__()
        self.temperature = temperature
        self.ce_loss = nn.CrossEntropyLoss()

    def forward(self, student_logits, teacher_logits, targets):
        # Soften probabilities
        soft_teacher_probs = F.softmax(teacher_logits / self.temperature, dim=1)
        soft_student_probs = F.log_softmax(student_logits / self.temperature, dim=1)

        # Calculate the KL Divergence for the soft targets
        distillation_loss = F.kl_div(
            soft_student_probs, soft_teacher_probs, reduction="batchmean"
        )

        # Calculate the standard loss with hard targets
        hard_loss = self.ce_loss(student_logits, targets)

        # Adaptive weighting could be implemented here based on confidence
        # For simplicity, this example uses a fixed ratio
        return distillation_loss + hard_loss


import torch
import torch.nn as nn
import torch.optim as optim
from torchvision.models import resnet18, resnet50
from torch.utils.data import DataLoader, TensorDataset
import numpy as np
from train_example import train_model

# Define the teacher (weak model) and student (strong model)
teacher_model = resnet18(pretrained=False)  # Using pretrained=False for simplicity
student_model = resnet50(pretrained=False)

# Define a simple synthetic dataset resembling CIFAR-100
# CIFAR-100 has 32x32 images with 3 channels
# We'll create random data with similar dimensions
num_samples = 1000
input_channels = 3
input_height = 32
input_width = 32
num_classes = 100

# Generate random images and labels
images = torch.randn(num_samples, input_channels, input_height, input_width)
labels = torch.randint(0, num_classes, (num_samples,))

# Create DataLoader for the synthetic dataset
dataset = TensorDataset(images, labels)
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

# Define optimizer and device
optimizer = optim.Adam(student_model.parameters(), lr=0.001)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Define a unit test function
def test_training_loop():
    train_model(student_model, teacher_model, dataloader, optimizer, device)

# Run the test function
test_training_loop()
print("test passed!")


num_samples = 1000
input_channels = 3
input_height = 32
input_width = 32
num_classes = 100

# Generate random images and labels
images = torch.randn(num_samples, input_channels, input_height, input_width)
labels = torch.randint(0, num_classes, (num_samples,))

# Create DataLoader for the synthetic dataset
dataset = TensorDataset(images, labels)
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

# Define optimizer and device
optimizer = optim.Adam(student_model.parameters(), lr=0.001)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Define a unit test function
def test_training_loop():
    train_model(student_model, teacher_model, dataloader, optimizer, device)

# Run the test function
test_training_loop()
print("test passed!")


dataset

import torch
from torch.utils.data import Dataset
import os
from PIL import Image
from generate_labels import generate_label_for_single_image
import json

# Custom Dataset Class
class CustomImageDataset(Dataset):
    def __init__(
        self, image_dir, transform=None, model_name="google/vit-base-patch16-224"
    ):
        self.image_dir = image_dir
        self.transform = transform
        self.memory_bank = load_memory_bank()  # Load existing labels from memory bank
        self.image_paths = [
            os.path.join(image_dir, img)
            for img in os.listdir(image_dir)
            if img.endswith((".png", ".jpg", ".jpeg"))
        ]
        self.model_name = model_name
        self.label_to_index = {}  # Dynamically map labels to indices
        self.current_index = 0  # Keep track of the next index to assign

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        image_path = self.image_paths[idx]
        image = Image.open(image_path).convert("RGB")

        if image_path not in self.memory_bank:  # Check if label already exists
            # Generate label for the image
            _, predicted_label = generate_label_for_single_image(image_path)
            self.memory_bank[image_path] = predicted_label  # Add to memory bank
        else:
            predicted_label = self.memory_bank[image_path]  # Use existing label
        # if self.transform:
        #    image = self.transform(image)

        # Dynamically assign index to new labels
        if predicted_label not in self.label_to_index:
            self.label_to_index[predicted_label] = self.current_index
            self.current_index += 1
        label_index = self.label_to_index[predicted_label]

        if self.transform:
            image = self.transform(image)
        return image, torch.tensor(label_index, dtype=torch.long)

def load_memory_bank(filename="label_memory_bank.json"):
    if os.path.exists(filename):
        with open(filename, "r") as file:
            return json.load(file)
    else:
        return {}


        if image_path not in self.memory_bank:  # Check if label already exists
            # Generate label for the image
            _, predicted_label = generate_label_for_single_image(image_path)
            self.memory_bank[image_path] = predicted_label  # Add to memory bank
        else:
            predicted_label = self.memory_bank[image_path]  # Use existing label
        # if self.transform:
        #    image = self.transform(image)

        # Dynamically assign index to new labels
        if predicted_label not in self.label_to_index:
            self.label_to_index[predicted_label] = self.current_index
            self.current_index += 1
        label_index = self.label_to_index[predicted_label]

        if self.transform:
            image = self.transform(image)
        return image, torch.tensor(label_index, dtype=torch.long)

def load_memory_bank(filename="label_memory_bank.json"):
    if os.path.exists(filename):
        with open(filename, "r") as file:
            return json.load(file)
    else:
        return {}


import torch
from torch.utils.data import Dataset
import os
from PIL import Image
from generate_labels import generate_label_for_single_image
import json

# Custom Dataset Class
class CustomImageDataset(Dataset):
    def __init__(
        self, image_dir, transform=None, model_name="google/vit-base-patch16-224"
    ):
        self.image_dir = image_dir
        self.transform = transform
        self.memory_bank = load_memory_bank()  # Load existing labels from memory bank
        self.image_paths = [
            os.path.join(image_dir, img)
            for img in os.listdir(image_dir)
            if img.endswith((".png", ".jpg", ".jpeg"))
        ]
        self.model_name = model_name
        self.label_to_index = {}  # Dynamically map labels to indices
        self.current_index = 0  # Keep track of the next index to assign

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        image_path = self.image_paths[idx]
        image = Image.open(image_path).convert("RGB")

        if image_path not in self.memory_bank:  # Check if label already exists
            # Generate label for the image
            _, predicted_label = generate_label_for_single_image(image_path)
            self.memory_bank[image_path] = predicted_label  # Add to memory bank
        else:
            predicted_label = self.memory_bank[image_path]  # Use existing label
        # if self.transform:
        #    image = self.transform(image)

        # Dynamically assign index to new labels
        if predicted_label not in self.label_to_index:
            self.label_to_index[predicted_label] = self.current_index


import json

# Custom Dataset Class
class CustomImageDataset(Dataset):
    def __init__(
        self, image_dir, transform=None, model_name="google/vit-base-patch16-224"
    ):
        self.image_dir = image_dir
        self.transform = transform
        self.memory_bank = load_memory_bank()  # Load existing labels from memory bank
        self.image_paths = [
            os.path.join(image_dir, img)
            for img in os.listdir(image_dir)
            if img.endswith((".png", ".jpg", ".jpeg"))
        ]
        self.model_name = model_name
        self.label_to_index = {}  # Dynamically map labels to indices
        self.current_index = 0  # Keep track of the next index to assign

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        image_path = self.image_paths[idx]
        image = Image.open(image_path).convert("RGB")

        if image_path not in self.memory_bank:  # Check if label already exists
            # Generate label for the image
            _, predicted_label = generate_label_for_single_image(image_path)
            self.memory_bank[image_path] = predicted_label  # Add to memory bank
        else:
            predicted_label = self.memory_bank[image_path]  # Use existing label
        # if self.transform:
        #    image = self.transform(image)

        # Dynamically assign index to new labels
        if predicted_label not in self.label_to_index:
            self.label_to_index[predicted_label] = self.current_index
            self.current_index += 1
        label_index = self.label_to_index[predicted_label]


# Custom Dataset Class
class CustomImageDataset(Dataset):
    def __init__(
        self, image_dir, transform=None, model_name="google/vit-base-patch16-224"
    ):
        self.image_dir = image_dir
        self.transform = transform
        self.memory_bank = load_memory_bank()  # Load existing labels from memory bank
        self.image_paths = [
            os.path.join(image_dir, img)
            for img in os.listdir(image_dir)
            if img.endswith((".png", ".jpg", ".jpeg"))
        ]
        self.model_name = model_name
        self.label_to_index = {}  # Dynamically map labels to indices
        self.current_index = 0  # Keep track of the next index to assign

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        image_path = self.image_paths[idx]
        image = Image.open(image_path).convert("RGB")

        if image_path not in self.memory_bank:  # Check if label already exists
            # Generate label for the image
            _, predicted_label = generate_label_for_single_image(image_path)
            self.memory_bank[image_path] = predicted_label  # Add to memory bank
        else:
            predicted_label = self.memory_bank[image_path]  # Use existing label
        # if self.transform:
        #    image = self.transform(image)

        # Dynamically assign index to new labels
        if predicted_label not in self.label_to_index:
            self.label_to_index[predicted_label] = self.current_index
            self.current_index += 1
        label_index = self.label_to_index[predicted_label]

        if self.transform:
            image = self.transform(image)
        return image, torch.tensor(label_index, dtype=torch.long)

def load_memory_bank(filename="label_memory_bank.json"):
    if os.path.exists(filename):
        with open(filename, "r") as file:
            return json.load(file)
    else:
        return {}


train

import torch
import torch.optim as optim
from torchvision import transforms
from torchvision.models import resnet18, resnet50
from torch.utils.data import DataLoader
import numpy as np
import os
import json
from utils import plot_and_save_metrics
from dataset import CustomImageDataset
from AdaptiveConfidenceDistillation import AdaptiveConfidenceDistillationLoss

# Define the teacher (weak model) and student (strong model)
teacher_model = resnet18(pretrained=True)  # Using pretrained=False for simplicity
student_model = resnet50(pretrained=True)

# Define a simple synthetic dataset resembling CIFAR-100
# CIFAR-100 has 32x32 images with 3 channels
# We'll create random data with similar dimensions
num_samples = 1000
input_channels = 3
input_height = 32
input_width = 32
num_classes = 100

# Define transformations
transform = transforms.Compose(
    [
        transforms.Resize((32, 32)),
        transforms.ToTensor(),
    ]
)

# Create an instance of the CustomImageDataset
image_dir = "./images"  # Specify the correct path to your images
dataset = CustomImageDataset(image_dir=image_dir, transform=transform)

def save_memory_bank(memory_bank, filename="label_memory_bank.json"):
    with open(filename, "w") as file:
        json.dump(memory_bank, file)

# After training is complete or when you're done using the dataset
save_memory_bank(dataset.memory_bank)  # Save the updated memory bank to a file
# Split dataset into train and validation sets
train_size = int(0.8 * len(dataset))
val_size = len(dataset) - train_size
train_dataset, val_dataset = torch.utils.data.random_split(
    dataset, [train_size, val_size]
)

# Create DataLoaders
train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)
val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False)

# Define optimizer and device
optimizer = optim.Adam(student_model.parameters(), lr=0.001)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")

# Step 1: Load Labels and Image Paths
def load_image_paths_labels(json_path):
    with open(json_path, "r") as f:
        data = json.load(f)
    image_paths = [os.path.join("./images", item["image_path"]) for item in data]
    labels = [item["caption"] for item in data]
    return image_paths, labels

# Example Training Loop Skeleton with Validation and Model Checkpointing
def train_model_with_validation(
    student_model,
    teacher_model,
    train_dataloader,
    val_dataloader,
    optimizer,
    device,
    num_epochs=10,
    checkpoint_path="checkpoint.pth",
):
    student_model.train()
    teacher_model.eval()  # Teacher model should be in eval mode

    # Initialize metrics dictionary
    metrics = {
        "train_loss": [],
        "val_loss": [],
    }  # Add 'train_accuracy': [], 'val_accuracy': [] if tracking accuracy
    loss_fn = AdaptiveConfidenceDistillationLoss(temperature=2.0)
    best_val_loss = float("inf")
    step = 0
    for epoch in range(num_epochs):
        # Training loop
        for inputs, labels in train_dataloader:
            if step % 200 == 0:
                # Validation loop
                val_loss = 0.0
                with torch.no_grad():
                    for inputs, labels in val_dataloader:
                        inputs, labels = inputs.to(device), labels.to(device)
                        teacher_outputs = teacher_model(inputs)
                        student_outputs = student_model(inputs)
                        loss = loss_fn(student_outputs, teacher_outputs, labels)
                        val_loss += loss.item() * inputs.size(
                            0
                        )  # Inside the training loop
                        metrics["train_loss"].append(
                            loss.item()
                        )  # Assuming loss is your loss variable

                        # Inside the validation loop
                        metrics["val_loss"].append(
                            val_loss
                        )  # Assuming val_loss is your accumulated validation loss
                        step += 1
                        print(f"val_loss: {val_loss}")
                # After training, plot and save metrics
                plot_and_save_metrics(metrics)
            inputs, labels = inputs.to(device), labels.to(device)
            optimizer.zero_grad()

            with torch.no_grad():
                teacher_outputs = teacher_model(inputs)

            student_outputs = student_model(inputs)

            loss = loss_fn(student_outputs, teacher_outputs, labels)
            loss.backward()
            optimizer.step()

        val_loss /= len(val_dataloader.dataset)
        print(f"Epoch [{epoch+1}/{num_epochs}], Validation Loss: {val_loss:.4f}")

        # Save the model checkpoint if validation loss improves
        if val_loss < best_val_loss:
            best_val_loss = val_loss
            torch.save(student_model.state_dict(), checkpoint_path)
            # After training, plot and save metrics
            plot_and_save_metrics(metrics)
            print(f"Saving model checkpoint at {checkpoint_path}")

# Example usage
# Assuming `train_dataloader`, `val_dataloader`, `optimizer`, and `device` are defined
train_model_with_validation(
    student_model,
    teacher_model,
    train_dataloader,
    val_dataloader,
    optimizer,
    device,
    num_epochs=10,
    checkpoint_path="checkpoint.pth",
)


# We'll create random data with similar dimensions
num_samples = 1000
input_channels = 3
input_height = 32
input_width = 32
num_classes = 100

# Define transformations
transform = transforms.Compose(
    [
        transforms.Resize((32, 32)),
        transforms.ToTensor(),
    ]
)

# Create an instance of the CustomImageDataset
image_dir = "./images"  # Specify the correct path to your images
dataset = CustomImageDataset(image_dir=image_dir, transform=transform)

def save_memory_bank(memory_bank, filename="label_memory_bank.json"):
    with open(filename, "w") as file:
        json.dump(memory_bank, file)

# After training is complete or when you're done using the dataset
save_memory_bank(dataset.memory_bank)  # Save the updated memory bank to a file
# Split dataset into train and validation sets
train_size = int(0.8 * len(dataset))
val_size = len(dataset) - train_size
train_dataset, val_dataset = torch.utils.data.random_split(
    dataset, [train_size, val_size]
)

# Create DataLoaders
train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)
val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False)

# Define optimizer and device
optimizer = optim.Adam(student_model.parameters(), lr=0.001)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")


# After training is complete or when you're done using the dataset
save_memory_bank(dataset.memory_bank)  # Save the updated memory bank to a file
# Split dataset into train and validation sets
train_size = int(0.8 * len(dataset))
val_size = len(dataset) - train_size
train_dataset, val_dataset = torch.utils.data.random_split(
    dataset, [train_size, val_size]
)

# Create DataLoaders
train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)
val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False)

# Define optimizer and device
optimizer = optim.Adam(student_model.parameters(), lr=0.001)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")

# Step 1: Load Labels and Image Paths
def load_image_paths_labels(json_path):
    with open(json_path, "r") as f:
        data = json.load(f)
    image_paths = [os.path.join("./images", item["image_path"]) for item in data]
    labels = [item["caption"] for item in data]
    return image_paths, labels

# Example Training Loop Skeleton with Validation and Model Checkpointing
def train_model_with_validation(
    student_model,
    teacher_model,
    train_dataloader,
    val_dataloader,
    optimizer,
    device,
    num_epochs=10,
    checkpoint_path="checkpoint.pth",
):
    student_model.train()
    teacher_model.eval()  # Teacher model should be in eval mode

    # Initialize metrics dictionary
    metrics = {
        "train_loss": [],
        "val_loss": [],


# Create DataLoaders
train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)
val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False)

# Define optimizer and device
optimizer = optim.Adam(student_model.parameters(), lr=0.001)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")

# Step 1: Load Labels and Image Paths
def load_image_paths_labels(json_path):
    with open(json_path, "r") as f:
        data = json.load(f)
    image_paths = [os.path.join("./images", item["image_path"]) for item in data]
    labels = [item["caption"] for item in data]
    return image_paths, labels

# Example Training Loop Skeleton with Validation and Model Checkpointing
def train_model_with_validation(
    student_model,
    teacher_model,
    train_dataloader,
    val_dataloader,
    optimizer,
    device,
    num_epochs=10,
    checkpoint_path="checkpoint.pth",
):
    student_model.train()
    teacher_model.eval()  # Teacher model should be in eval mode

    # Initialize metrics dictionary
    metrics = {
        "train_loss": [],
        "val_loss": [],
    }  # Add 'train_accuracy': [], 'val_accuracy': [] if tracking accuracy
    loss_fn = AdaptiveConfidenceDistillationLoss(temperature=2.0)
    best_val_loss = float("inf")
    step = 0
    for epoch in range(num_epochs):
        # Training loop
        for inputs, labels in train_dataloader:
            if step % 200 == 0:
                # Validation loop
                val_loss = 0.0
                with torch.no_grad():
                    for inputs, labels in val_dataloader:
                        inputs, labels = inputs.to(device), labels.to(device)
                        teacher_outputs = teacher_model(inputs)
                        student_outputs = student_model(inputs)
                        loss = loss_fn(student_outputs, teacher_outputs, labels)
                        val_loss += loss.item() * inputs.size(
                            0
                        )  # Inside the training loop
                        metrics["train_loss"].append(
                            loss.item()
                        )  # Assuming loss is your loss variable

                        # Inside the validation loop
                        metrics["val_loss"].append(
                            val_loss
                        )  # Assuming val_loss is your accumulated validation loss
                        step += 1
                        print(f"val_loss: {val_loss}")
                # After training, plot and save metrics
                plot_and_save_metrics(metrics)
            inputs, labels = inputs.to(device), labels.to(device)
            optimizer.zero_grad()

            with torch.no_grad():
                teacher_outputs = teacher_model(inputs)

            student_outputs = student_model(inputs)

            loss = loss_fn(student_outputs, teacher_outputs, labels)
            loss.backward()
            optimizer.step()

        val_loss /= len(val_dataloader.dataset)
        print(f"Epoch [{epoch+1}/{num_epochs}], Validation Loss: {val_loss:.4f}")

        # Save the model checkpoint if validation loss improves
        if val_loss < best_val_loss:
            best_val_loss = val_loss
            torch.save(student_model.state_dict(), checkpoint_path)
            # After training, plot and save metrics
            plot_and_save_metrics(metrics)
            print(f"Saving model checkpoint at {checkpoint_path}")

# Example usage
# Assuming `train_dataloader`, `val_dataloader`, `optimizer`, and `device` are defined
train_model_with_validation(
    student_model,
    teacher_model,
    train_dataloader,
    val_dataloader,
    optimizer,
    device,
    num_epochs=10,
    checkpoint_path="checkpoint.pth",
)


utils

import matplotlib.pyplot as plt
import json

def plot_and_save_metrics(
    metrics, filename="training_metrics.png", metrics_path="training_metrics.json"
):
    epochs = range(1, len(metrics["train_loss"]) + 1)

    plt.figure(figsize=(10, 5))
    plt.subplot(1, 2, 1)
    plt.plot(epochs, metrics["train_loss"], label="Training Loss")
    plt.plot(epochs, metrics["val_loss"], label="Validation Loss")
    plt.title("Loss Metrics")
    plt.xlabel("Epoch")
    plt.ylabel("Loss")
    plt.legend()

    # If you track accuracy, you can plot it here similarly.
    # plt.subplot(1, 2, 2)
    # plt.plot(epochs, metrics['train_accuracy'], label='Training Accuracy')
    # plt.plot(epochs, metrics['val_accuracy'], label='Validation Accuracy')
    # plt.title('Accuracy Metrics')
    # plt.xlabel('Epoch')
    # plt.ylabel('Accuracy')
    # plt.legend()

    plt.tight_layout()
    plt.savefig(filename)
    plt.close()

    # Save metrics to a JSON file for further analysis
    with open(metrics_path, "w") as f:
        json.dump(metrics, f, indent=4)


import os
from transformers import ViTImageProcessor, ViTForImageClassification
from PIL import Image
import json
import torch

def generate_label_for_single_image(image_path):
    # Initialize the model and processor
    model = ViTForImageClassification.from_pretrained("google/vit-base-patch16-224")
    feature_extractor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224")

    # Load and process the image
    image = Image.open(image_path).convert("RGB")
    inputs = feature_extractor(images=image, return_tensors="pt")

    # Predict the class of the image
    with torch.no_grad():
        outputs = model(**inputs)

    logits = outputs.logits
    predicted_class_idx = logits.argmax(-1).item()

    # Optionally, print the predicted class (if id2label is available)
    predicted_label = (
        model.config.id2label[predicted_class_idx]
        if model.config.id2label
        else str(predicted_class_idx)
    )

    print("Predicted class index:", predicted_class_idx)
    print("Predicted label:", predicted_label)

    return predicted_class_idx, predicted_label

def generate_labels(image_dir, model_name, output_file, max_images=100):
    # Load the CLIP model and processor
    processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224")
    model = ViTForImageClassification.from_pretrained("google/vit-base-patch16-224")

    # Fetch image paths
    images = os.listdir(image_dir)
    images = images[:max_images]  # Limit to max_images

    labels = []

    for image_name in images:
        image_path = os.path.join(image_dir, image_name)

        # Load image
        image = Image.open(image_path).convert("RGB")

        # Prepare the image for the model
        inputs = processor(images=image, return_tensors="pt")

        # Generate image features (embeddings)
        with torch.no_grad():
            outputs = model(**inputs)

        # Here you would typically use the embeddings to find the closest text descriptions, but since
        # OpenCLIP doesn't provide direct label generation, we'll just save the embeddings for now.
        # This step is a placeholder for any specific label generation or nearest neighbor search you might implement.
        # For demonstration, let's assume a dummy label based on the max value index in the embeddings.
        logits = outputs.logits
        # model predicts one of the 1000 ImageNet classes
        predicted_class_idx = logits.argmax(-1).item()

        labels.append({"image_path": image_name, "label": predicted_class_idx})
        print("Predicted class: ", model.config.id2label[predicted_class_idx])
    # Save the labels to a JSON file
    with open(output_file, "w") as f:
        json.dump(labels, f, indent=4)

image_dir = "./images"
model_name = "google/vit-base-patch16-224"
output_file = "imagelabels.json"


import os
from transformers import ViTImageProcessor, ViTForImageClassification
from PIL import Image
import json
import torch

def generate_label_for_single_image(image_path):
    # Initialize the model and processor
    model = ViTForImageClassification.from_pretrained("google/vit-base-patch16-224")
    feature_extractor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224")

    # Load and process the image
    image = Image.open(image_path).convert("RGB")
    inputs = feature_extractor(images=image, return_tensors="pt")

    # Predict the class of the image
    with torch.no_grad():
        outputs = model(**inputs)

    logits = outputs.logits
    predicted_class_idx = logits.argmax(-1).item()

    # Optionally, print the predicted class (if id2label is available)
    predicted_label = (
        model.config.id2label[predicted_class_idx]
        if model.config.id2label
        else str(predicted_class_idx)
    )

    print("Predicted class index:", predicted_class_idx)
    print("Predicted label:", predicted_label)

    return predicted_class_idx, predicted_label

def generate_labels(image_dir, model_name, output_file, max_images=100):
    # Load the CLIP model and processor
    processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224")
    model = ViTForImageClassification.from_pretrained("google/vit-base-patch16-224")

    # Fetch image paths
    images = os.listdir(image_dir)
    images = images[:max_images]  # Limit to max_images

    labels = []

    for image_name in images:
        image_path = os.path.join(image_dir, image_name)

        # Load image
        image = Image.open(image_path).convert("RGB")

        # Prepare the image for the model


    # Predict the class of the image
    with torch.no_grad():
        outputs = model(**inputs)

    logits = outputs.logits
    predicted_class_idx = logits.argmax(-1).item()

    # Optionally, print the predicted class (if id2label is available)
    predicted_label = (
        model.config.id2label[predicted_class_idx]
        if model.config.id2label
        else str(predicted_class_idx)
    )

    print("Predicted class index:", predicted_class_idx)
    print("Predicted label:", predicted_label)

    return predicted_class_idx, predicted_label

def generate_labels(image_dir, model_name, output_file, max_images=100):
    # Load the CLIP model and processor
    processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224")
    model = ViTForImageClassification.from_pretrained("google/vit-base-patch16-224")

    # Fetch image paths
    images = os.listdir(image_dir)
    images = images[:max_images]  # Limit to max_images

    labels = []

    for image_name in images:
        image_path = os.path.join(image_dir, image_name)

        # Load image
        image = Image.open(image_path).convert("RGB")

        # Prepare the image for the model
        inputs = processor(images=image, return_tensors="pt")

        # Generate image features (embeddings)
        with torch.no_grad():
            outputs = model(**inputs)

        # Here you would typically use the embeddings to find the closest text descriptions, but since
        # OpenCLIP doesn't provide direct label generation, we'll just save the embeddings for now.
        # This step is a placeholder for any specific label generation or nearest neighbor search you might implement.
        # For demonstration, let's assume a dummy label based on the max value index in the embeddings.
        logits = outputs.logits
        # model predicts one of the 1000 ImageNet classes
        predicted_class_idx = logits.argmax(-1).item()

        labels.append({"image_path": image_name, "label": predicted_class_idx})
        print("Predicted class: ", model.config.id2label[predicted_class_idx])
    # Save the labels to a JSON file
    with open(output_file, "w") as f:
        json.dump(labels, f, indent=4)

image_dir = "./images"
model_name = "google/vit-base-patch16-224"
output_file = "imagelabels.json"


import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet18, resnet50  # Example models

# Define the teacher (weak model) and student (strong model)
teacher_model = resnet18(pretrained=True)
student_model = resnet50(pretrained=True)

# Custom Adaptive Confidence Distillation Loss
class AdaptiveConfidenceDistillationLoss(nn.Module):
    def __init__(self, temperature=1.0):
        super().__init__()
        self.temperature = temperature
        self.ce_loss = nn.CrossEntropyLoss()

    def forward(self, student_logits, teacher_logits, targets):
        # Soften probabilities
        soft_teacher_probs = F.softmax(teacher_logits / self.temperature, dim=1)
        soft_student_probs = F.log_softmax(student_logits / self.temperature, dim=1)


AdaptiveConfidenceDistillation.py

import torch.nn as nn
import torch.nn.functional as F

# Custom Adaptive Confidence Distillation Loss
class AdaptiveConfidenceDistillationLoss(nn.Module):
    def __init__(self, temperature=1.0):
        super().__init__()
        self.temperature = temperature
        self.ce_loss = nn.CrossEntropyLoss()

    def forward(self, student_logits, teacher_logits, targets):
        # Soften probabilities
        soft_teacher_probs = F.softmax(teacher_logits / self.temperature, dim=1)
        soft_student_probs = F.log_softmax(student_logits / self.temperature, dim=1)

        # Calculate the KL Divergence for the soft targets
        distillation_loss = F.kl_div(
            soft_student_probs, soft_teacher_probs, reduction="batchmean"
        )


import torch
import torch.nn as nn
import torch.optim as optim
from torchvision.models import resnet18, resnet50
from torch.utils.data import DataLoader, TensorDataset
import numpy as np
from train_example import train_model

# Define the teacher (weak model) and student (strong model)
teacher_model = resnet18(pretrained=False)  # Using pretrained=False for simplicity
student_model = resnet50(pretrained=False)

# Define a simple synthetic dataset resembling CIFAR-100
# CIFAR-100 has 32x32 images with 3 channels
# We'll create random data with similar dimensions
num_samples = 1000
input_channels = 3
input_height = 32
input_width = 32
num_classes = 100


dataset.py

import torch
from torch.utils.data import Dataset
import os
from PIL import Image
from generate_labels import generate_label_for_single_image
import json

# Custom Dataset Class
class CustomImageDataset(Dataset):
    def __init__(
        self, image_dir, transform=None, model_name="google/vit-base-patch16-224"
    ):
        self.image_dir = image_dir
        self.transform = transform
        self.memory_bank = load_memory_bank()  # Load existing labels from memory bank
        self.image_paths = [
            os.path.join(image_dir, img)
            for img in os.listdir(image_dir)
            if img.endswith((".png", ".jpg", ".jpeg"))
        ]


train.py

import torch
import torch.optim as optim
from torchvision import transforms
from torchvision.models import resnet18, resnet50
from torch.utils.data import DataLoader
import numpy as np
import os
import json
from utils import plot_and_save_metrics
from dataset import CustomImageDataset
from AdaptiveConfidenceDistillation import AdaptiveConfidenceDistillationLoss

# Define the teacher (weak model) and student (strong model)
teacher_model = resnet18(pretrained=True)  # Using pretrained=False for simplicity
student_model = resnet50(pretrained=True)

# Define a simple synthetic dataset resembling CIFAR-100
# CIFAR-100 has 32x32 images with 3 channels
# We'll create random data with similar dimensions
num_samples = 1000
input_channels = 3


utils.py

import matplotlib.pyplot as plt
import json

def plot_and_save_metrics(
    metrics, filename="training_metrics.png", metrics_path="training_metrics.json"
):
    epochs = range(1, len(metrics["train_loss"]) + 1)

    plt.figure(figsize=(10, 5))
    plt.subplot(1, 2, 1)
    plt.plot(epochs, metrics["train_loss"], label="Training Loss")
    plt.plot(epochs, metrics["val_loss"], label="Validation Loss")
    plt.title("Loss Metrics")
    plt.xlabel("Epoch")
    plt.ylabel("Loss")
    plt.legend()

    # If you track accuracy, you can plot it here similarly.
    # plt.subplot(1, 2, 2)
    # plt.plot(epochs, metrics['train_accuracy'], label='Training Accuracy')


import os
from transformers import ViTImageProcessor, ViTForImageClassification
from PIL import Image
import json
import torch

def generate_label_for_single_image(image_path):
    # Initialize the model and processor
    model = ViTForImageClassification.from_pretrained("google/vit-base-patch16-224")
    feature_extractor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224")

    # Load and process the image
    image = Image.open(image_path).convert("RGB")
    inputs = feature_extractor(images=image, return_tensors="pt")

    # Predict the class of the image
    with torch.no_grad():
        outputs = model(**inputs)

    logits = outputs.logits