Initial commit of transfer learning project files

config.yaml

@@ -0,0 +1,42 @@
+#网页设置
+wandb_setup:
+  project: "cifar10_transfer_learning"
+  experiment: "ResNet50_ImageNet_224px"
+  tags: ["cifar10","resnet50"]
+  seed: 42
+  job_type: "train"
+
+#数据加载参数
+data:
+ data_path: "./data"
+ batch_size: 64
+ num_workers: 4
+ image_size: [224,224]
+ in_channels: 3
+
+#模型结构参数
+model:
+ type: "TransferResNet50"
+ dropout_rate: 0.0
+ num_classes: 10
+
+#训练超参数
+train:
+ epochs: 30
+ save_dir: "./models"
+
+#优化器与调度器
+optimizer:
+ name: "adamw"
+ lr: 0.001
+ backbone_lr: 0.00005
+ weight_decay: 1e-3
+
+scheduler:
+ use_scheduler: True
+ type: "CosineAnnealingLR"
+ T_max: 30
+ eta_min: 1e-6
+
+
+

models/ResNet50_ImageNet_224px_best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca15171214c8a6d75fca585e6ce2b16ba4b6b735bbba58d92f729936a4b16a02
+size 94445757

src/dataset.py

@@ -0,0 +1,47 @@
+import torch
+from torch.utils.data import DataLoader
+from torchvision import datasets,transforms
+from src.path import DATA_DIR
+
+def get_dataloader(config):
+    batch_size = config.get('batch_size',64)
+    data_path = config.get('data_path',DATA_DIR)
+    num_workers = config.get('num_workers',4)
+
+    mean = [0.485, 0.456, 0.406]
+    std  = [0.229, 0.224, 0.225]
+
+    train_transform = transforms.Compose([
+        transforms.Resize((224,224)),
+        transforms.RandomHorizontalFlip(),
+        transforms.ToTensor(),
+        transforms.Normalize(mean,std),
+    ])
+    val_transform = transforms.Compose([
+        transforms.Resize((224,224)),
+        transforms.ToTensor(),
+        transforms.Normalize(mean,std),
+    ])
+
+    train_data = datasets.CIFAR10(root=data_path,train=True,download=True,transform=train_transform)
+    test_data = datasets.CIFAR10(root=data_path,train=False,download=True,transform=val_transform)
+
+    train_loader = DataLoader(
+        train_data,
+        batch_size=batch_size,
+        num_workers=num_workers,
+        shuffle=True,
+        pin_memory=True,
+        persistent_workers=True,
+    )
+
+    test_loader = DataLoader(
+        test_data,
+        batch_size=batch_size,
+        num_workers=num_workers,
+        shuffle=False,
+        pin_memory=True,
+        persistent_workers=True,
+    )
+
+    return train_loader,test_loader

src/engine.py

@@ -0,0 +1,79 @@
+import torch 
+from torch import nn
+import wandb
+from torch.amp import autocast, GradScaler
+
+def train_one_epoch(epoch_id,model,data_loader,loss_fn,optimizer,device,scaler):
+    model.train()
+    training_loss = 0.0
+    running_correct = 0
+    total_samples = 0
+
+    for batch,(X,y) in enumerate(data_loader):
+        if not X.is_cuda:
+            X,y = X.to(device,non_blocking=True),y.to(device,non_blocking=True)
+        X = X.to(memory_format=torch.channels_last)
+        optimizer.zero_grad(set_to_none=True)
+
+        with autocast('cuda',dtype=torch.float16):
+            pred = model(X)
+            loss = loss_fn(pred,y)
+
+        scaler.scale(loss).backward()
+        scaler.step(optimizer)
+        scaler.update()
+
+        pred_ids = pred.argmax(1)
+        running_correct += (pred_ids == y).type(torch.int).sum().item()
+        total_samples += y.size(0)
+
+        training_loss += loss.item()
+        
+    train_epoch_loss = training_loss / len(data_loader)
+    train_epoch_acc = running_correct / total_samples
+
+    return train_epoch_loss,train_epoch_acc
+
+def evaluate(epoch_id,model,data_loader,loss_fn,device):
+    model.eval()
+    testing_loss = 0.0
+    testing_correct = 0
+    total_samples = 0
+    bad_cases = []
+
+    with torch.no_grad():
+        for X,y in data_loader:
+            if not X.is_cuda:
+                X,y = X.to(device,non_blocking=True),y.to(device,non_blocking=True)
+
+            pred = model(X)
+            loss = loss_fn(pred,y)
+            testing_loss += loss.item()
+
+            pred_ids = pred.argmax(1) 
+            testing_correct += (pred_ids == y).type(torch.int).sum().item()
+            total_samples += y.size(0)
+
+            if len(bad_cases) < 20:
+                wrong_idx = (pred_ids != y).nonzero()
+                for idx in wrong_idx:
+                    if len(bad_cases) < 20:
+                        raw_img = X[idx.item()].cpu()
+
+                        mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
+                        std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)
+
+                        img = raw_img * std + mean
+                        img = torch.clamp(img,0,1)
+                        
+                        bad_cases.append(
+                            wandb.Image(img,caption=f"Pred: {pred_ids[idx].item()} | True: {y[idx].item()}")
+                        )
+    
+    val_epoch_loss = testing_loss / len(data_loader)
+    val_epoch_acc = testing_correct / total_samples
+
+    return val_epoch_loss,val_epoch_acc,bad_cases
+
+
+

src/model.py

@@ -0,0 +1,97 @@
+import torch
+from torch import nn
+from torchvision import transforms
+from torchvision.models import resnet18,resnet50,ResNet50_Weights
+
+class SimpleCNN(nn.Module):
+    def __init__(self, num_inputs=1,input_size=28,num_classes=10, dropout_rate=0.3):
+        super().__init__()
+
+        self.features = nn.Sequential(
+            # Block 1
+            nn.Conv2d(num_inputs, 32, kernel_size=3, padding=1),
+            nn.BatchNorm2d(32),
+            nn.ReLU(),
+            nn.MaxPool2d(2),
+
+            # Block 2
+            nn.Conv2d(32, 64, kernel_size=3, padding=1),
+            # 【修复 1】你之前写的是 62，必须是 64 才能匹配上一层的输出
+            nn.BatchNorm2d(64), 
+            nn.ReLU(),
+            nn.MaxPool2d(2),
+        )
+        final_size = input_size // 4
+        flatten_dim = 64 * final_size * final_size
+
+        self.classifier = nn.Sequential(
+            nn.Flatten(),
+            # 计算逻辑: 28 -> 14 -> 7，通道 64
+            nn.Linear(flatten_dim, 512),
+            nn.BatchNorm1d(512),
+            nn.ReLU(),
+            nn.Dropout(dropout_rate),
+            nn.Linear(512, num_classes),
+        )
+
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        # 【修复 2】你之前写的是 nn.Linaer (拼写错误)，导致全连接层没有被正确初始化！
+        if isinstance(m, (nn.Conv2d, nn.Linear)): 
+            nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+        
+        elif isinstance(m, (nn.BatchNorm2d, nn.BatchNorm1d)):
+            nn.init.constant_(m.weight, 1)
+            # 【修复 3】你之前写的是 m.weiht (拼写错误)，导致偏置没有归零
+            nn.init.constant_(m.bias, 0) 
+
+    # 【修复 4】你之前的代码里完全漏掉了 forward 函数！
+    # 没有这个函数，模型根本不知道怎么跑数据，虽然不报错(如果没调用)，但跑起来就是随机的
+    def forward(self, x):
+        x = self.features(x)
+        x = self.classifier(x)
+        return x
+
+class ResNet18_CIFAR(nn.Module):
+    def __init__(self,num_inputs=3,num_classes=10,dropout_rate=0.0):
+        super().__init__()
+
+        self.aug = nn.Sequential(
+            transforms.RandomHorizontalFlip(),
+            transforms.RandomCrop(32,padding=4,padding_mode='reflect'),
+        )
+
+        self.net = resnet18(weights=None)
+
+        self.net.conv1 = nn.Conv2d(num_inputs,64,kernel_size=3,stride=1,padding=1,bias=False)
+        self.net.maxpool = nn.Identity()
+
+        self.net.fc = nn.Linear(512,num_classes)
+
+    def forward(self,x):
+        if self.training:
+            x = self.aug(x)
+        return self.net(x)
+
+class TransferResNet50(nn.Module):
+    def __init__(self, num_classes=10, dropout_rate=0.0):
+        super().__init__()
+
+        print("⬇️ Loading Pre-trained ResNet50 (ImageNet)...")
+        # 1. 正确加载权重
+        self.net = resnet50(weights=ResNet50_Weights.DEFAULT)
+
+        # 2. 全网微调 (不冻结)
+        # 因为 CIFAR-10 和 ImageNet 差异较大 (清晰度、物体类别)，微调 Backbone 是必须的
+        # 我们已经在 train.py 里用了极小的 backbone_lr (1e-5) 来保护它，所以这里不需要 freeze
+        
+        # 3. 替换分类头
+        num_ftrs = self.net.fc.in_features
+        self.net.fc = nn.Sequential(
+            nn.Dropout(dropout_rate),
+            nn.Linear(num_ftrs, num_classes),
+        )
+    
+    def forward(self, x):
+        return self.net(x)

src/path.py

@@ -0,0 +1,23 @@
+import os
+from pathlib import Path
+import sys
+
+def get_project_root() -> Path:
+    """获取项目根目录的绝对路径"""
+    # 检查当前文件是否被打包
+    if getattr(sys, 'frozen', False):
+        # 如果是打包后的可执行文件
+        return Path(sys.executable).parent
+    else:
+        # 开发环境下定位项目根目录
+        current_file = Path(__file__).resolve()
+        # 返回 src 目录的父目录作为项目根目录
+        return current_file.parent.parent
+
+PROJECT_ROOT = get_project_root()
+CONFIG_PATH = PROJECT_ROOT / 'config.yaml'
+DATA_DIR = PROJECT_ROOT / 'data'
+MODELS_DIR = PROJECT_ROOT / 'models'
+
+for directory in [DATA_DIR, MODELS_DIR]:
+    directory.mkdir(exist_ok=True)

src/utils.py

@@ -0,0 +1,25 @@
+import torch
+import random
+import numpy as np
+import os
+import sys
+
+def get_device():
+    if torch.cuda.is_available():
+        return "cuda"
+    elif torch.backends.mps.is_available():
+        return "mps"
+    else:
+        return "cpu"
+    
+def seed_everthing(seed=42):
+    random.seed(seed)
+    os.environ['PYTHONHASHSEED'] = str(seed)
+    np.random.seed(seed)
+
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+
+    torch.backends.cudnn.deterministic = False
+    torch.backends.cudnn.benchmark = True
+

start_sweep.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# === 修改这里 ===
+# 1. 你的 Sweep ID (从 wandb sweep sweep.yaml 命令的输出中获得)
+SWEEP_ID="1217820711-sun-yat-sen-university/cifar10_chanllenge/srzfvp0g"
+
+# 2. 你想开几个 Agent (并行数)
+# 你的 9800X3D + 5070Ti 建议开 3 个
+NUM_AGENTS=2
+
+# 3. Tmux 会话名称 (随便起)
+SESSION_NAME="sweep_resnet18_try1"
+
+# 4. 你的 Conda 环境名
+CONDA_ENV="deep_learning"
+# ===============
+
+# 检查是否已经存在同名会话，如果有，先杀掉 (防止报错)
+tmux has-session -t $SESSION_NAME 2>/dev/null
+if [ $? == 0 ]; then
+    echo "⚠️  Session $SESSION_NAME already exists. Killing it..."
+    tmux kill-session -t $SESSION_NAME
+fi
+
+# 创建新会话 (后台模式)
+tmux new-session -d -s $SESSION_NAME
+
+# 循环创建窗口并运行 Agent
+for ((i=1; i<=NUM_AGENTS; i++)); do
+    # 如果不是第一个，就切分屏幕
+    if [ $i -gt 1 ]; then
+        tmux split-window -t $SESSION_NAME
+        tmux select-layout -t $SESSION_NAME tiled
+    fi
+
+    # 发送命令：激活环境 -> 运行 Agent
+    # C-m 代表回车键
+    tmux send-keys -t $SESSION_NAME "conda activate $CONDA_ENV" C-m
+    tmux send-keys -t $SESSION_NAME "wandb agent $SWEEP_ID" C-m
+    
+    echo "🚀 Agent $i started..."
+done
+
+# 进入 Tmux 界面
+echo "All agents running! Attaching..."
+tmux attach -t $SESSION_NAME

sweep.yaml

@@ -0,0 +1,82 @@
+program: train.py
+# 注意：Hyperband 必须配合 random 使用，而不是 bayes
+# 因为 Hyperband 依靠随机采样来覆盖搜索空间，然后靠剪枝来提高效率
+method: random
+
+project: "cifar10_chanllenge"
+name: "20251210-Hyperband-AdamW-LrSearch"
+description: >
+  本次实验目的是为了验证 ResNet18 在 CIFAR-10 上
+  使用 AdamW 配合强正则化 (Weight Decay > 0.01) 的效果。
+  使用了全显存加载优化。
+run_cap: 100
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - ${args}
+
+metric:
+  name: test_epoch_acc
+  goal: maximize
+
+# 🔥 核心：Hyperband 提前终止策略
+early_terminate:
+  type: hyperband
+  # 最小迭代次数：跑满 10 个 Epoch 后才开始评估是否要杀掉
+  # 避免模型还没热身就被误杀
+  min_iter: 10
+  # 淘汰比例：每次淘汰 2/3 的落后分子，保留 1/3 进入下一轮
+  eta: 3
+
+parameters:
+  project_name:
+    value: "cifar10_hyperband_search"
+
+  # --- 训练轮数 ---
+  train:
+    parameters:
+      epochs:
+        # 这里设置最大轮数。Hyperband 会自动在中间截断
+        # 设为 150，保证“幸存者”能跑完全程，收敛到极致
+        value: 150
+
+  # --- 数据参数 ---
+  data:
+    parameters:
+      batch_size:
+        # 搜索区间：涵盖了 SGD 喜欢的小 Batch 和 AdamW 喜欢的大 Batch
+        values: [256, 512, 1024]
+
+  # --- 模型参数 ---
+  model:
+    parameters:
+      type:
+        value: "ResNet18"
+      num_classes:
+        value: 10
+      dropout_rate:
+        # ResNet 自带 BN，通常不需要大 Dropout，搜一个小范围
+        distribution: uniform
+        min: 0.0
+        max: 0.2
+
+  # --- 优化器 (搜索重点) ---
+  optimizer:
+    parameters:
+      name:
+        # 同时尝试 SGD (传统SOTA王者) 和 AdamW (现代万金油)
+        values: ['sgd', 'adamw']
+      
+      lr:
+        # 学习率跨度要大！因为 SGD 需要 ~0.1，而 AdamW 需要 ~0.001
+        # log_uniform_values 会在对数尺度上均匀采样，保证两头都能搜到
+        distribution: log_uniform_values
+        min: 0.0001
+        max: 0.2
+      
+      weight_decay:
+        # 正则化力度的搜索
+        distribution: log_uniform_values
+        min: 1e-4  # 0.0001 (适合 SGD)
+        max: 1e-1  # 0.1 (适合 AdamW)

train.py

@@ -0,0 +1,229 @@
+import torch
+torch.set_float32_matmul_precision('high')
+import os
+import yaml
+import wandb
+from torch import nn
+from pathlib import Path
+import sys
+from torch.amp import GradScaler
+
+os.environ["CXX"] = "/usr/bin/g++"
+os.environ["CC"] = "/usr/bin/gcc"
+
+ROOT_DIR = Path(__file__).resolve().parent
+if ROOT_DIR  not in sys.path:
+    sys.path.append(str(ROOT_DIR))
+
+from src.dataset import get_dataloader
+from src.utils import get_device,seed_everthing
+from src.model import ResNet18_CIFAR,SimpleCNN,TransferResNet50
+from src.engine import train_one_epoch,evaluate
+
+def load_yaml(config_path=None):
+    if config_path is None:
+        config_path = ROOT_DIR / 'config.yaml'
+    try:
+        with open(config_path,'r',encoding='utf-8') as f:
+            config = yaml.safe_load(f)
+            return config
+    except FileNotFoundError:
+        print(f"{config_path} File not found!!")
+        exit(1)
+
+def main():
+    static_config = load_yaml()
+    wandb_cfg = static_config['wandb_setup']
+
+    wandb.init(
+        project=wandb_cfg.get('project','my_project'),
+        group=wandb_cfg.get('experiment','default'),
+        tags=wandb_cfg.get('tags',[]),
+        job_type=wandb_cfg.get('job_type','train'),
+        config=static_config,
+    )
+
+    cfg = wandb.config
+
+    relative_save_dir = cfg['train']['save_dir']
+    save_dir = (ROOT_DIR / relative_save_dir).resolve()
+    os.makedirs(save_dir,exist_ok=True)
+
+    best_acc = 0.0
+    print(f"        Save dir: {save_dir}")
+
+    print(f"    Model: {cfg['model']['type']}")
+    print(f"Experiment Start! Mode: {'Sweep' if wandb.run.sweep_id else 'Manual'}")
+    print(f"    Lr: {cfg['optimizer']['lr']}, Batch: {cfg['data']['batch_size']}, Opt: {cfg['optimizer']['name']}")
+
+    seed_everthing(cfg.get('seed',42))
+    device = get_device()
+
+    relative_data_path = cfg['data']['data_path']
+
+    absolute_data_path = (ROOT_DIR / relative_data_path).resolve()
+
+    data_cfg = cfg['data'].copy()
+    data_cfg['data_path'] = str(absolute_data_path)
+
+    print(f'Loading data from {absolute_data_path}...')
+    train_loader,test_loader = get_dataloader(data_cfg)
+
+    # 🔍【听诊器】检查一个 batch 的形状
+    dummy_x, dummy_y = next(iter(train_loader))
+    print(f"🧐 Inspection - Input Shape: {dummy_x.shape}")
+
+    model_type = cfg['model']['type']
+    num_classes = cfg['model']['num_classes']
+
+    dropout_rate = cfg['model'].get('dropout_rate',0.0)
+    num_inputs = cfg['model'].get('num_inputs',3)
+
+    input_size = cfg['model'].get('input_size',32)
+
+    if model_type == 'SimpleCNN':
+        model = SimpleCNN(
+            num_inputs = num_inputs,
+            input_size = input_size,
+            num_classes = num_classes,
+            dropout_rate = dropout_rate,
+        )
+    elif model_type == 'ResNet18':
+        model = ResNet18_CIFAR(
+            num_inputs = num_inputs,
+            num_classes = num_classes,
+            dropout_rate = dropout_rate,
+        )
+    elif model_type == 'TransferResNet50':
+        model = TransferResNet50(
+            num_classes=num_classes,
+            dropout_rate=dropout_rate,
+        )
+    else:
+        raise ValueError(f"Unknown model type: {model_type}")
+
+    model.to(device)
+    model = model.to(memory_format=torch.channels_last)
+
+    if hasattr(model,'net'):
+        print("⚡ Compiling ResNet backbone...")
+        model.net = torch.compile(model.net,mode='reduce-overhead')
+    else:
+        print("⚡ Compiling Full Model...")
+        model = torch.compile(model,mode='reduce-overhead')
+
+    opt_cfg = cfg['optimizer']
+    opt_name = opt_cfg['name'].lower()
+
+    # 1. 读取配置中的两个学习率 (务必转为 float)
+    lr_head = float(opt_cfg['lr'])           # 对应 config 里的 lr
+    lr_backbone = float(opt_cfg.get('backbone_lr', lr_head * 0.1)) # 对应 config 里的 backbone_lr，没填默认是 head 的 1/10
+    weight_decay = float(opt_cfg.get('weight_decay', 0.0))
+
+    # 2. 将模型参数分组 (Backbone vs Head)
+    # 逻辑：检查参数名里是否包含 "fc" (ResNet 的最后一层通常叫 fc)
+    backbone_params = []
+    head_params = []
+    
+    for name, param in model.named_parameters():
+        if "fc" in name:
+            head_params.append(param)
+        else:
+            backbone_params.append(param)
+
+    print(f"🔧 Optimizer Setup: Head LR={lr_head}, Backbone LR={lr_backbone}")
+
+    # 3. 初始化优化器 (传入参数组 list)
+    if opt_name == "adam":
+        optimizer = torch.optim.Adam([
+            {'params': backbone_params, 'lr': lr_backbone},
+            {'params': head_params,     'lr': lr_head}
+        ], weight_decay=weight_decay)
+        
+    elif opt_name == "adamw":
+        optimizer = torch.optim.AdamW([
+            {'params': backbone_params, 'lr': lr_backbone},
+            {'params': head_params,     'lr': lr_head}
+        ], weight_decay=weight_decay)
+        
+    elif opt_name == "sgd":
+        optimizer = torch.optim.SGD([
+            {'params': backbone_params, 'lr': lr_backbone},
+            {'params': head_params,     'lr': lr_head}
+        ], momentum=0.9, weight_decay=weight_decay)
+        
+    else:
+        raise ValueError(f"不支持的优化器： {opt_name}")
+    
+    scheduler = None
+    if 'scheduler' in cfg and cfg['scheduler'].get('use_scheduler',False):
+        sch_cfg = cfg['scheduler']
+
+        if sch_cfg['type'] == 'CosineAnnealingLR':
+            t_max = cfg['train']['epochs']
+            eta_min = float(sch_cfg.get('eta_min',0.0))
+
+            scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
+                optimizer,
+                T_max = t_max,
+                eta_min = eta_min,
+
+            )
+        
+        elif sch_cfg['type'] == 'StepLR':
+            step_size = sch_cfg.get('step_size',10)
+            gamma = sch_cfg.get('gamma',0.1)
+            scheduler = torch.optim.lr_scheduler.StepLR(
+                optimizer,
+                step_size=step_size,
+                gamma=gamma,
+            )
+        else:
+            print('Not using Learning Rate Scheduler')
+    
+    loss_fn = nn.CrossEntropyLoss(label_smoothing=0.1)
+
+    epochs = cfg['train']['epochs']
+    scaler = GradScaler('cuda')
+
+    for epoch in range(epochs):
+        train_epoch_loss,train_epoch_acc = train_one_epoch(epoch,model,train_loader,loss_fn,optimizer,device,scaler)
+        val_epoch_loss,val_epoch_acc,bad_cases =  evaluate(epoch,model,test_loader,loss_fn,device)
+
+        current_lr = optimizer.param_groups[0]['lr']
+
+        if scheduler is not None:
+            scheduler.step()
+
+        print(f"Epoch {epoch+1}/{epochs}\t[LR: {current_lr:>.6f}]\tTrain Loss: {train_epoch_loss:>.3f}\tTrain Acc: {train_epoch_acc:>.2%}\t|\tVal Loss: {val_epoch_loss:>.3f}\tVal Acc: {val_epoch_acc:>.2%}")
+
+
+        if val_epoch_acc > best_acc:
+            best_acc = val_epoch_acc
+            save_name = f"{cfg['wandb_setup']['experiment']}_best.pth"
+            save_path = save_dir / save_name
+
+            torch.save(model.state_dict(),save_path)
+
+            print(f"🌟 New Best Acc: {best_acc:.2f} -> Model save to: {save_path}")
+            
+
+        wandb.log({
+            "train_epoch_loss":train_epoch_loss,
+            "train_epoch_acc":train_epoch_acc,
+            "test_epoch_loss":val_epoch_loss,
+            "test_epoch_acc":val_epoch_acc,
+            'best_acc':best_acc,
+            "bad_cases":bad_cases,
+            "learning_rate": current_lr,
+            "epoch": epoch,
+        })
+    wandb.finish()
+
+if __name__ == '__main__':
+    main()
+
+
+        
+
+