Spaces:
Sleeping
Sleeping
OOP complete and functional rewrite
Browse files- main.py +58 -162
- src/config/config.py +15 -49
- src/data/augment.py +145 -153
- src/data/dataset.py +68 -0
- src/data/download.py +5 -14
- src/models/cnn.py +3 -5
- src/models/predict.py +66 -185
- src/models/traincnn.py +225 -274
main.py
CHANGED
|
@@ -2,18 +2,30 @@ import os
|
|
| 2 |
import numpy as np
|
| 3 |
import torch
|
| 4 |
import json
|
|
|
|
| 5 |
import matplotlib.pyplot as plt
|
| 6 |
import argparse
|
| 7 |
from sklearn.model_selection import train_test_split
|
| 8 |
|
| 9 |
-
from src.data.download import
|
| 10 |
-
from src.data.augment import
|
| 11 |
from src.models.cnn import CNN
|
| 12 |
-
from src.models.
|
| 13 |
-
from src.models.
|
| 14 |
-
from src.config.config import
|
| 15 |
|
| 16 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
parser = argparse.ArgumentParser(
|
| 18 |
description="ESC50 Audio Classification",
|
| 19 |
formatter_class=argparse.RawDescriptionHelpFormatter
|
|
@@ -64,8 +76,8 @@ def main():
|
|
| 64 |
|
| 65 |
resume_parser = subparsers.add_parser('resume', help='Resume training from checkpoint')
|
| 66 |
resume_parser.add_argument('--resume-from', type=str, required=True, help='Path to checkpoint file')
|
| 67 |
-
resume_parser.add_argument('--X-path', type=str,
|
| 68 |
-
resume_parser.add_argument('--y-path', type=str,
|
| 69 |
resume_parser.add_argument('--epochs', type=int, default=100, help='Number of epochs (default: 100)')
|
| 70 |
resume_parser.add_argument('--batch-size', type=int, default=100, help='Batch size (default: 100)')
|
| 71 |
resume_parser.add_argument('--lr', type=float, default=0.01, help='Learning rate (default: 0.01)')
|
|
@@ -76,7 +88,7 @@ def main():
|
|
| 76 |
|
| 77 |
predict_parser = subparsers.add_parser('predict', help='Predict audio file class')
|
| 78 |
predict_parser.add_argument('audio_file', type=str, help='Path to .wav file to classify')
|
| 79 |
-
predict_parser.add_argument('--model', type=str, default='
|
| 80 |
predict_parser.add_argument('--top-k', type=int, default=5, help='Number of top predictions (default: 5)')
|
| 81 |
predict_parser.add_argument('--device', type=str, default='cuda' if torch.cuda.is_available() else 'cpu', help='Device (default: auto)')
|
| 82 |
predict_parser.set_defaults(func=cmd_predict)
|
|
@@ -84,194 +96,78 @@ def main():
|
|
| 84 |
args = parser.parse_args()
|
| 85 |
args.func(args)
|
| 86 |
|
| 87 |
-
def cmd_download(args):
|
| 88 |
-
print("
|
|
|
|
|
|
|
| 89 |
|
| 90 |
-
download_clean()
|
| 91 |
|
| 92 |
-
print("
|
| 93 |
|
| 94 |
-
def cmd_augment(args):
|
| 95 |
print("Augmenting audio data...")
|
| 96 |
-
create_augmented_datasets(args.input_dir, args.output_dir)
|
| 97 |
|
| 98 |
-
|
| 99 |
|
| 100 |
-
|
| 101 |
-
print("Processing audio data...")
|
| 102 |
-
|
| 103 |
-
print("Creating log-mel spectrograms...")
|
| 104 |
-
X, y = create_log_mel(args.input_dir, args.output_dir)
|
| 105 |
-
|
| 106 |
-
print(f"Dataset size: {len(X)} samples, {len(np.unique(y))} classes")
|
| 107 |
-
print(f"Saved to {args.output_dir}")
|
| 108 |
|
| 109 |
-
|
| 110 |
-
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 111 |
-
print(f"Using device: {device}")
|
| 112 |
|
| 113 |
-
|
| 114 |
-
|
| 115 |
|
| 116 |
-
|
| 117 |
-
print("Loading existing processed data...")
|
| 118 |
-
X = np.load(X_path, allow_pickle=True)
|
| 119 |
-
y = np.load(y_path)
|
| 120 |
-
else:
|
| 121 |
-
print("Processing audio data...")
|
| 122 |
-
audio_training_path = args.audio_dir or "data/audio/0"
|
| 123 |
-
directories = os.listdir(audio_training_path)
|
| 124 |
|
| 125 |
-
|
| 126 |
-
print("Creating augmented datasets...")
|
| 127 |
-
create_augmented_datasets(audio_training_path, "data/audio")
|
| 128 |
|
| 129 |
-
|
| 130 |
-
X, y = create_log_mel(args.audio_dir or "data/audio", args.output_dir or "data/preprocessed")
|
| 131 |
-
|
| 132 |
-
print(f"Dataset size: {len(X)} samples, {len(np.unique(y))} classes")
|
| 133 |
-
|
| 134 |
-
X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.2, random_state=42, stratify=y )
|
| 135 |
-
print(f"Train: {len(X_train)}, Val: {len(X_val)}")
|
| 136 |
-
model = CNN(n_classes=len(np.unique(y)))
|
| 137 |
-
best_val_acc = train_cnn(
|
| 138 |
-
model,
|
| 139 |
-
X_train, y_train,
|
| 140 |
-
X_val, y_val,
|
| 141 |
-
epochs=args.epochs,
|
| 142 |
-
batch_size=args.batch_size,
|
| 143 |
-
lr=args.lr,
|
| 144 |
-
fold_num=0,
|
| 145 |
-
device=device,
|
| 146 |
-
use_all_patches=True,
|
| 147 |
-
samples_per_epoch_fraction=args.sample_fraction,
|
| 148 |
-
checkpoint_dir=args.checkpoint_dir,
|
| 149 |
-
save_every_n_epoch=args.save_every,
|
| 150 |
-
resume_from=None )
|
| 151 |
-
|
| 152 |
-
print(f"\nTraining complete! Best validation accuracy: {best_val_acc:.4f}")
|
| 153 |
-
return best_val_acc
|
| 154 |
|
| 155 |
-
def
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
X_path = args.X_path or "data/preprocessed/X.npy"
|
| 160 |
-
y_path = args.y_path or "data/preprocessed/y.npy"
|
| 161 |
-
|
| 162 |
-
if os.path.exists(X_path) and os.path.exists(y_path):
|
| 163 |
-
print("Loading existing processed data...")
|
| 164 |
-
X = np.load(X_path, allow_pickle=True)
|
| 165 |
-
y = np.load(y_path)
|
| 166 |
-
else:
|
| 167 |
-
print("Processing audio data...")
|
| 168 |
-
audio_training_path = args.audio_dir or "data/audio/0"
|
| 169 |
-
directories = os.listdir(audio_training_path)
|
| 170 |
-
|
| 171 |
-
if len(directories) == 1 and args.augment:
|
| 172 |
-
print("Creating augmented datasets...")
|
| 173 |
-
create_augmented_datasets(audio_training_path, "data/audio")
|
| 174 |
-
|
| 175 |
-
print("Creating log-mel spectrograms...")
|
| 176 |
-
X, y = create_log_mel(args.audio_dir or "data/audio", args.output_dir or "data/preprocessed")
|
| 177 |
-
|
| 178 |
-
print(f"Dataset size: {len(X)} samples, {len(np.unique(y))} classes")
|
| 179 |
-
|
| 180 |
-
X_train, X_val, y_train, y_val = train_test_split(
|
| 181 |
-
X, y, test_size=0.2, random_state=42, stratify=y
|
| 182 |
-
)
|
| 183 |
-
print(f"Train: {len(X_train)}, Val: {len(X_val)}")
|
| 184 |
-
|
| 185 |
-
model = CNN(n_classes=len(np.unique(y)))
|
| 186 |
-
|
| 187 |
-
fold_accs, mean_acc = train_k_fold_cnn(
|
| 188 |
-
model_class=lambda: CNN(),
|
| 189 |
-
X=X,
|
| 190 |
-
y=y,
|
| 191 |
epochs=args.epochs,
|
| 192 |
batch_size=args.batch_size,
|
| 193 |
lr=args.lr,
|
| 194 |
-
k_fold=args.k_fold,
|
| 195 |
-
device=device,
|
| 196 |
-
use_all_patches=True,
|
| 197 |
samples_per_epoch_fraction=args.sample_fraction,
|
| 198 |
checkpoint_dir=args.checkpoint_dir,
|
| 199 |
-
save_every_n_epoch=args.save_every
|
| 200 |
-
)
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
|
| 205 |
-
def
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
print("Loading processed data...")
|
| 210 |
-
X = np.load(args.X_path or "data/preprocessed/X.npy", allow_pickle=True)
|
| 211 |
-
y = np.load(args.y_path or "data/preprocessed/y.npy")
|
| 212 |
-
|
| 213 |
-
X_train, X_val, y_train, y_val = train_test_split(
|
| 214 |
-
X, y, test_size=0.2, random_state=42, stratify=y
|
| 215 |
-
)
|
| 216 |
-
print(f"Train: {len(X_train)}, Val: {len(X_val)}")
|
| 217 |
-
|
| 218 |
-
n_classes = len(np.unique(y))
|
| 219 |
-
model = CNN(n_classes=n_classes)
|
| 220 |
-
|
| 221 |
-
print(f"Resuming from: {args.resume_from}")
|
| 222 |
-
|
| 223 |
-
best_val_acc = train_cnn(
|
| 224 |
-
model,
|
| 225 |
-
X_train, y_train,
|
| 226 |
-
X_val, y_val,
|
| 227 |
epochs=args.epochs,
|
| 228 |
batch_size=args.batch_size,
|
| 229 |
lr=args.lr,
|
| 230 |
-
device=device,
|
| 231 |
-
use_all_patches=True,
|
| 232 |
samples_per_epoch_fraction=args.sample_fraction,
|
| 233 |
checkpoint_dir=args.checkpoint_dir,
|
| 234 |
save_every_n_epoch=args.save_every,
|
| 235 |
-
|
| 236 |
-
)
|
| 237 |
-
|
| 238 |
-
print(f"\nTraining complete! Best validation accuracy: {best_val_acc:.4f}")
|
| 239 |
-
return best_val_acc
|
| 240 |
|
| 241 |
-
def cmd_predict(args):
|
| 242 |
if not os.path.exists(args.audio_file):
|
| 243 |
-
print(f"Error: Audio file not found: {args.audio_file}")
|
| 244 |
-
sys.exit(1)
|
| 245 |
-
|
| 246 |
if not os.path.exists(args.model):
|
| 247 |
-
print(f"Error: Model file not found: {args.model}")
|
| 248 |
-
sys.exit(1)
|
| 249 |
-
|
| 250 |
-
try:
|
| 251 |
-
model = load_model(args.model, device=args.device)
|
| 252 |
-
except Exception as e:
|
| 253 |
-
print(f"Error loading model: {e}")
|
| 254 |
-
import traceback
|
| 255 |
-
traceback.print_exc()
|
| 256 |
-
sys.exit(1)
|
| 257 |
-
|
| 258 |
try:
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
)
|
| 262 |
-
|
| 263 |
print("\n" + "=" * 60)
|
| 264 |
print(f"Top {args.top_k} Predictions:")
|
| 265 |
print("=" * 60)
|
| 266 |
-
|
| 267 |
for i, (prob, idx) in enumerate(zip(top_probs, top_indices)):
|
| 268 |
-
class_name = esc50_labels[idx]
|
| 269 |
marker = "★" if idx == predicted_class else " "
|
| 270 |
-
print(f"{marker} {i+1}. {
|
| 271 |
-
|
| 272 |
except Exception as e:
|
| 273 |
-
print(f"\nError during prediction: {e}")
|
| 274 |
import traceback
|
|
|
|
| 275 |
traceback.print_exc()
|
| 276 |
sys.exit(1)
|
| 277 |
|
|
|
|
| 2 |
import numpy as np
|
| 3 |
import torch
|
| 4 |
import json
|
| 5 |
+
import sys
|
| 6 |
import matplotlib.pyplot as plt
|
| 7 |
import argparse
|
| 8 |
from sklearn.model_selection import train_test_split
|
| 9 |
|
| 10 |
+
from src.data.download import ESC50Downloader
|
| 11 |
+
from src.data.augment import AudioAugment
|
| 12 |
from src.models.cnn import CNN
|
| 13 |
+
from src.models.predict import AudioPredictor
|
| 14 |
+
from src.models.traincnn import CNNTrainer
|
| 15 |
+
from src.config.config import ProcessingConfig, DatasetConfig, DownloadConfig, TrainConfig
|
| 16 |
|
| 17 |
+
def _load_or_preprocess(args) -> tuple[np.ndarray, np.ndarray]:
|
| 18 |
+
X_path = args.X_path or "data/preprocessed/X.npy"
|
| 19 |
+
y_path = args.y_path or "data/preprocessed/y.npy"
|
| 20 |
+
if os.path.exists(X_path) and os.path.exists(y_path):
|
| 21 |
+
print("Loading existing processed data...")
|
| 22 |
+
return np.load(X_path, allow_pickle=True), np.load(y_path)
|
| 23 |
+
print("Processing audio data...")
|
| 24 |
+
augmenter = AudioAugment()
|
| 25 |
+
augmenter.run(augment=True, preprocess=True)
|
| 26 |
+
return np.load(X_path, allow_pickle=True), np.load(y_path)
|
| 27 |
+
|
| 28 |
+
def main() -> None:
|
| 29 |
parser = argparse.ArgumentParser(
|
| 30 |
description="ESC50 Audio Classification",
|
| 31 |
formatter_class=argparse.RawDescriptionHelpFormatter
|
|
|
|
| 76 |
|
| 77 |
resume_parser = subparsers.add_parser('resume', help='Resume training from checkpoint')
|
| 78 |
resume_parser.add_argument('--resume-from', type=str, required=True, help='Path to checkpoint file')
|
| 79 |
+
resume_parser.add_argument('--X-path', type=str, default="data/preprocessed/X.npy")
|
| 80 |
+
resume_parser.add_argument('--y-path', type=str, default="data/preprocessed/y.npy")
|
| 81 |
resume_parser.add_argument('--epochs', type=int, default=100, help='Number of epochs (default: 100)')
|
| 82 |
resume_parser.add_argument('--batch-size', type=int, default=100, help='Batch size (default: 100)')
|
| 83 |
resume_parser.add_argument('--lr', type=float, default=0.01, help='Learning rate (default: 0.01)')
|
|
|
|
| 88 |
|
| 89 |
predict_parser = subparsers.add_parser('predict', help='Predict audio file class')
|
| 90 |
predict_parser.add_argument('audio_file', type=str, help='Path to .wav file to classify')
|
| 91 |
+
predict_parser.add_argument('--model', type=str, default='final_model.pt', help='Path to model checkpoint (default: best_model.pt)')
|
| 92 |
predict_parser.add_argument('--top-k', type=int, default=5, help='Number of top predictions (default: 5)')
|
| 93 |
predict_parser.add_argument('--device', type=str, default='cuda' if torch.cuda.is_available() else 'cpu', help='Device (default: auto)')
|
| 94 |
predict_parser.set_defaults(func=cmd_predict)
|
|
|
|
| 96 |
args = parser.parse_args()
|
| 97 |
args.func(args)
|
| 98 |
|
| 99 |
+
def cmd_download(args) -> None:
|
| 100 |
+
print("Downloading ESC50 audio data...")
|
| 101 |
+
|
| 102 |
+
downloader = ESC50Downloader()
|
| 103 |
|
| 104 |
+
downloader.download_clean()
|
| 105 |
|
| 106 |
+
print("Downloaded and cleaned data.")
|
| 107 |
|
| 108 |
+
def cmd_augment(args) -> None:
|
| 109 |
print("Augmenting audio data...")
|
|
|
|
| 110 |
|
| 111 |
+
augmentater = AudioAugment()
|
| 112 |
|
| 113 |
+
augmentater.run(augment=True, preprocess=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
|
| 115 |
+
print(f"Augmented data and saved to {args.output_dir}")
|
|
|
|
|
|
|
| 116 |
|
| 117 |
+
def cmd_preprocess(args) -> None:
|
| 118 |
+
print("Processing audio data...")
|
| 119 |
|
| 120 |
+
augmentater = AudioAugment()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
|
| 122 |
+
augmentater.run(augment=False, preprocess=True)
|
|
|
|
|
|
|
| 123 |
|
| 124 |
+
print(f"Preprocessed data and saved to {args.output_dir}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
|
| 126 |
+
def cmd_train(args) -> None:
|
| 127 |
+
X, y = _load_or_preprocess(args)
|
| 128 |
+
trainer = CNNTrainer(TrainConfig(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
epochs=args.epochs,
|
| 130 |
batch_size=args.batch_size,
|
| 131 |
lr=args.lr,
|
|
|
|
|
|
|
|
|
|
| 132 |
samples_per_epoch_fraction=args.sample_fraction,
|
| 133 |
checkpoint_dir=args.checkpoint_dir,
|
| 134 |
+
save_every_n_epoch=args.save_every,
|
| 135 |
+
))
|
| 136 |
+
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
|
| 137 |
+
best_val_acc = trainer.train_cnn(CNN(n_classes=len(np.unique(y))), X_train, y_train, X_val, y_val, fold_num=0)
|
| 138 |
+
print(f"\nTraining complete! Best validation accuracy: {best_val_acc:.4f}")
|
| 139 |
|
| 140 |
+
def cmd_train_cv(args) -> None:
|
| 141 |
+
X, y = _load_or_preprocess(args)
|
| 142 |
+
trainer = CNNTrainer(TrainConfig(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
epochs=args.epochs,
|
| 144 |
batch_size=args.batch_size,
|
| 145 |
lr=args.lr,
|
|
|
|
|
|
|
| 146 |
samples_per_epoch_fraction=args.sample_fraction,
|
| 147 |
checkpoint_dir=args.checkpoint_dir,
|
| 148 |
save_every_n_epoch=args.save_every,
|
| 149 |
+
))
|
| 150 |
+
fold_accs, mean_acc = trainer.train_k_fold_cnn(CNN, X, y)
|
| 151 |
+
print(f"\nTraining complete! Mean validation accuracy: {mean_acc:.4f}")
|
|
|
|
|
|
|
| 152 |
|
| 153 |
+
def cmd_predict(args) -> None:
|
| 154 |
if not os.path.exists(args.audio_file):
|
| 155 |
+
print(f"Error: Audio file not found: {args.audio_file}"); sys.exit(1)
|
|
|
|
|
|
|
| 156 |
if not os.path.exists(args.model):
|
| 157 |
+
print(f"Error: Model file not found: {args.model}"); sys.exit(1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
try:
|
| 159 |
+
predictor = AudioPredictor(model_path=args.model, device=args.device)
|
| 160 |
+
predicted_class, top_probs, top_indices = predictor.predict_file(args.audio_file, top_k=args.top_k)
|
| 161 |
+
labels = DatasetConfig().esc50_labels
|
|
|
|
| 162 |
print("\n" + "=" * 60)
|
| 163 |
print(f"Top {args.top_k} Predictions:")
|
| 164 |
print("=" * 60)
|
|
|
|
| 165 |
for i, (prob, idx) in enumerate(zip(top_probs, top_indices)):
|
|
|
|
| 166 |
marker = "★" if idx == predicted_class else " "
|
| 167 |
+
print(f"{marker} {i+1}. {labels[idx]:20s} - {prob*100:6.2f}%")
|
|
|
|
| 168 |
except Exception as e:
|
|
|
|
| 169 |
import traceback
|
| 170 |
+
print(f"\nError during prediction: {e}")
|
| 171 |
traceback.print_exc()
|
| 172 |
sys.exit(1)
|
| 173 |
|
src/config/config.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import os
|
| 2 |
from dataclasses import dataclass, field
|
| 3 |
from pathlib import Path
|
| 4 |
-
from typing import List
|
| 5 |
|
| 6 |
-
@dataclass
|
| 7 |
class ProcessingConfig:
|
| 8 |
audio_path: Path = Path("data/audio/0")
|
| 9 |
augmented_path: Path = Path("data/audio/")
|
|
@@ -23,7 +23,7 @@ class ProcessingConfig:
|
|
| 23 |
pitch_shift_rates = [-3.5, -2.5, -2, -1, 1, 2.5, 3, 3.5]
|
| 24 |
drc_types = ["radio", "filmstandard", "musicstandard", "speech"]
|
| 25 |
|
| 26 |
-
@dataclass
|
| 27 |
class DatasetConfig:
|
| 28 |
cnn_input_length: int = 128
|
| 29 |
sample_rate: int = 44100
|
|
@@ -40,7 +40,7 @@ class DatasetConfig:
|
|
| 40 |
'train', 'church_bells', 'airplane', 'fireworks', 'hand_saw'
|
| 41 |
])
|
| 42 |
|
| 43 |
-
@dataclass
|
| 44 |
class DownloadConfig:
|
| 45 |
repo_url: str = "https://github.com/karolpiczak/ESC-50/archive/refs/heads/master.zip"
|
| 46 |
repo_dst_dir: Path = Path("data")
|
|
@@ -55,48 +55,14 @@ class DownloadConfig:
|
|
| 55 |
def __post_init__(self):
|
| 56 |
object.__setattr__(self, "audio_dst_dir", self.repo_dst_dir / "audio" / "0")
|
| 57 |
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
cnn_input_length = 128
|
| 70 |
-
|
| 71 |
-
sample_rate = 44100
|
| 72 |
-
|
| 73 |
-
esc50_labels = [
|
| 74 |
-
'dog', 'rooster', 'pig', 'cow', 'frog',
|
| 75 |
-
'cat', 'hen', 'insects', 'sheep', 'crow',
|
| 76 |
-
'rain', 'sea_waves', 'crackling_fire', 'crickets', 'chirping_birds',
|
| 77 |
-
'water_drops', 'wind', 'pouring_water', 'toilet_flush', 'thunderstorm',
|
| 78 |
-
'crying_baby', 'sneezing', 'clapping', 'breathing', 'coughing',
|
| 79 |
-
'footsteps', 'laughing', 'brushing_teeth', 'snoring', 'drinking_sipping',
|
| 80 |
-
'door_wood_knock', 'mouse_click', 'keyboard_typing', 'door_wood_creaks', 'can_opening',
|
| 81 |
-
'washing_machine', 'vacuum_cleaner', 'clock_alarm', 'clock_tick', 'glass_breaking',
|
| 82 |
-
'helicopter', 'chainsaw', 'siren', 'car_horn', 'engine',
|
| 83 |
-
'train', 'church_bells', 'airplane', 'fireworks', 'hand_saw'
|
| 84 |
-
]
|
| 85 |
-
|
| 86 |
-
# download.py
|
| 87 |
-
repo_url = "https://github.com/karolpiczak/ESC-50/archive/refs/heads/master.zip"
|
| 88 |
-
repo_dst_dir = "data"
|
| 89 |
-
audio_dst_dir = os.path.join(repo_dst_dir, "audio", "0")
|
| 90 |
-
|
| 91 |
-
paths_to_delete = [
|
| 92 |
-
".gitignore",
|
| 93 |
-
"esc50.gif",
|
| 94 |
-
"LICENSE",
|
| 95 |
-
"pytest.ini",
|
| 96 |
-
"README.md",
|
| 97 |
-
"requirements.txt",
|
| 98 |
-
"tests",
|
| 99 |
-
"meta",
|
| 100 |
-
".github",
|
| 101 |
-
".circleci"
|
| 102 |
-
]
|
|
|
|
| 1 |
import os
|
| 2 |
from dataclasses import dataclass, field
|
| 3 |
from pathlib import Path
|
| 4 |
+
from typing import List, Optional
|
| 5 |
|
| 6 |
+
@dataclass
|
| 7 |
class ProcessingConfig:
|
| 8 |
audio_path: Path = Path("data/audio/0")
|
| 9 |
augmented_path: Path = Path("data/audio/")
|
|
|
|
| 23 |
pitch_shift_rates = [-3.5, -2.5, -2, -1, 1, 2.5, 3, 3.5]
|
| 24 |
drc_types = ["radio", "filmstandard", "musicstandard", "speech"]
|
| 25 |
|
| 26 |
+
@dataclass
|
| 27 |
class DatasetConfig:
|
| 28 |
cnn_input_length: int = 128
|
| 29 |
sample_rate: int = 44100
|
|
|
|
| 40 |
'train', 'church_bells', 'airplane', 'fireworks', 'hand_saw'
|
| 41 |
])
|
| 42 |
|
| 43 |
+
@dataclass
|
| 44 |
class DownloadConfig:
|
| 45 |
repo_url: str = "https://github.com/karolpiczak/ESC-50/archive/refs/heads/master.zip"
|
| 46 |
repo_dst_dir: Path = Path("data")
|
|
|
|
| 55 |
def __post_init__(self):
|
| 56 |
object.__setattr__(self, "audio_dst_dir", self.repo_dst_dir / "audio" / "0")
|
| 57 |
|
| 58 |
+
@dataclass
|
| 59 |
+
class TrainConfig:
|
| 60 |
+
epochs: int = 50
|
| 61 |
+
batch_size: int = 100
|
| 62 |
+
lr: int = 0.001
|
| 63 |
+
device = "cuda"
|
| 64 |
+
use_all_patches: bool = True
|
| 65 |
+
samples_per_epoch_fraction: float = 1/8
|
| 66 |
+
checkpoint_dir: str = "models/checkpoints"
|
| 67 |
+
save_every_n_epoch: int = 1
|
| 68 |
+
resume_from: Optional[str] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/data/augment.py
CHANGED
|
@@ -3,173 +3,165 @@ import librosa
|
|
| 3 |
import numpy as np
|
| 4 |
import os
|
| 5 |
import soundfile as sf
|
|
|
|
| 6 |
|
| 7 |
-
from src.config.config import
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
):
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
for filename in tqdm.tqdm(filenames, desc="Processing audio files"):
|
| 18 |
-
filename_splitted = filename.split("-")
|
| 19 |
-
label = filename_splitted[-1].split(".")[0]
|
| 20 |
-
label = label.split("_")[0]
|
| 21 |
-
labels.append(int(label))
|
| 22 |
-
|
| 23 |
-
file_path = os.path.join(audio_path, filename)
|
| 24 |
-
audio, sr = librosa.load(file_path, sr=sample_rate)
|
| 25 |
-
|
| 26 |
mel_spec = librosa.feature.melspectrogram(
|
| 27 |
y=audio,
|
| 28 |
-
sr=
|
| 29 |
-
n_fft=fft_size,
|
| 30 |
-
hop_length=hop_size,
|
| 31 |
-
win_length=frame_size,
|
| 32 |
-
n_mels=n_bands,
|
| 33 |
fmin=0,
|
| 34 |
-
fmax=sample_rate / 2,
|
| 35 |
window='hann'
|
| 36 |
)
|
| 37 |
-
|
| 38 |
mel_spectrogram_db = 10 * np.log10(mel_spec.T + 1e-10)
|
| 39 |
max_db = mel_spectrogram_db.max()
|
| 40 |
mel_spectrogram_db = mel_spectrogram_db - max_db
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
y=audio,
|
| 54 |
-
sr=sr,
|
| 55 |
-
n_fft=fft_size,
|
| 56 |
-
hop_length=hop_size,
|
| 57 |
-
win_length=frame_size,
|
| 58 |
-
n_mels=n_bands,
|
| 59 |
-
fmin=0,
|
| 60 |
-
fmax=sample_rate / 2,
|
| 61 |
-
window='hann'
|
| 62 |
-
)
|
| 63 |
-
|
| 64 |
-
mel_spectrogram_db = 10 * np.log10(mel_spec.T + 1e-10)
|
| 65 |
-
max_db = mel_spectrogram_db.max()
|
| 66 |
-
mel_spectrogram_db = mel_spectrogram_db - max_db
|
| 67 |
-
|
| 68 |
-
return [mel_spectrogram_db]
|
| 69 |
-
|
| 70 |
-
def pad(audio, target_seconds, sample_rate):
|
| 71 |
-
target_len = int(sample_rate * target_seconds)
|
| 72 |
-
n = len(audio)
|
| 73 |
-
|
| 74 |
-
if n < target_len:
|
| 75 |
-
audio = np.pad(audio, (0, target_len - n), mode="constant")
|
| 76 |
-
return audio
|
| 77 |
-
|
| 78 |
-
def time_stretch_augmentation(file_path, sample_rate, rate):
|
| 79 |
-
audio, _ = librosa.load(file_path, sr=sample_rate)
|
| 80 |
-
audio_timestretch = librosa.effects.time_stretch(audio.astype(np.float32), rate=rate)
|
| 81 |
-
return pad(audio_timestretch, 5, sample_rate)
|
| 82 |
-
|
| 83 |
-
def pitch_shift_augmentation(file_path, sample_rate, semitones):
|
| 84 |
-
audio, _ = librosa.load(file_path, sr=sample_rate)
|
| 85 |
-
return librosa.effects.pitch_shift(audio.astype(np.float32), sr=sample_rate, n_steps=semitones)
|
| 86 |
-
|
| 87 |
-
def drc_augmentation(file_path, sample_rate, compression):
|
| 88 |
-
if compression == "musicstandard": threshold_db=-20; ratio=2.0; attack_ms=5; release_ms=50
|
| 89 |
-
elif compression == "filmstandard": threshold_db=-25; ratio=4.0; attack_ms=10; release_ms= 100
|
| 90 |
-
elif compression == "speech": threshold_db=-18; ratio=3.0; attack_ms=2; release_ms= 40
|
| 91 |
-
elif compression == "radio": threshold_db=-15; ratio=3.5; attack_ms=1; release_ms= 200
|
| 92 |
-
|
| 93 |
-
audio, _ = librosa.load(file_path, sr=sample_rate)
|
| 94 |
-
threshold = 10**(threshold_db / 20)
|
| 95 |
-
|
| 96 |
-
attack_coeff = np.exp(-1.0 / (0.001 * attack_ms * sample_rate))
|
| 97 |
-
release_coeff = np.exp(-1.0 / (0.001 * release_ms * sample_rate))
|
| 98 |
-
|
| 99 |
-
audio_filtered = np.zeros_like(audio)
|
| 100 |
-
gain = 1.0
|
| 101 |
-
|
| 102 |
-
for n in range(len(audio)):
|
| 103 |
-
abs_audio = abs(audio[n])
|
| 104 |
-
if abs_audio > threshold:
|
| 105 |
-
desired_gain = (threshold / abs_audio) ** (ratio - 1)
|
| 106 |
-
else:
|
| 107 |
-
desired_gain = 1.0
|
| 108 |
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
|
| 114 |
-
|
| 115 |
|
| 116 |
-
|
| 117 |
|
| 118 |
-
def
|
| 119 |
-
|
|
|
|
| 120 |
|
| 121 |
-
|
| 122 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
|
| 124 |
-
|
|
|
|
| 125 |
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
if
|
| 134 |
-
|
| 135 |
-
semitone = np.random.choice(semitones)
|
| 136 |
-
audio = pitch_shift_augmentation(os.path.join(audio_path, filename), sample_rate, semitone)
|
| 137 |
-
# DRC
|
| 138 |
-
if np.random.rand() > p3:
|
| 139 |
-
compressions = ["radio", "filmstandard", "musicstandard", "speech"]
|
| 140 |
-
compression = np.random.choice(compressions)
|
| 141 |
-
audio = drc_augmentation(os.path.join(audio_path, filename), sample_rate, compression)
|
| 142 |
-
|
| 143 |
-
sf.write(os.path.join(output_path, filename), audio, 44100)
|
| 144 |
-
|
| 145 |
-
def create_augmented_datasets(input_path, output_path):
|
| 146 |
-
probability_lists = [
|
| 147 |
-
[0.0 , 1.0, 1.0],
|
| 148 |
-
[1.0 , 1.0, 0.0],
|
| 149 |
-
[1.0 , 0.0, 1.0],
|
| 150 |
-
[0.0 , 0.0, 0.0],
|
| 151 |
-
[0.5 , 0.5, 0.5]]
|
| 152 |
-
for i, probability_list in enumerate(probability_lists):
|
| 153 |
-
augmented_path = os.path.join(output_path, f"{i+1}")
|
| 154 |
-
os.makedirs(augmented_path, exist_ok=True)
|
| 155 |
-
augment_dataset(input_path, augmented_path, probability_list)
|
| 156 |
-
|
| 157 |
-
def create_log_mel(input_path, output_path):
|
| 158 |
-
directories = os.listdir(input_path)
|
| 159 |
-
X, y = [], []
|
| 160 |
-
|
| 161 |
-
for directory in directories:
|
| 162 |
-
log_mels, labels = data_treatment_training(os.path.join(input_path, directory), **parameters)
|
| 163 |
-
X.extend(log_mels)
|
| 164 |
-
y.extend(labels)
|
| 165 |
-
|
| 166 |
-
X_array = np.empty(len(X), dtype=object)
|
| 167 |
-
for i, spec in enumerate(X):
|
| 168 |
-
X_array[i] = spec
|
| 169 |
-
|
| 170 |
-
y = np.array(y)
|
| 171 |
-
os.makedirs(output_path, exist_ok=True)
|
| 172 |
-
|
| 173 |
-
np.save(os.path.join(output_path, "X.npy"), X_array, allow_pickle=True)
|
| 174 |
-
np.save(os.path.join(output_path, 'y.npy'), y)
|
| 175 |
-
return X, y
|
|
|
|
| 3 |
import numpy as np
|
| 4 |
import os
|
| 5 |
import soundfile as sf
|
| 6 |
+
from typing import Optional
|
| 7 |
|
| 8 |
+
from src.config.config import ProcessingConfig
|
| 9 |
+
|
| 10 |
+
config = ProcessingConfig()
|
| 11 |
+
|
| 12 |
+
class AudioAugment:
|
| 13 |
+
def __init__(self, config: ProcessingConfig = config) -> None:
|
| 14 |
+
self.config = config
|
| 15 |
+
|
| 16 |
+
def _mel_spectrogram(self, audio: np.ndarray) -> np.ndarray:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
mel_spec = librosa.feature.melspectrogram(
|
| 18 |
y=audio,
|
| 19 |
+
sr=self.config.sample_rate,
|
| 20 |
+
n_fft=self.config.fft_size,
|
| 21 |
+
hop_length=self.config.hop_size,
|
| 22 |
+
win_length=self.config.frame_size,
|
| 23 |
+
n_mels=self.config.n_bands,
|
| 24 |
fmin=0,
|
| 25 |
+
fmax=self.config.sample_rate / 2,
|
| 26 |
window='hann'
|
| 27 |
)
|
| 28 |
+
|
| 29 |
mel_spectrogram_db = 10 * np.log10(mel_spec.T + 1e-10)
|
| 30 |
max_db = mel_spectrogram_db.max()
|
| 31 |
mel_spectrogram_db = mel_spectrogram_db - max_db
|
| 32 |
+
|
| 33 |
+
return mel_spectrogram_db
|
| 34 |
+
|
| 35 |
+
def _data_treatment_training(self, audio_path: str) -> tuple[list[np.ndarray], np.ndarray]:
|
| 36 |
+
labels = []
|
| 37 |
+
log_mel_spectrograms = []
|
| 38 |
+
filenames = os.listdir(audio_path)
|
| 39 |
|
| 40 |
+
for filename in tqdm.tqdm(filenames, desc="Processing audio files"):
|
| 41 |
+
filename_splitted = filename.split("-")
|
| 42 |
+
label = filename_splitted[-1].split(".")[0]
|
| 43 |
+
label = label.split("_")[0]
|
| 44 |
+
labels.append(int(label))
|
| 45 |
+
|
| 46 |
+
file_path = os.path.join(audio_path, filename)
|
| 47 |
+
audio, sr = librosa.load(file_path, sr=self.config.sample_rate)
|
| 48 |
+
|
| 49 |
+
mel_spectrogram_db = self._mel_spectrogram(audio)
|
| 50 |
+
log_mel_spectrograms.append(mel_spectrogram_db)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
|
| 52 |
+
return log_mel_spectrograms, np.array(labels)
|
| 53 |
+
|
| 54 |
+
def _data_treatment_testing(self, file_path: str) -> list[np.ndarray]:
|
| 55 |
+
audio, sr = librosa.load(file_path, sr=self.config.sample_rate)
|
| 56 |
|
| 57 |
+
mel_spectrogram_db = self._mel_spectrogram(audio)
|
| 58 |
|
| 59 |
+
return [mel_spectrogram_db]
|
| 60 |
|
| 61 |
+
def _pad(self, audio: np.ndarray) -> np.ndarray:
|
| 62 |
+
target_len = int(self.config.sample_rate * self.config.target_seconds)
|
| 63 |
+
n = len(audio)
|
| 64 |
|
| 65 |
+
if n < target_len:
|
| 66 |
+
audio = np.pad(audio, (0, target_len - n), mode="constant")
|
| 67 |
+
return audio
|
| 68 |
+
|
| 69 |
+
def _time_stretch_augmentation(self, file_path: str, rate: float) -> np.ndarray:
|
| 70 |
+
audio, _ = librosa.load(file_path, sr=self.config.sample_rate)
|
| 71 |
+
audio_timestretch = librosa.effects.time_stretch(audio.astype(np.float32), rate=rate)
|
| 72 |
+
return self._pad(audio_timestretch)
|
| 73 |
+
|
| 74 |
+
def _pitch_shift_augmentation(self, file_path: str, semitones: float) -> np.ndarray:
|
| 75 |
+
audio, _ = librosa.load(file_path, sr=self.config.sample_rate)
|
| 76 |
+
return librosa.effects.pitch_shift(audio.astype(np.float32), sr=self.config.sample_rate, n_steps=semitones)
|
| 77 |
+
|
| 78 |
+
def _drc_augmentation(self, file_path: str, compression: float) -> np.ndarray:
|
| 79 |
+
if compression == "musicstandard": threshold_db=-20; ratio=2.0; attack_ms=5; release_ms=50
|
| 80 |
+
elif compression == "filmstandard": threshold_db=-25; ratio=4.0; attack_ms=10; release_ms= 100
|
| 81 |
+
elif compression == "speech": threshold_db=-18; ratio=3.0; attack_ms=2; release_ms= 40
|
| 82 |
+
elif compression == "radio": threshold_db=-15; ratio=3.5; attack_ms=1; release_ms= 200
|
| 83 |
+
|
| 84 |
+
audio, _ = librosa.load(file_path, sr=self.config.sample_rate)
|
| 85 |
+
threshold = 10**(threshold_db / 20)
|
| 86 |
+
|
| 87 |
+
attack_coeff = np.exp(-1.0 / (0.001 * attack_ms * self.config.sample_rate))
|
| 88 |
+
release_coeff = np.exp(-1.0 / (0.001 * release_ms * self.config.sample_rate))
|
| 89 |
+
|
| 90 |
+
audio_filtered = np.zeros_like(audio)
|
| 91 |
+
gain = 1.0
|
| 92 |
+
|
| 93 |
+
for n in range(len(audio)):
|
| 94 |
+
abs_audio = abs(audio[n])
|
| 95 |
+
if abs_audio > threshold:
|
| 96 |
+
desired_gain = (threshold / abs_audio) ** (ratio - 1)
|
| 97 |
+
else:
|
| 98 |
+
desired_gain = 1.0
|
| 99 |
+
|
| 100 |
+
if desired_gain < gain:
|
| 101 |
+
gain = attack_coeff * (gain - desired_gain) + desired_gain
|
| 102 |
+
else:
|
| 103 |
+
gain = release_coeff * (gain - desired_gain) + desired_gain
|
| 104 |
+
|
| 105 |
+
audio_filtered[n] = audio[n] * gain
|
| 106 |
+
|
| 107 |
+
return audio_filtered
|
| 108 |
+
|
| 109 |
+
def _augment_dataset(self, audio_path: str, output_path: str, probability_list: list[float]) -> None:
|
| 110 |
+
filenames = os.listdir(audio_path)
|
| 111 |
+
|
| 112 |
+
p1, p2, p3 = probability_list
|
| 113 |
+
os.makedirs(output_path, exist_ok=True)
|
| 114 |
+
|
| 115 |
+
for filename in tqdm.tqdm(filenames, desc="Augmenting audio files"):
|
| 116 |
+
|
| 117 |
+
audio, _ = librosa.load(os.path.join(audio_path, filename), sr=self.config.sample_rate)
|
| 118 |
+
# TS
|
| 119 |
+
if np.random.rand() > p1:
|
| 120 |
+
stretch_rates = [0.81, 0.93, 1.07, 1.23]
|
| 121 |
+
stretch_rate = np.random.choice(stretch_rates)
|
| 122 |
+
audio = self._time_stretch_augmentation(os.path.join(audio_path, filename), stretch_rate)
|
| 123 |
+
# PS
|
| 124 |
+
if np.random.rand() > p2:
|
| 125 |
+
semitones = [-3.5, -2.5, -2, -1, 1, 2.5, 3, 3.5]
|
| 126 |
+
semitone = np.random.choice(semitones)
|
| 127 |
+
audio = self._pitch_shift_augmentation(os.path.join(audio_path, filename), semitone)
|
| 128 |
+
# DRC
|
| 129 |
+
if np.random.rand() > p3:
|
| 130 |
+
compressions = ["radio", "filmstandard", "musicstandard", "speech"]
|
| 131 |
+
compression = np.random.choice(compressions)
|
| 132 |
+
audio = self._drc_augmentation(os.path.join(audio_path, filename), compression)
|
| 133 |
+
|
| 134 |
+
sf.write(os.path.join(output_path, filename), audio, self.config.sample_rate)
|
| 135 |
+
|
| 136 |
+
def _create_augmented_datasets(self, input_path: str, output_path: str) -> None:
|
| 137 |
+
probability_lists = self.config.augmentation_probability_lists
|
| 138 |
+
for i, probability_list in enumerate(probability_lists):
|
| 139 |
+
augmented_path = os.path.join(output_path, f"{i+1}")
|
| 140 |
+
os.makedirs(augmented_path, exist_ok=True)
|
| 141 |
+
self._augment_dataset(input_path, augmented_path, probability_list)
|
| 142 |
+
|
| 143 |
+
def _create_log_mel(self, input_path: str, output_path: str) -> tuple[list[np.ndarray], np.ndarray]:
|
| 144 |
+
directories = os.listdir(input_path)
|
| 145 |
+
X, y = [], []
|
| 146 |
+
|
| 147 |
+
for directory in directories:
|
| 148 |
+
log_mels, labels = self._data_treatment_training(os.path.join(input_path, directory))
|
| 149 |
+
X.extend(log_mels)
|
| 150 |
+
y.extend(labels)
|
| 151 |
+
|
| 152 |
+
X_array = np.empty(len(X), dtype=object)
|
| 153 |
+
for i, spec in enumerate(X):
|
| 154 |
+
X_array[i] = spec
|
| 155 |
|
| 156 |
+
y = np.array(y)
|
| 157 |
+
os.makedirs(output_path, exist_ok=True)
|
| 158 |
|
| 159 |
+
np.save(os.path.join(output_path, "X.npy"), X_array, allow_pickle=True)
|
| 160 |
+
np.save(os.path.join(output_path, 'y.npy'), y)
|
| 161 |
+
return X, y
|
| 162 |
+
|
| 163 |
+
def run(self, augment: bool = True, preprocess : bool = True) -> None:
|
| 164 |
+
if augment:
|
| 165 |
+
self._create_augmented_datasets(self.config.audio_path, self.config.augmented_path)
|
| 166 |
+
if preprocess:
|
| 167 |
+
self._create_log_mel(self.config.augmented_path, self.config.log_mel_path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/data/dataset.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import numpy as np
|
| 3 |
+
from torch.utils.data import Dataset
|
| 4 |
+
from typing import Sequence
|
| 5 |
+
|
| 6 |
+
from src.config.config import DatasetConfig
|
| 7 |
+
|
| 8 |
+
config = DatasetConfig()
|
| 9 |
+
|
| 10 |
+
class FullTFPatchesDataset(Dataset):
|
| 11 |
+
def __init__(self, spectrograms: Sequence[np.ndarray], labels: Sequence[int], config: DatasetConfig = config) -> None:
|
| 12 |
+
self.config = config
|
| 13 |
+
self.patch_indices = []
|
| 14 |
+
|
| 15 |
+
for spec_idx, spec in enumerate(spectrograms):
|
| 16 |
+
n_frames = spec.shape[0]
|
| 17 |
+
label = labels[spec_idx]
|
| 18 |
+
|
| 19 |
+
if n_frames >= self.config.cnn_input_length:
|
| 20 |
+
for start_frame in range(n_frames - self.config.cnn_input_length + 1):
|
| 21 |
+
self.patch_indices.append((spec_idx, start_frame, label))
|
| 22 |
+
else:
|
| 23 |
+
self.patch_indices.append((spec_idx, 0, label))
|
| 24 |
+
|
| 25 |
+
self.spectrograms = spectrograms
|
| 26 |
+
|
| 27 |
+
def __len__(self) -> int:
|
| 28 |
+
return len(self.patch_indices)
|
| 29 |
+
|
| 30 |
+
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 31 |
+
spec_idx, start_frame, label = self.patch_indices[idx]
|
| 32 |
+
spec = self.spectrograms[spec_idx]
|
| 33 |
+
|
| 34 |
+
n_frames = spec.shape[0]
|
| 35 |
+
|
| 36 |
+
if n_frames >= self.config.cnn_input_length:
|
| 37 |
+
patch = spec[start_frame:start_frame + self.config.cnn_input_length]
|
| 38 |
+
else:
|
| 39 |
+
pad = self.config.cnn_input_length - n_frames
|
| 40 |
+
patch = np.pad(spec, ((0, pad), (0, 0)), mode='constant')
|
| 41 |
+
|
| 42 |
+
patch = patch[np.newaxis, :, :]
|
| 43 |
+
|
| 44 |
+
return torch.tensor(patch, dtype=torch.float32), torch.tensor(label, dtype=torch.long)
|
| 45 |
+
|
| 46 |
+
class RandomPatchDataset(Dataset):
|
| 47 |
+
def __init__(self, spectrograms: Sequence[np.ndarray], labels: Sequence[int], config: DatasetConfig = config) -> None:
|
| 48 |
+
self.config = config
|
| 49 |
+
self.spectrograms = spectrograms
|
| 50 |
+
self.labels = labels
|
| 51 |
+
|
| 52 |
+
def __len__(self) -> int:
|
| 53 |
+
return len(self.labels)
|
| 54 |
+
|
| 55 |
+
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 56 |
+
spec = self.spectrograms[idx]
|
| 57 |
+
label = self.labels[idx]
|
| 58 |
+
n_frames = spec.shape[0]
|
| 59 |
+
|
| 60 |
+
if n_frames >= self.config.cnn_input_length:
|
| 61 |
+
start = np.random.randint(0, n_frames - self.config.cnn_input_length + 1)
|
| 62 |
+
patch = spec[start:start + self.config.cnn_input_length]
|
| 63 |
+
else:
|
| 64 |
+
pad = self.config.cnn_input_length - n_frames
|
| 65 |
+
patch = np.pad(spec, ((0, pad), (0, 0)), mode='constant')
|
| 66 |
+
|
| 67 |
+
patch = patch[np.newaxis, :, :]
|
| 68 |
+
return torch.tensor(patch, dtype=torch.float32), torch.tensor(label, dtype=torch.long)
|
src/data/download.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
| 1 |
import requests
|
| 2 |
import zipfile
|
| 3 |
-
import tarfile
|
| 4 |
import io
|
| 5 |
import os
|
| 6 |
import shutil
|
|
@@ -13,11 +12,7 @@ from src.config.config import DownloadConfig
|
|
| 13 |
config = DownloadConfig()
|
| 14 |
|
| 15 |
class ESC50Downloader:
|
| 16 |
-
def __init__(
|
| 17 |
-
self,
|
| 18 |
-
repo_url: str = config.repo_url,
|
| 19 |
-
repo_dst_dir: str = config.repo_dst_dir
|
| 20 |
-
):
|
| 21 |
self.repo_url = repo_url
|
| 22 |
self.repo_dst_dir = Path(repo_dst_dir)
|
| 23 |
self.audio_dst_dir = config.audio_dst_dir
|
|
@@ -25,7 +20,7 @@ class ESC50Downloader:
|
|
| 25 |
self.extracted_dir = config.extracted_dir
|
| 26 |
self.audio_src_dir = config.audio_src_dir
|
| 27 |
|
| 28 |
-
def download_and_extract(self):
|
| 29 |
os.makedirs(self.repo_dst_dir, exist_ok=True)
|
| 30 |
print(f"Downloading from {self.repo_url}")
|
| 31 |
|
|
@@ -46,7 +41,7 @@ class ESC50Downloader:
|
|
| 46 |
z.extractall(self.repo_dst_dir)
|
| 47 |
print("Done extracting.")
|
| 48 |
|
| 49 |
-
def clean_files(self):
|
| 50 |
for f in self.paths_to_delete:
|
| 51 |
path = os.path.join(self.extracted_dir, f)
|
| 52 |
if os.path.isfile(path):
|
|
@@ -56,7 +51,7 @@ class ESC50Downloader:
|
|
| 56 |
shutil.rmtree(path)
|
| 57 |
print(f"Deleted directory: {path}")
|
| 58 |
|
| 59 |
-
def move_audio_files(self):
|
| 60 |
os.makedirs(self.audio_dst_dir, exist_ok=True)
|
| 61 |
print(f"Moving audio files from {self.audio_src_dir} to {self.audio_dst_dir}")
|
| 62 |
|
|
@@ -67,11 +62,7 @@ class ESC50Downloader:
|
|
| 67 |
shutil.move(src_file, dst_file)
|
| 68 |
print(f"Moved all audio files to {self.audio_dst_dir}")
|
| 69 |
|
| 70 |
-
def download_clean(self):
|
| 71 |
self.download_and_extract()
|
| 72 |
self.clean_files()
|
| 73 |
self.move_audio_files()
|
| 74 |
-
|
| 75 |
-
if __name__ == "__main__":
|
| 76 |
-
downloader = ESC50Downloader()
|
| 77 |
-
downloader.download_clean()
|
|
|
|
| 1 |
import requests
|
| 2 |
import zipfile
|
|
|
|
| 3 |
import io
|
| 4 |
import os
|
| 5 |
import shutil
|
|
|
|
| 12 |
config = DownloadConfig()
|
| 13 |
|
| 14 |
class ESC50Downloader:
|
| 15 |
+
def __init__(self, repo_url: str = config.repo_url, repo_dst_dir: str = config.repo_dst_dir) -> None:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
self.repo_url = repo_url
|
| 17 |
self.repo_dst_dir = Path(repo_dst_dir)
|
| 18 |
self.audio_dst_dir = config.audio_dst_dir
|
|
|
|
| 20 |
self.extracted_dir = config.extracted_dir
|
| 21 |
self.audio_src_dir = config.audio_src_dir
|
| 22 |
|
| 23 |
+
def download_and_extract(self) -> None:
|
| 24 |
os.makedirs(self.repo_dst_dir, exist_ok=True)
|
| 25 |
print(f"Downloading from {self.repo_url}")
|
| 26 |
|
|
|
|
| 41 |
z.extractall(self.repo_dst_dir)
|
| 42 |
print("Done extracting.")
|
| 43 |
|
| 44 |
+
def clean_files(self) -> None:
|
| 45 |
for f in self.paths_to_delete:
|
| 46 |
path = os.path.join(self.extracted_dir, f)
|
| 47 |
if os.path.isfile(path):
|
|
|
|
| 51 |
shutil.rmtree(path)
|
| 52 |
print(f"Deleted directory: {path}")
|
| 53 |
|
| 54 |
+
def move_audio_files(self) -> None:
|
| 55 |
os.makedirs(self.audio_dst_dir, exist_ok=True)
|
| 56 |
print(f"Moving audio files from {self.audio_src_dir} to {self.audio_dst_dir}")
|
| 57 |
|
|
|
|
| 62 |
shutil.move(src_file, dst_file)
|
| 63 |
print(f"Moved all audio files to {self.audio_dst_dir}")
|
| 64 |
|
| 65 |
+
def download_clean(self) -> None:
|
| 66 |
self.download_and_extract()
|
| 67 |
self.clean_files()
|
| 68 |
self.move_audio_files()
|
|
|
|
|
|
|
|
|
|
|
|
src/models/cnn.py
CHANGED
|
@@ -1,19 +1,18 @@
|
|
| 1 |
import torch.nn as nn
|
|
|
|
| 2 |
|
| 3 |
class CNN(nn.Module):
|
| 4 |
-
def __init__(self, n_classes=50):
|
| 5 |
super().__init__()
|
| 6 |
self.features = nn.Sequential(
|
| 7 |
nn.Conv2d(1, 24, kernel_size=(5, 5)),
|
| 8 |
nn.ReLU(),
|
| 9 |
nn.MaxPool2d(kernel_size=(4, 2), stride=(4, 2)),
|
| 10 |
|
| 11 |
-
|
| 12 |
nn.Conv2d(24, 48, kernel_size=(5, 5)),
|
| 13 |
nn.ReLU(),
|
| 14 |
nn.MaxPool2d(kernel_size=(4, 2), stride=(4, 2)),
|
| 15 |
|
| 16 |
-
|
| 17 |
nn.Conv2d(48, 48, kernel_size=(5, 5)),
|
| 18 |
nn.ReLU(),
|
| 19 |
)
|
|
@@ -25,8 +24,7 @@ class CNN(nn.Module):
|
|
| 25 |
nn.Linear(64, n_classes)
|
| 26 |
)
|
| 27 |
|
| 28 |
-
|
| 29 |
-
def forward(self, x):
|
| 30 |
x = self.features(x)
|
| 31 |
x = x.flatten(1)
|
| 32 |
return self.classifier(x)
|
|
|
|
| 1 |
import torch.nn as nn
|
| 2 |
+
import torch
|
| 3 |
|
| 4 |
class CNN(nn.Module):
|
| 5 |
+
def __init__(self, n_classes: int = 50) -> None:
|
| 6 |
super().__init__()
|
| 7 |
self.features = nn.Sequential(
|
| 8 |
nn.Conv2d(1, 24, kernel_size=(5, 5)),
|
| 9 |
nn.ReLU(),
|
| 10 |
nn.MaxPool2d(kernel_size=(4, 2), stride=(4, 2)),
|
| 11 |
|
|
|
|
| 12 |
nn.Conv2d(24, 48, kernel_size=(5, 5)),
|
| 13 |
nn.ReLU(),
|
| 14 |
nn.MaxPool2d(kernel_size=(4, 2), stride=(4, 2)),
|
| 15 |
|
|
|
|
| 16 |
nn.Conv2d(48, 48, kernel_size=(5, 5)),
|
| 17 |
nn.ReLU(),
|
| 18 |
)
|
|
|
|
| 24 |
nn.Linear(64, n_classes)
|
| 25 |
)
|
| 26 |
|
| 27 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
|
|
| 28 |
x = self.features(x)
|
| 29 |
x = x.flatten(1)
|
| 30 |
return self.classifier(x)
|
src/models/predict.py
CHANGED
|
@@ -1,193 +1,74 @@
|
|
| 1 |
import numpy as np
|
| 2 |
import torch
|
| 3 |
import torch.nn as nn
|
|
|
|
| 4 |
import argparse
|
| 5 |
-
import os
|
| 6 |
-
import sys
|
| 7 |
|
| 8 |
from src.models.cnn import CNN
|
| 9 |
-
from src.data.augment import
|
| 10 |
-
from src.config.config import
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
def
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
patches = torch.tensor(patches, dtype=torch.float32).to(device)
|
| 63 |
-
|
| 64 |
-
all_outputs = []
|
| 65 |
-
with torch.no_grad():
|
| 66 |
-
for i in range(0, len(patches), batch_size):
|
| 67 |
-
batch = patches[i:i + batch_size]
|
| 68 |
-
outputs = model(batch)
|
| 69 |
-
all_outputs.append(outputs)
|
| 70 |
-
|
| 71 |
-
all_outputs = torch.cat(all_outputs, dim=0)
|
| 72 |
-
mean_logits = all_outputs.mean(dim=0)
|
| 73 |
-
probabilities = torch.nn.functional.softmax(mean_logits, dim=0)
|
| 74 |
-
|
| 75 |
-
top_probs, top_indices = torch.topk(probabilities, min(top_k, 50))
|
| 76 |
-
top_probs = top_probs.cpu().numpy()
|
| 77 |
-
top_indices = top_indices.cpu().numpy()
|
| 78 |
-
|
| 79 |
-
return top_probs, top_indices
|
| 80 |
|
| 81 |
-
def predict_file(model, audio_file, device="cpu", top_k=5):
|
| 82 |
-
parameters = {
|
| 83 |
-
"n_bands" : 128,
|
| 84 |
-
"n_mels" : 128,
|
| 85 |
-
"frame_size" : 1024,
|
| 86 |
-
"hop_size": 1024,
|
| 87 |
-
"sample_rate": sample_rate,
|
| 88 |
-
"fft_size": 8192,
|
| 89 |
-
}
|
| 90 |
-
spectrogram = data_treatment_testing(audio_file, **parameters)
|
| 91 |
-
|
| 92 |
-
spectrogram = np.array(spectrogram)
|
| 93 |
-
|
| 94 |
-
spectrogram = spectrogram.squeeze()
|
| 95 |
-
|
| 96 |
-
predicted_class = predict_with_overlapping_patches(
|
| 97 |
-
model, spectrogram, patch_length=128, hop=1, batch_size=100, device=device
|
| 98 |
-
)
|
| 99 |
-
top_probs, top_indices = predict_top_k(
|
| 100 |
-
model, spectrogram, patch_length=128, hop=1, batch_size=100, device=device, top_k=top_k
|
| 101 |
-
)
|
| 102 |
-
|
| 103 |
-
return predicted_class, top_probs, top_indices
|
| 104 |
-
|
| 105 |
-
def load_model(model_path, device='cpu'):
|
| 106 |
-
print(f"Loading model from {model_path}...")
|
| 107 |
-
|
| 108 |
-
model = CNN(n_classes=50)
|
| 109 |
-
checkpoint = torch.load(model_path, map_location=device)
|
| 110 |
-
|
| 111 |
-
if isinstance(checkpoint, dict):
|
| 112 |
-
if 'model_state_dict' in checkpoint:
|
| 113 |
-
model.load_state_dict(checkpoint['model_state_dict'])
|
| 114 |
-
if 'best_val_acc' in checkpoint:
|
| 115 |
-
print(f"Model validation accuracy: {checkpoint['best_val_acc']:.4f}")
|
| 116 |
-
else:
|
| 117 |
-
model.load_state_dict(checkpoint)
|
| 118 |
-
else:
|
| 119 |
-
model.load_state_dict(checkpoint)
|
| 120 |
-
|
| 121 |
-
model.to(device)
|
| 122 |
-
model.eval()
|
| 123 |
-
print("Model loaded successfully!\n")
|
| 124 |
-
return model
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
def main():
|
| 129 |
-
parser = argparse.ArgumentParser(
|
| 130 |
-
description='Predict environmental sound class using trained ESC-50 model'
|
| 131 |
-
)
|
| 132 |
-
parser.add_argument(
|
| 133 |
-
'audio_file',
|
| 134 |
-
type=str,
|
| 135 |
-
help='Path to .wav file to classify'
|
| 136 |
-
)
|
| 137 |
-
parser.add_argument(
|
| 138 |
-
'--model',
|
| 139 |
-
type=str,
|
| 140 |
-
default='best_model.pt',
|
| 141 |
-
help='Path to trained model checkpoint (default: best_model.pt)'
|
| 142 |
-
)
|
| 143 |
-
parser.add_argument(
|
| 144 |
-
'--top-k',
|
| 145 |
-
type=int,
|
| 146 |
-
default=5,
|
| 147 |
-
help='Number of top predictions to show (default: 5)'
|
| 148 |
-
)
|
| 149 |
-
parser.add_argument(
|
| 150 |
-
'--device',
|
| 151 |
-
type=str,
|
| 152 |
-
default='cuda' if torch.cuda.is_available() else 'cpu',
|
| 153 |
-
help='Device to use (default: auto-detect)'
|
| 154 |
-
)
|
| 155 |
-
|
| 156 |
-
args = parser.parse_args()
|
| 157 |
-
|
| 158 |
-
if not os.path.exists(args.audio_file):
|
| 159 |
-
print(f"Error: Audio file not found: {args.audio_file}")
|
| 160 |
-
sys.exit(1)
|
| 161 |
-
|
| 162 |
-
if not os.path.exists(args.model):
|
| 163 |
-
print(f"Error: Model file not found: {args.model}")
|
| 164 |
-
sys.exit(1)
|
| 165 |
-
|
| 166 |
-
try:
|
| 167 |
-
model = load_model(args.model, device=args.device)
|
| 168 |
-
except Exception as e:
|
| 169 |
-
print(f"Error loading model: {e}")
|
| 170 |
-
import traceback
|
| 171 |
-
traceback.print_exc()
|
| 172 |
-
sys.exit(1)
|
| 173 |
-
|
| 174 |
-
try:
|
| 175 |
-
predicted_class, top_probs, top_indices = predict_file(
|
| 176 |
-
model, args.audio_file, device=args.device, top_k=args.top_k
|
| 177 |
-
)
|
| 178 |
-
|
| 179 |
-
print("\n" + "=" * 60)
|
| 180 |
-
print(f"Top {args.top_k} Predictions:")
|
| 181 |
-
print("=" * 60)
|
| 182 |
-
|
| 183 |
-
for i, (prob, idx) in enumerate(zip(top_probs, top_indices)):
|
| 184 |
-
class_name = esc50_labels[idx]
|
| 185 |
-
marker = "★" if idx == predicted_class else " "
|
| 186 |
-
print(f"{marker} {i+1}. {class_name:20s} - {prob*100:6.2f}%")
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
except Exception as e:
|
| 190 |
-
print(f"\nError during prediction: {e}")
|
| 191 |
-
import traceback
|
| 192 |
-
traceback.print_exc()
|
| 193 |
-
sys.exit(1)
|
|
|
|
| 1 |
import numpy as np
|
| 2 |
import torch
|
| 3 |
import torch.nn as nn
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
import argparse
|
|
|
|
|
|
|
| 6 |
|
| 7 |
from src.models.cnn import CNN
|
| 8 |
+
from src.data.augment import AudioAugment
|
| 9 |
+
from src.config.config import ProcessingConfig, DatasetConfig, TrainConfig
|
| 10 |
+
|
| 11 |
+
config = ProcessingConfig()
|
| 12 |
+
|
| 13 |
+
class AudioPredictor:
|
| 14 |
+
def __init__(
|
| 15 |
+
self,
|
| 16 |
+
model_path: str,
|
| 17 |
+
config: ProcessingConfig = config,
|
| 18 |
+
device: str = 'cuda'
|
| 19 |
+
) -> None:
|
| 20 |
+
self.config = config
|
| 21 |
+
self.audio_dataset = AudioAugment()
|
| 22 |
+
self.dataset_config = DatasetConfig()
|
| 23 |
+
self.train_config = TrainConfig()
|
| 24 |
+
self.device = device
|
| 25 |
+
self.model = self._load_model(model_path)
|
| 26 |
+
|
| 27 |
+
def _load_model(self, model_path: str) -> CNN:
|
| 28 |
+
model = CNN(n_classes=len(self.dataset_config.esc50_labels))
|
| 29 |
+
checkpoint = torch.load(model_path, map_location=self.device)
|
| 30 |
+
state_dict = checkpoint.get("model_state_dict", checkpoint) if isinstance(checkpoint, dict) else checkpoint
|
| 31 |
+
model.load_state_dict(state_dict)
|
| 32 |
+
if isinstance(checkpoint, dict) and "best_val_acc" in checkpoint:
|
| 33 |
+
print(f"Model validation accuracy: {checkpoint['best_val_acc']:.4f}")
|
| 34 |
+
model.to(self.device).eval()
|
| 35 |
+
print("Model loaded successfully!\n")
|
| 36 |
+
return model
|
| 37 |
+
|
| 38 |
+
def _extract_patches(self, spectrogram: np.ndarray, hop: int) -> torch.Tensor:
|
| 39 |
+
n_frames, _ = spectrogram.shape
|
| 40 |
+
if n_frames < self.dataset_config.cnn_input_length:
|
| 41 |
+
spectrogram = np.pad(spectrogram, ((0, self.dataset_config.cnn_input_length - n_frames), (0, 0)), mode="constant")
|
| 42 |
+
n_frames = self.dataset_config.cnn_input_length
|
| 43 |
+
|
| 44 |
+
patches = np.concatenate([
|
| 45 |
+
spectrogram[s:s + self.dataset_config.cnn_input_length][np.newaxis, np.newaxis]
|
| 46 |
+
for s in range(0, n_frames - self.dataset_config.cnn_input_length + 1, hop)
|
| 47 |
+
], axis=0)
|
| 48 |
+
return torch.tensor(patches, dtype=torch.float32).to(self.device)
|
| 49 |
|
| 50 |
+
def _run_inference(self, patches: torch.Tensor, batch_size: int) -> torch.Tensor:
|
| 51 |
+
all_outputs = []
|
| 52 |
+
with torch.no_grad():
|
| 53 |
+
for i in range(0, len(patches), batch_size):
|
| 54 |
+
all_outputs.append(self.model(patches[i:i + batch_size]))
|
| 55 |
+
return torch.cat(all_outputs, dim=0).mean(dim=0)
|
| 56 |
+
|
| 57 |
+
def predict_class(self, spectrogram: np.ndarray, hop: int = 1) -> int:
|
| 58 |
+
patches = self._extract_patches(spectrogram, hop)
|
| 59 |
+
mean_activations = self._run_inference(patches, self.train_config.batch_size)
|
| 60 |
+
return mean_activations.argmax().item()
|
| 61 |
+
|
| 62 |
+
def predict_top_k(self, spectrogram: np.ndarray, hop: int = 1, top_k: int = 5):
|
| 63 |
+
patches = self._extract_patches(spectrogram, hop)
|
| 64 |
+
mean_logits = self._run_inference(patches, self.train_config.batch_size)
|
| 65 |
+
probs = F.softmax(mean_logits, dim=0)
|
| 66 |
+
top_probs, top_indices = torch.topk(probs, min(top_k, len(self.dataset_config.esc50_labels)))
|
| 67 |
+
return top_probs.cpu().numpy(), top_indices.cpu().numpy()
|
| 68 |
+
|
| 69 |
+
def predict_file(self, audio_file: str, top_k: int = 5):
|
| 70 |
+
spectrogram = np.array(self.audio_dataset._data_treatment_testing(audio_file)).squeeze()
|
| 71 |
+
predicted_class = self.predict_class(spectrogram)
|
| 72 |
+
top_probs, top_indices = self.predict_top_k(spectrogram, top_k=top_k)
|
| 73 |
+
return predicted_class, top_probs, top_indices
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/models/traincnn.py
CHANGED
|
@@ -4,293 +4,244 @@ import tqdm
|
|
| 4 |
import json
|
| 5 |
import numpy as np
|
| 6 |
from torch.utils.data import DataLoader
|
|
|
|
| 7 |
|
| 8 |
-
from src.models.predict import
|
| 9 |
from src.data.dataset import FullTFPatchesDataset, RandomPatchDataset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
X_val, y_val,
|
| 15 |
-
fold_num,
|
| 16 |
-
epochs=50,
|
| 17 |
-
batch_size=100,
|
| 18 |
-
lr=0.001,
|
| 19 |
-
device="cuda",
|
| 20 |
-
use_all_patches=True,
|
| 21 |
-
samples_per_epoch_fraction=1/8,
|
| 22 |
-
checkpoint_dir="models/checkpoints",
|
| 23 |
-
save_every_n_epoch=1,
|
| 24 |
-
resume_from=None
|
| 25 |
-
):
|
| 26 |
-
os.makedirs(checkpoint_dir, exist_ok=True)
|
| 27 |
-
|
| 28 |
-
model.to(device)
|
| 29 |
-
|
| 30 |
-
if use_all_patches:
|
| 31 |
-
train_dataset = FullTFPatchesDataset(X_train, y_train, patch_length=128)
|
| 32 |
-
print(f"\n{'='*60}")
|
| 33 |
-
print("Using ALL PATCHES method (as per paper)")
|
| 34 |
-
print(f"{'='*60}")
|
| 35 |
-
else:
|
| 36 |
-
train_dataset = RandomPatchDataset(X_train, y_train, patch_length=128)
|
| 37 |
-
print(f"\n{'='*60}")
|
| 38 |
-
print("Using RANDOM PATCHES method (simpler)")
|
| 39 |
-
print(f"{'='*60}")
|
| 40 |
-
|
| 41 |
-
# unique, counts = np.unique(y_train, return_counts=True)
|
| 42 |
-
# print(f"\nClass distribution in y_train:")
|
| 43 |
-
# print(f"Classes: {len(unique)}")
|
| 44 |
-
# print(f"Min samples: {counts.min()}, Max samples: {counts.max()}, Mean: {counts.mean():.1f}")
|
| 45 |
-
# print(f"\nPer-class counts:")
|
| 46 |
-
# for cls, count in zip(unique, counts):
|
| 47 |
-
# print(f"Class {cls}: {count}")
|
| 48 |
-
|
| 49 |
-
train_loader = DataLoader(
|
| 50 |
-
train_dataset,
|
| 51 |
-
batch_size=batch_size,
|
| 52 |
-
shuffle=True,
|
| 53 |
-
num_workers=4,
|
| 54 |
-
pin_memory=True
|
| 55 |
-
)
|
| 56 |
-
|
| 57 |
-
total_patches = len(train_dataset)
|
| 58 |
-
patches_per_epoch = int(total_patches * samples_per_epoch_fraction)
|
| 59 |
-
batches_per_epoch = patches_per_epoch // batch_size
|
| 60 |
-
|
| 61 |
-
print(f"Total available patches: {total_patches:,}")
|
| 62 |
-
print(f"Patches per epoch ({samples_per_epoch_fraction}): {patches_per_epoch:,}")
|
| 63 |
-
print(f"Batches per epoch: {batches_per_epoch:,}")
|
| 64 |
-
print(f"{'='*60}\n")
|
| 65 |
-
|
| 66 |
-
criterion = torch.nn.CrossEntropyLoss()
|
| 67 |
-
optimizer = torch.optim.AdamW([
|
| 68 |
-
{'params': model.features.parameters(), 'weight_decay': 0.0},
|
| 69 |
-
{'params': model.classifier.parameters(), 'weight_decay': 0.001}
|
| 70 |
-
], lr=lr)#, momentum=0.9)
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
start_epoch = 0
|
| 74 |
-
best_val_acc = 0.0
|
| 75 |
-
training_history = {
|
| 76 |
-
'train_loss': [],
|
| 77 |
-
'train_acc': [],
|
| 78 |
-
'val_acc': [],
|
| 79 |
-
'epochs': []
|
| 80 |
-
}
|
| 81 |
-
|
| 82 |
-
if resume_from and os.path.exists(resume_from):
|
| 83 |
-
print(f"Resuming from checkpoint: {resume_from}")
|
| 84 |
-
checkpoint = torch.load(resume_from, map_location=device)
|
| 85 |
-
|
| 86 |
-
model.load_state_dict(checkpoint['model_state_dict'])
|
| 87 |
-
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
|
| 88 |
-
start_epoch = checkpoint['epoch'] + 1
|
| 89 |
-
best_val_acc = checkpoint['best_val_acc']
|
| 90 |
-
training_history = checkpoint['history']
|
| 91 |
-
|
| 92 |
-
print(f"Resuming training from epoch: {checkpoint['epoch']}")
|
| 93 |
-
print(f"Best val acc: {best_val_acc:.4f}\n")
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
for epoch in range(start_epoch, epochs):
|
| 98 |
-
model.train()
|
| 99 |
-
train_loss = 0.0
|
| 100 |
-
correct = 0
|
| 101 |
-
total = 0
|
| 102 |
-
batches_processed = 0
|
| 103 |
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
|
| 108 |
-
xb
|
| 109 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
loss = criterion(out, yb)
|
| 115 |
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
optimizer.step()
|
| 120 |
-
|
| 121 |
-
train_loss += loss.item() * xb.size(0)
|
| 122 |
-
_, pred = out.max(1)
|
| 123 |
-
correct += (pred == yb).sum().item()
|
| 124 |
-
total += yb.size(0)
|
| 125 |
-
batches_processed += 1
|
| 126 |
-
|
| 127 |
-
train_loss /= total
|
| 128 |
-
train_acc = correct / total
|
| 129 |
-
|
| 130 |
-
model.eval()
|
| 131 |
-
val_correct = 0
|
| 132 |
-
val_total = len(y_val)
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
for i in tqdm.tqdm(range(val_total), desc=f"Epoch {epoch+1} Val", leave=False):
|
| 136 |
-
spec = X_val[i]
|
| 137 |
-
true_label = y_val[i]
|
| 138 |
|
| 139 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
-
|
| 142 |
-
val_correct += 1
|
| 143 |
-
|
| 144 |
-
val_acc = val_correct / val_total
|
| 145 |
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
|
| 151 |
-
|
| 152 |
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
|
|
|
| 178 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
}
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
)
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
def train_k_fold_cnn(
|
| 216 |
-
model_class,
|
| 217 |
-
X, y,
|
| 218 |
-
epochs=50,
|
| 219 |
-
batch_size=100,
|
| 220 |
-
lr=0.01,
|
| 221 |
-
k_fold=5,
|
| 222 |
-
device="cuda",
|
| 223 |
-
use_all_patches=True,
|
| 224 |
-
samples_per_epoch_fraction=1/8,
|
| 225 |
-
checkpoint_dir="models/checkpoints",
|
| 226 |
-
save_every_n_epoch=1
|
| 227 |
-
):
|
| 228 |
-
|
| 229 |
-
X = np.array(X)
|
| 230 |
-
y = np.array(y)
|
| 231 |
-
n_samples = len(y)
|
| 232 |
-
indices = np.arange(n_samples)
|
| 233 |
-
np.random.shuffle(indices)
|
| 234 |
-
|
| 235 |
-
fold_sizes = (n_samples // 5) * np.ones(5, dtype=int)
|
| 236 |
-
fold_sizes[:n_samples % 5] += 1
|
| 237 |
-
current = 0
|
| 238 |
-
|
| 239 |
-
fold_accuracies = []
|
| 240 |
-
|
| 241 |
-
for fold_num, fold_size in enumerate(fold_sizes, 1):
|
| 242 |
-
start, stop = current, current + fold_size
|
| 243 |
-
val_idx = indices[start:stop]
|
| 244 |
-
train_idx = np.concatenate([indices[:start], indices[stop:]])
|
| 245 |
-
current = stop
|
| 246 |
-
|
| 247 |
-
X_train, y_train = X[train_idx].tolist(), y[train_idx]
|
| 248 |
-
X_val, y_val = X[val_idx].tolist(), y[val_idx]
|
| 249 |
-
|
| 250 |
-
print(f"\n{'='*80}")
|
| 251 |
-
print(f"FOLD {fold_num}/5 | Train: {len(X_train)}, Val: {len(X_val)}")
|
| 252 |
-
print(f"{'='*80}\n")
|
| 253 |
-
|
| 254 |
-
model = model_class()
|
| 255 |
-
|
| 256 |
-
best_acc = train_cnn(
|
| 257 |
-
model=model,
|
| 258 |
-
X_train=X_train,
|
| 259 |
-
y_train=y_train,
|
| 260 |
-
X_val=X_val,
|
| 261 |
-
y_val=y_val,
|
| 262 |
-
fold_num=fold_num,
|
| 263 |
-
epochs=epochs,
|
| 264 |
-
batch_size=batch_size,
|
| 265 |
-
lr=lr,
|
| 266 |
-
device=device,
|
| 267 |
-
use_all_patches=use_all_patches,
|
| 268 |
-
samples_per_epoch_fraction=samples_per_epoch_fraction,
|
| 269 |
-
checkpoint_dir=os.path.join(checkpoint_dir, f"fold_{fold_num}"),
|
| 270 |
-
save_every_n_epoch=save_every_n_epoch
|
| 271 |
-
)
|
| 272 |
-
fold_accuracies.append(best_acc)
|
| 273 |
-
|
| 274 |
-
print(f"\nFold {fold_num} Best Accuracy: {best_acc:.4f}\n")
|
| 275 |
-
|
| 276 |
-
mean_acc = np.mean(fold_accuracies)
|
| 277 |
-
std_acc = np.std(fold_accuracies)
|
| 278 |
-
|
| 279 |
-
print(f"\n{'='*80}")
|
| 280 |
-
print("FINAL 5-FOLD CROSS-VALIDATION RESULTS")
|
| 281 |
-
print(f"Fold Accuracies: {fold_accuracies}")
|
| 282 |
-
print(f"Mean Accuracy: {mean_acc:.4f} ± {std_acc:.4f}")
|
| 283 |
-
print(f"{'='*80}\n")
|
| 284 |
-
|
| 285 |
-
# Save results
|
| 286 |
-
results_path = os.path.join(checkpoint_dir, "5fold_cv_results.json")
|
| 287 |
-
os.makedirs(checkpoint_dir, exist_ok=True)
|
| 288 |
-
with open(results_path, 'w') as f:
|
| 289 |
-
json.dump({
|
| 290 |
-
'fold_accuracies': fold_accuracies,
|
| 291 |
-
'mean_accuracy': mean_acc,
|
| 292 |
-
'std_accuracy': std_acc
|
| 293 |
-
}, f, indent=2)
|
| 294 |
-
print(f"Results saved to {results_path}")
|
| 295 |
-
|
| 296 |
-
return fold_accuracies, mean_acc
|
|
|
|
| 4 |
import json
|
| 5 |
import numpy as np
|
| 6 |
from torch.utils.data import DataLoader
|
| 7 |
+
from typing import Sequence
|
| 8 |
|
| 9 |
+
from src.models.predict import AudioPredictor
|
| 10 |
from src.data.dataset import FullTFPatchesDataset, RandomPatchDataset
|
| 11 |
+
from src.config.config import TrainConfig
|
| 12 |
+
|
| 13 |
+
config = TrainConfig()
|
| 14 |
+
|
| 15 |
+
class CNNTrainer:
|
| 16 |
+
def __init__(self, config: TrainConfig = config) -> None:
|
| 17 |
+
self.config = config
|
| 18 |
+
|
| 19 |
+
def train_cnn(
|
| 20 |
+
self,
|
| 21 |
+
model: torch.nn.Module,
|
| 22 |
+
X_train: Sequence[np.ndarray],
|
| 23 |
+
y_train: Sequence[int],
|
| 24 |
+
X_val: Sequence[np.ndarray],
|
| 25 |
+
y_val: Sequence[int],
|
| 26 |
+
fold_num: int,
|
| 27 |
+
) -> float:
|
| 28 |
+
device = self.config.device
|
| 29 |
+
os.makedirs(self.config.checkpoint_dir, exist_ok=True)
|
| 30 |
+
|
| 31 |
+
model.to(device)
|
| 32 |
+
|
| 33 |
+
if self.config.use_all_patches:
|
| 34 |
+
train_dataset = FullTFPatchesDataset(X_train, y_train)
|
| 35 |
+
print(f"\n{'='*60}\nUsing ALL PATCHES method\n{'='*60}")
|
| 36 |
+
else:
|
| 37 |
+
train_dataset = RandomPatchDataset(X_train, y_train)
|
| 38 |
+
print(f"\n{'='*60}\nUsing ALL PATCHES method\n{'='*60}")
|
| 39 |
+
|
| 40 |
+
train_loader = DataLoader(
|
| 41 |
+
train_dataset,
|
| 42 |
+
batch_size=self.config.batch_size,
|
| 43 |
+
shuffle=True,
|
| 44 |
+
num_workers=4,
|
| 45 |
+
pin_memory=True
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
total_patches = len(train_dataset)
|
| 49 |
+
patches_per_epoch = int(total_patches * self.config.samples_per_epoch_fraction)
|
| 50 |
+
batches_per_epoch = patches_per_epoch // self.config.batch_size
|
| 51 |
|
| 52 |
+
print(f"Total available patches: {total_patches:,}")
|
| 53 |
+
print(f"Patches per epoch ({self.config.samples_per_epoch_fraction}): {patches_per_epoch:,}")
|
| 54 |
+
print(f"Batches per epoch: {batches_per_epoch:,}\n{'='*60}\n")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
|
| 56 |
+
criterion = torch.nn.CrossEntropyLoss()
|
| 57 |
+
optimizer = torch.optim.AdamW([
|
| 58 |
+
{'params': model.features.parameters(), 'weight_decay': 0.0},
|
| 59 |
+
{'params': model.classifier.parameters(), 'weight_decay': 0.001}
|
| 60 |
+
], lr=self.config.lr)
|
| 61 |
+
|
| 62 |
+
start_epoch = 0
|
| 63 |
+
best_val_acc = 0.0
|
| 64 |
+
training_history: dict = {'train_loss': [], 'train_acc': [], 'val_acc': [], 'epochs': []}
|
| 65 |
+
|
| 66 |
+
if self.config.resume_from and os.path.exists(self.config.resume_from):
|
| 67 |
+
print(f"Resuming from checkpoint: {self.config.resume_from}")
|
| 68 |
+
checkpoint = torch.load(self.config.resume_from, map_location=device)
|
| 69 |
+
model.load_state_dict(checkpoint['model_state_dict'])
|
| 70 |
+
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
|
| 71 |
+
start_epoch = checkpoint['epoch'] + 1
|
| 72 |
+
best_val_acc = checkpoint['best_val_acc']
|
| 73 |
+
training_history = checkpoint['history']
|
| 74 |
+
print(f"Resuming from epoch {checkpoint['epoch']}, best val acc: {best_val_acc:.4f}\n")
|
| 75 |
+
|
| 76 |
+
for epoch in range(start_epoch, self.config.epochs):
|
| 77 |
+
model.train()
|
| 78 |
+
train_loss, correct, total, batches_processed = 0.0, 0, 0, 0
|
| 79 |
|
| 80 |
+
for xb, yb in tqdm.tqdm(train_loader, f"Epoch {epoch+1} Train", leave=False):
|
| 81 |
+
if batches_processed >= batches_per_epoch:
|
| 82 |
+
break
|
| 83 |
+
xb, yb = xb.to(device), yb.to(device)
|
| 84 |
+
optimizer.zero_grad()
|
| 85 |
+
out = model(xb)
|
| 86 |
+
loss = criterion(out, yb)
|
| 87 |
+
loss.backward()
|
| 88 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
|
| 89 |
+
optimizer.step()
|
| 90 |
+
|
| 91 |
+
train_loss += loss.item() * xb.size(0)
|
| 92 |
+
_, pred = out.max(1)
|
| 93 |
+
correct += (pred == yb).sum().item()
|
| 94 |
+
total += yb.size(0)
|
| 95 |
+
batches_processed += 1
|
| 96 |
|
| 97 |
+
train_loss /= total
|
| 98 |
+
train_acc = correct / total
|
|
|
|
|
|
|
| 99 |
|
| 100 |
+
model.eval()
|
| 101 |
+
val_correct = 0
|
| 102 |
+
val_total = len(y_val)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
+
for i in tqdm.tqdm(range(val_total), desc=f"Epoch {epoch+1} Val", leave=False):
|
| 105 |
+
spec = X_val[i]
|
| 106 |
+
true_label = y_val[i]
|
| 107 |
+
|
| 108 |
+
pred_label = self._predict_val(model, spec, device)
|
| 109 |
+
|
| 110 |
+
if pred_label == true_label:
|
| 111 |
+
val_correct += 1
|
| 112 |
|
| 113 |
+
val_acc = val_correct / val_total
|
|
|
|
|
|
|
|
|
|
| 114 |
|
| 115 |
+
training_history['train_loss'].append(train_loss)
|
| 116 |
+
training_history['train_acc'].append(train_acc)
|
| 117 |
+
training_history['val_acc'].append(val_acc)
|
| 118 |
+
training_history['epochs'].append(epoch + 1)
|
| 119 |
|
| 120 |
+
is_best = val_acc > best_val_acc
|
| 121 |
|
| 122 |
+
if is_best:
|
| 123 |
+
best_val_acc = val_acc
|
| 124 |
+
torch.save(model.state_dict(), "best_model.pt")
|
| 125 |
+
|
| 126 |
+
print(
|
| 127 |
+
f"Fold {fold_num} | Epoch {epoch+1}/{self.config.epochs} | "
|
| 128 |
+
f"Train loss: {train_loss:.4f}, Train acc: {train_acc:.4f} | "
|
| 129 |
+
f"Val acc: {val_acc:.4f} (best: {best_val_acc:.4f})"
|
| 130 |
+
)
|
| 131 |
|
| 132 |
+
if (epoch + 1) % self.config.save_every_n_epoch == 0:
|
| 133 |
+
checkpoint = {
|
| 134 |
+
'epoch': epoch,
|
| 135 |
+
'model_state_dict': model.state_dict(),
|
| 136 |
+
'optimizer_state_dict': optimizer.state_dict(),
|
| 137 |
+
'train_loss': train_loss,
|
| 138 |
+
'train_acc': train_acc,
|
| 139 |
+
'val_acc': val_acc,
|
| 140 |
+
'best_val_acc': best_val_acc,
|
| 141 |
+
'history': training_history,
|
| 142 |
+
'config': {
|
| 143 |
+
'batch_size': self.config.batch_size,
|
| 144 |
+
'lr': self.config.lr,
|
| 145 |
+
'total_patches': total_patches,
|
| 146 |
+
'patches_per_epoch': patches_per_epoch,
|
| 147 |
+
}
|
| 148 |
}
|
| 149 |
+
checkpoint_path = os.path.join(
|
| 150 |
+
self.config.checkpoint_dir,
|
| 151 |
+
f"checkpoint_epoch_{epoch+1}.pt"
|
| 152 |
+
)
|
| 153 |
+
torch.save(checkpoint, checkpoint_path)
|
| 154 |
+
|
| 155 |
+
if is_best:
|
| 156 |
+
best_path = os.path.join(self.config.checkpoint_dir, "best_model.pt")
|
| 157 |
+
torch.save(checkpoint, best_path)
|
| 158 |
+
|
| 159 |
+
latest_path = os.path.join(self.config.checkpoint_dir, "latest_checkpoint.pt")
|
| 160 |
+
torch.save(checkpoint, latest_path)
|
| 161 |
+
|
| 162 |
+
history_path = os.path.join(self.config.checkpoint_dir, "training_history.json")
|
| 163 |
+
with open(history_path, 'w') as f:
|
| 164 |
+
json.dump(training_history, f, indent=2)
|
| 165 |
+
|
| 166 |
+
final_model_dir = "models/saved"
|
| 167 |
+
os.makedirs(final_model_dir, exist_ok=True)
|
| 168 |
+
final_model_path = os.path.join(final_model_dir, "final_model.pt")
|
| 169 |
+
torch.save({
|
| 170 |
+
'model_state_dict': model.state_dict(),
|
| 171 |
+
'best_val_acc': best_val_acc,
|
| 172 |
+
'config': {
|
| 173 |
+
'batch_size': self.config.batch_size,
|
| 174 |
+
'lr': self.config.lr,
|
| 175 |
+
'epochs': self.config.epochs,
|
| 176 |
}
|
| 177 |
+
}, final_model_path)
|
| 178 |
+
print(f"\nTraining complete! Final model saved to {final_model_path}")
|
| 179 |
+
|
| 180 |
+
return best_val_acc
|
| 181 |
+
|
| 182 |
+
def train_k_fold_cnn(
|
| 183 |
+
self,
|
| 184 |
+
model_class: type,
|
| 185 |
+
X: Sequence[np.ndarray],
|
| 186 |
+
y: Sequence[int],
|
| 187 |
+
) -> tuple[list[float], float]:
|
| 188 |
+
|
| 189 |
+
X_arr = np.array(X)
|
| 190 |
+
y_arr = np.array(y)
|
| 191 |
+
n_samples = len(y_arr)
|
| 192 |
+
indices = np.arange(n_samples)
|
| 193 |
+
np.random.shuffle(indices)
|
| 194 |
+
|
| 195 |
+
fold_sizes = (n_samples // 5) * np.ones(5, dtype=int)
|
| 196 |
+
fold_sizes[:n_samples % 5] += 1
|
| 197 |
+
current = 0
|
| 198 |
+
fold_accuracies: list[float] = []
|
| 199 |
+
|
| 200 |
+
for fold_num, fold_size in enumerate(fold_sizes, 1):
|
| 201 |
+
start, stop = current, current + fold_size
|
| 202 |
+
val_idx = indices[start:stop]
|
| 203 |
+
train_idx = np.concatenate([indices[:start], indices[stop:]])
|
| 204 |
+
current = stop
|
| 205 |
+
|
| 206 |
+
X_train, y_train = X_arr[train_idx].tolist(), y_arr[train_idx]
|
| 207 |
+
X_val, y_val = X_arr[val_idx].tolist(), y_arr[val_idx]
|
| 208 |
+
|
| 209 |
+
print(f"\n{'='*80}\nFOLD {fold_num}/5 | Train: {len(X_train)}, Val: {len(X_val)}\n{'='*80}\n")
|
| 210 |
+
|
| 211 |
+
model = model_class()
|
| 212 |
+
best_acc = self.train_cnn(
|
| 213 |
+
model=model,
|
| 214 |
+
X_train=X_train, y_train=y_train,
|
| 215 |
+
X_val=X_val, y_val=y_val,
|
| 216 |
+
fold_num=fold_num,
|
| 217 |
)
|
| 218 |
+
fold_accuracies.append(best_acc)
|
| 219 |
+
print(f"\nFold {fold_num} Best Accuracy: {best_acc:.4f}\n")
|
| 220 |
+
|
| 221 |
+
mean_acc = float(np.mean(fold_accuracies))
|
| 222 |
+
std_acc = float(np.std(fold_accuracies))
|
| 223 |
+
|
| 224 |
+
print(f"\n{'='*80}\nFINAL 5-FOLD CV RESULTS\nFold Accuracies: {fold_accuracies}\nMean: {mean_acc:.4f} ± {std_acc:.4f}\n{'='*80}\n")
|
| 225 |
+
|
| 226 |
+
results_path = os.path.join(self.config.checkpoint_dir, "5fold_cv_results.json")
|
| 227 |
+
os.makedirs(self.config.checkpoint_dir, exist_ok=True)
|
| 228 |
+
with open(results_path, 'w') as f:
|
| 229 |
+
json.dump({'fold_accuracies': fold_accuracies, 'mean_accuracy': mean_acc, 'std_accuracy': std_acc}, f, indent=2)
|
| 230 |
+
|
| 231 |
+
return fold_accuracies, mean_acc
|
| 232 |
+
|
| 233 |
+
def _predict_val(self, model: torch.nn.Module, spec: np.ndarray, device: str) -> int:
|
| 234 |
+
from src.config.config import DatasetConfig
|
| 235 |
+
cfg = DatasetConfig()
|
| 236 |
+
n_frames = spec.shape[0]
|
| 237 |
+
if n_frames < cfg.cnn_input_length:
|
| 238 |
+
spec = np.pad(spec, ((0, cfg.cnn_input_length - n_frames), (0, 0)), mode="constant")
|
| 239 |
+
n_frames = cfg.cnn_input_length
|
| 240 |
+
patches = np.stack([
|
| 241 |
+
spec[s:s + cfg.cnn_input_length]
|
| 242 |
+
for s in range(0, n_frames - cfg.cnn_input_length + 1)
|
| 243 |
+
])[:, np.newaxis]
|
| 244 |
+
patches_t = torch.tensor(patches, dtype=torch.float32).to(device)
|
| 245 |
+
with torch.no_grad():
|
| 246 |
+
out = model(patches_t).mean(dim=0)
|
| 247 |
+
return out.argmax().item()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|