model/model.py

import torch
import torch.nn as nn
from transformers import EncodecModel, EncodecConfig
from model.attention_weighted_pooling import AttentionWeightedPooling
from model.projection import ProjectionHead
from model.chordbeat_encoder import ChordBeatEncoder

class MusicConRec(nn.Module):
    def __init__(self, codebook_size=1024, feature_dim=128, proj_dim=128):
        super().__init__()

        # === AUDIO SIDE ===
        self.encodec = EncodecModel.from_pretrained("facebook/encodec_24khz")
        
        # Do not freeze EncodecModel parameters — allow fine-tuning
        for param in self.encodec.parameters():
            param.requires_grad = True

        self.code_embedding = nn.Embedding(codebook_size, feature_dim)

        self.audio_pool = AttentionWeightedPooling(feature_dim)
        self.audio_proj = ProjectionHead(feature_dim, out_dim=proj_dim)

        # === CHORD SIDE ===
        self.chord_encoder = ChordBeatEncoder(
            input_dim=13,
            d_model=feature_dim
        )

    def forward(self, audio, chord):
        """
        audio: (B, 1, T)
        chord: (B, T_chord, 13)
        """


        # =========================
        # ENCODE
        # =========================

        encoder_outputs = self.encodec.encode(audio)  

        audio_codes = encoder_outputs['audio_codes'].long()
        audio_scales = encoder_outputs['audio_scales']


        codes = audio_codes.squeeze(0).permute(0, 2, 1)
        codes = self.code_embedding(codes) # (B, T, Q, D) 
        codes = codes.mean(dim=2) # (B, T, D)
        
        # =========================
        # POOL + PROJECT
        # =========================
        h_audio = self.audio_pool(codes)     # (B, D)
        z_audio = self.audio_proj(h_audio)   # (B, proj_dim)

        # =========================
        # RECONSTRUCTION
        # =========================
        x_recon = self.encodec.decode(audio_codes, audio_scales)['audio_values']
        x_recon = torch.tanh(x_recon).clamp(-1.0, 1.0)

        # =========================
        # CHORD BRANCH
        # =========================
        z_chord, h_chord = self.chord_encoder(chord)

        return {
            "x_recon": x_recon,
            "z_audio": z_audio,
            "z_chord": z_chord,
            "h_audio": h_audio,
            "h_chord": h_chord
        }