modeling_grid.py

import os
import sys
import types

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import torch
import torch.nn as nn
from transformers import PreTrainedModel
from transformers.modeling_outputs import CausalLMOutput

from .configuration_grid import FelaGridConfig
from .modeling import FELA_Grid, FNO1D


def _fno1d_forward(self, x):
    w = torch.view_as_complex(self.w)
    P = x.shape[1]
    xf = torch.fft.rfft(x, dim=1)
    mm = min(self.modes, xf.shape[1])
    o = torch.zeros_like(xf)
    o[:, :mm] = torch.einsum("bpd,pde->bpe", xf[:, :mm], w[:mm])
    return torch.fft.irfft(o, n=P, dim=1)


def _realify(model):
    for m in model.modules():
        if isinstance(m, FNO1D):
            m.w = nn.Parameter(torch.view_as_real(m.w.detach()).contiguous())
            m.forward = types.MethodType(_fno1d_forward, m)


class FelaGridModel(PreTrainedModel):
    config_class = FelaGridConfig
    base_model_prefix = "model"
    main_input_name = "x"

    def __init__(self, config):
        super().__init__(config)
        self.model = FELA_Grid(
            config.Fin,
            config.L,
            D=config.D,
            modes=config.modes,
            nblk=config.nblk,
            nq=config.nq,
            arch=config.arch,
        )
        self.model._non_persistent_buffers_set.add("qidx")
        _realify(self.model)
        self.post_init()

    def forward(self, x=None, input_values=None, **kwargs):
        if x is None:
            x = input_values
        out = self.model(x)
        return CausalLMOutput(logits=out)