infer/modules/ipex/attention.py

import torch
import intel_extension_for_pytorch as ipex  # pylint: disable=import-error, unused-import

# pylint: disable=protected-access, missing-function-docstring, line-too-long

original_torch_bmm = torch.bmm


def torch_bmm(input, mat2, *, out=None):
    if input.dtype != mat2.dtype:
        mat2 = mat2.to(input.dtype)

    # ARC GPUs can't allocate more than 4GB to a single block, Slice it:
    batch_size_attention, input_tokens, mat2_shape = (
        input.shape[0],
        input.shape[1],
        mat2.shape[2],
    )
    block_multiply = input.element_size()
    slice_block_size = input_tokens * mat2_shape / 1024 / 1024 * block_multiply
    block_size = batch_size_attention * slice_block_size

    split_slice_size = batch_size_attention
    if block_size > 4:
        do_split = True
        # Find something divisible with the input_tokens
        while (split_slice_size * slice_block_size) > 4:
            split_slice_size = split_slice_size // 2
            if split_slice_size <= 1:
                split_slice_size = 1
                break
    else:
        do_split = False

    split_2_slice_size = input_tokens
    if split_slice_size * slice_block_size > 4:
        slice_block_size2 = split_slice_size * mat2_shape / 1024 / 1024 * block_multiply
        do_split_2 = True
        # Find something divisible with the input_tokens
        while (split_2_slice_size * slice_block_size2) > 4:
            split_2_slice_size = split_2_slice_size // 2
            if split_2_slice_size <= 1:
                split_2_slice_size = 1
                break
    else:
        do_split_2 = False

    if do_split:
        hidden_states = torch.zeros(
            input.shape[0],
            input.shape[1],
            mat2.shape[2],
            device=input.device,
            dtype=input.dtype,
        )
        for i in range(batch_size_attention // split_slice_size):
            start_idx = i * split_slice_size
            end_idx = (i + 1) * split_slice_size
            if do_split_2:
                for i2 in range(
                    input_tokens // split_2_slice_size
                ):  # pylint: disable=invalid-name
                    start_idx_2 = i2 * split_2_slice_size
                    end_idx_2 = (i2 + 1) * split_2_slice_size
                    hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = (
                        original_torch_bmm(
                            input[start_idx:end_idx, start_idx_2:end_idx_2],
                            mat2[start_idx:end_idx, start_idx_2:end_idx_2],
                            out=out,
                        )
                    )
            else:
                hidden_states[start_idx:end_idx] = original_torch_bmm(
                    input[start_idx:end_idx], mat2[start_idx:end_idx], out=out
                )
    else:
        return original_torch_bmm(input, mat2, out=out)
    return hidden_states


original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention


def scaled_dot_product_attention(
    query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False
):
    # ARC GPUs can't allocate more than 4GB to a single block, Slice it:
    if len(query.shape) == 3:
        batch_size_attention, query_tokens, shape_four = query.shape
        shape_one = 1
        no_shape_one = True
    else:
        shape_one, batch_size_attention, query_tokens, shape_four = query.shape
        no_shape_one = False

    block_multiply = query.element_size()
    slice_block_size = (
        shape_one * query_tokens * shape_four / 1024 / 1024 * block_multiply
    )
    block_size = batch_size_attention * slice_block_size

    split_slice_size = batch_size_attention
    if block_size > 4:
        do_split = True
        # Find something divisible with the shape_one
        while (split_slice_size * slice_block_size) > 4:
            split_slice_size = split_slice_size // 2
            if split_slice_size <= 1:
                split_slice_size = 1
                break
    else:
        do_split = False

    split_2_slice_size = query_tokens
    if split_slice_size * slice_block_size > 4:
        slice_block_size2 = (
            shape_one * split_slice_size * shape_four / 1024 / 1024 * block_multiply
        )
        do_split_2 = True
        # Find something divisible with the batch_size_attention
        while (split_2_slice_size * slice_block_size2) > 4:
            split_2_slice_size = split_2_slice_size // 2
            if split_2_slice_size <= 1:
                split_2_slice_size = 1
                break
    else:
        do_split_2 = False

    if do_split:
        hidden_states = torch.zeros(query.shape, device=query.device, dtype=query.dtype)
        for i in range(batch_size_attention // split_slice_size):
            start_idx = i * split_slice_size
            end_idx = (i + 1) * split_slice_size
            if do_split_2:
                for i2 in range(
                    query_tokens // split_2_slice_size
                ):  # pylint: disable=invalid-name
                    start_idx_2 = i2 * split_2_slice_size
                    end_idx_2 = (i2 + 1) * split_2_slice_size
                    if no_shape_one:
                        hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = (
                            original_scaled_dot_product_attention(
                                query[start_idx:end_idx, start_idx_2:end_idx_2],
                                key[start_idx:end_idx, start_idx_2:end_idx_2],
                                value[start_idx:end_idx, start_idx_2:end_idx_2],
                                attn_mask=(
                                    attn_mask[start_idx:end_idx, start_idx_2:end_idx_2]
                                    if attn_mask is not None
                                    else attn_mask
                                ),
                                dropout_p=dropout_p,
                                is_causal=is_causal,
                            )
                        )
                    else:
                        hidden_states[:, start_idx:end_idx, start_idx_2:end_idx_2] = (
                            original_scaled_dot_product_attention(
                                query[:, start_idx:end_idx, start_idx_2:end_idx_2],
                                key[:, start_idx:end_idx, start_idx_2:end_idx_2],
                                value[:, start_idx:end_idx, start_idx_2:end_idx_2],
                                attn_mask=(
                                    attn_mask[
                                        :, start_idx:end_idx, start_idx_2:end_idx_2
                                    ]
                                    if attn_mask is not None
                                    else attn_mask
                                ),
                                dropout_p=dropout_p,
                                is_causal=is_causal,
                            )
                        )
            else:
                if no_shape_one:
                    hidden_states[start_idx:end_idx] = (
                        original_scaled_dot_product_attention(
                            query[start_idx:end_idx],
                            key[start_idx:end_idx],
                            value[start_idx:end_idx],
                            attn_mask=(
                                attn_mask[start_idx:end_idx]
                                if attn_mask is not None
                                else attn_mask
                            ),
                            dropout_p=dropout_p,
                            is_causal=is_causal,
                        )
                    )
                else:
                    hidden_states[:, start_idx:end_idx] = (
                        original_scaled_dot_product_attention(
                            query[:, start_idx:end_idx],
                            key[:, start_idx:end_idx],
                            value[:, start_idx:end_idx],
                            attn_mask=(
                                attn_mask[:, start_idx:end_idx]
                                if attn_mask is not None
                                else attn_mask
                            ),
                            dropout_p=dropout_p,
                            is_causal=is_causal,
                        )
                    )
    else:
        return original_scaled_dot_product_attention(
            query,
            key,
            value,
            attn_mask=attn_mask,
            dropout_p=dropout_p,
            is_causal=is_causal,
        )
    return hidden_states


def attention_init():
    # ARC GPUs can't allocate more than 4GB to a single block:
    torch.bmm = torch_bmm
    torch.nn.functional.scaled_dot_product_attention = scaled_dot_product_attention