import torch import triton import triton.language as tl @triton.jit def _causal_rope_apply_kernel( out_ptr, x_ptr, freqs_real_ptr, grid_sizes_ptr, num_heads: tl.constexpr, head_dim: tl.constexpr, seq_stride: tl.constexpr, t_dim: tl.constexpr, h_dim: tl.constexpr, start_frame: tl.constexpr, conjugate: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): row_idx = tl.program_id(0) batch_idx = row_idx // (seq_stride * num_heads) seq_head_idx = row_idx - batch_idx * seq_stride * num_heads token_idx = seq_head_idx // num_heads f = tl.load(grid_sizes_ptr + batch_idx * 3) h = tl.load(grid_sizes_ptr + batch_idx * 3 + 1) w = tl.load(grid_sizes_ptr + batch_idx * 3 + 2) valid_seq_len = f * h * w half_head_dim: tl.constexpr = head_dim // 2 col_offsets = tl.arange(0, BLOCK_SIZE) mask = col_offsets < half_head_dim x_row_ptr = x_ptr + row_idx * head_dim out_row_ptr = out_ptr + row_idx * head_dim x_real = tl.load(x_row_ptr + col_offsets * 2, mask=mask, other=0.0) x_imag = tl.load(x_row_ptr + col_offsets * 2 + 1, mask=mask, other=0.0) is_valid = token_idx < valid_seq_len hw = h * w t = token_idx // hw rem = token_idx - t * hw y = rem // w z = rem - y * w freq_row = tl.where(col_offsets < t_dim, start_frame + t, tl.where(col_offsets < t_dim + h_dim, y, z)) freq_offset = (freq_row * half_head_dim + col_offsets) * 2 load_mask = mask & is_valid cos_vals = tl.load(freqs_real_ptr + freq_offset, mask=load_mask, other=1.0) sin_vals = tl.load(freqs_real_ptr + freq_offset + 1, mask=load_mask, other=0.0) if conjugate: sin_vals = -sin_vals rotated_real = x_real.to(tl.float32) * cos_vals.to(tl.float32) - x_imag.to(tl.float32) * sin_vals.to(tl.float32) rotated_imag = x_real.to(tl.float32) * sin_vals.to(tl.float32) + x_imag.to(tl.float32) * cos_vals.to(tl.float32) out_real = tl.where(is_valid, rotated_real, x_real) out_imag = tl.where(is_valid, rotated_imag, x_imag) tl.store(out_row_ptr + col_offsets * 2, out_real, mask=mask) tl.store(out_row_ptr + col_offsets * 2 + 1, out_imag, mask=mask) class _CausalRoPEApplyTriton(torch.autograd.Function): @staticmethod def forward(ctx, x, grid_sizes, freqs, start_frame): b, seq_len, num_heads, head_dim = x.shape c = head_dim // 2 t_dim = c - 2 * (c // 3) h_dim = c // 3 x_contig = x.contiguous() out = torch.empty_like(x_contig) grid_sizes = grid_sizes.to(device=x.device, dtype=torch.int64, non_blocking=True).contiguous() freqs_real = torch.view_as_real(freqs.to(device=x.device)) block_size = triton.next_power_of_2(c) _causal_rope_apply_kernel[(b * seq_len * num_heads,)]( out, x_contig, freqs_real, grid_sizes, num_heads, head_dim, seq_len, t_dim, h_dim, start_frame, False, BLOCK_SIZE=block_size, num_warps=1, num_stages=2, ) ctx.save_for_backward(grid_sizes, freqs_real) ctx.num_heads = num_heads ctx.head_dim = head_dim ctx.seq_len = seq_len ctx.t_dim = t_dim ctx.h_dim = h_dim ctx.start_frame = start_frame ctx.block_size = block_size ctx.input_shape = x.shape return out @staticmethod def backward(ctx, grad_output): grid_sizes, freqs_real = ctx.saved_tensors grad_contig = grad_output.contiguous() grad_x = torch.empty_like(grad_contig) b, seq_len, num_heads, _ = grad_contig.shape _causal_rope_apply_kernel[(b * seq_len * num_heads,)]( grad_x, grad_contig, freqs_real, grid_sizes, ctx.num_heads, ctx.head_dim, ctx.seq_len, ctx.t_dim, ctx.h_dim, ctx.start_frame, True, BLOCK_SIZE=ctx.block_size, num_warps=1, num_stages=2, ) return grad_x.reshape(ctx.input_shape), None, None, None def causal_rope_apply_triton(x, grid_sizes, freqs, start_frame=0): return _CausalRoPEApplyTriton.apply(x, grid_sizes, freqs, start_frame).type_as(x)