src/nvidia_pipecat/services/base_action_service.py

# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD 2-Clause License

"""Base classes for action services and modality management.

This module provides the foundation for implementing action services.
It includes:
- Base classes for action services and modality managers
- Support for different modality policies (Override, Parallel, Replace)
- Action state machine management
"""

import asyncio
from collections import deque
from collections.abc import Callable
from typing import Any

from loguru import logger
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor

from nvidia_pipecat.frames.action import (
    ActionFrame,
    ChangeActionFrame,
    FinishedActionFrame,
    StartActionFrame,
    StartedActionFrame,
    StopActionFrame,
    UpdatedActionFrame,
)
from nvidia_pipecat.services.action_handler import (
    ActionHandler,
    InternalStateMachineAbortFrame,
    InternalStateMachineTriggerFrame,
)


def short_id(id) -> str:
    """Returns shortened form of an ID.

    Args:
        id: ID to shorten.

    Returns:
        Shortened ID (first 5 chars + '..') if longer than 5 chars,
        otherwise original ID.
    """
    return f"{id[:5]}.." if len(id) > 5 else id


class ModalityManager:
    """Base class for UMIM action modality managers.

    Manages the interaction between different modalities (e.g., speech, gestures)
    and handles the state transitions of actions within each modality.

    Attributes:
        frame_types_to_process: Frame types this manager processes.
        action_name: Name of managed action type.
        service: Parent action service reference.
        action_handler_factory: Creates action handlers.
        failure_action_factory: Creates failure frames.
    """

    def __init__(
        self,
        frame_types_to_process: tuple[type[Frame], ...],
        action_name: str,
        service: "BaseActionService",
        action_handler_factory: Callable[[Frame], ActionHandler],
        failure_action_factory: Callable[[str, Frame], ActionFrame],
    ):
        """Initializes modality manager.

        Args:
            frame_types_to_process: Frame types this manager should handle.
            action_name: Name of action type managed by this instance.
            service: Reference to parent action service.
            action_handler_factory: Factory function creating action handlers.
            failure_action_factory: Factory function creating failure action frames.
        """
        self.frame_types_to_process: tuple[type[Frame], ...] = frame_types_to_process
        self.action_name = action_name
        self.service = service
        self.action_handler_factory = action_handler_factory
        self.failure_action_factory = failure_action_factory

    def can_handle_frame(self, frame: Frame) -> bool:
        """Check if the modality manager can handle a given frame."""
        if isinstance(frame, InternalStateMachineTriggerFrame):
            return frame.action_name == self.action_name
        else:
            return isinstance(frame, self.frame_types_to_process)

    async def on_started(self, frame: ActionFrame) -> None:
        """Handle when an action is started."""
        action_handler = self._get_action_handler(frame)
        if not action_handler:
            logger.warning(f"Unknown action with ID={frame.action_id}. Ignoring {type(frame)} frame {frame}")
            await self.service.queue_for_internal_processing(self.failure_action_factory("Action crashed", frame))
            return
        if not await action_handler.may_started():  # type: ignore
            logger.warning(f"Got frame {frame} when not in 'starting' state")
            logger.warning(f"On action state {action_handler.action_state}")
        await action_handler.started(frame)  # type: ignore

    async def on_stop(self, frame: ActionFrame) -> None:
        """Handle when an action is stopped."""
        action_handler = self._get_action_handler(frame)
        if not action_handler:
            logger.warning(f"Unknown action with ID={frame.action_id}. Ignoring {type(frame)} frame {frame}")
            await self.service.queue_for_internal_processing(
                self.failure_action_factory("Action does not exist", frame)
            )
            return
        await action_handler.stop(frame)  # type: ignore

    async def on_change(self, frame: ActionFrame) -> None:
        """Handle when an action is changed."""
        action_handler = self._get_action_handler(frame)
        if not action_handler:
            logger.warning(f"Unknown action with ID={frame.action_id}. Ignoring {type(frame)} frame {frame}")
            await self.service.queue_for_internal_processing(self.failure_action_factory("Action crashed", frame))
            return
        await action_handler.change(frame)  # type: ignore

    async def on_updated(self, frame: ActionFrame) -> None:
        """Handle when an action is updated."""
        action_handler = self._get_action_handler(frame)
        if not action_handler:
            logger.warning(f"Unknown action with ID={frame.action_id}. Ignoring {type(frame)} frame {frame}")
            await self.service.queue_for_internal_processing(self.failure_action_factory("Action crashed", frame))
            return
        await action_handler.updated(frame)  # type: ignore

    async def on_internal_frame(self, frame: InternalStateMachineTriggerFrame) -> None:
        """Handle when an internal frame is received."""
        action_handler = self._get_action_handler(frame)
        if not action_handler:
            action_handler = await self._add_action_handler(frame)
        if not action_handler:
            logger.error(f"Unknown action with ID={frame.action_id}. Ignoring internal {frame.trigger} frame {frame}")
            return

        await action_handler.trigger(frame.trigger, frame)  # type: ignore

    async def on_start(self, frame: ActionFrame) -> None:
        """Handle when an action is starting."""
        if self._get_action_handler(frame):
            logger.warning(f"Action with ID={frame.action_id} already exists. Ignoring {type(frame)} frame {frame}")
            return

        action_handler = await self._add_action_handler(frame)
        await action_handler.start(frame)  # type: ignore
        await self.service.queue_for_internal_processing(
            self._get_internal_frame("check_modality", action_id=frame.action_id)
        )

    async def on_check_modality(self, frame: InternalStateMachineTriggerFrame) -> None:
        """Handle when a modality check is requested."""
        await self._manage_modality_policy()

    async def on_modality_available(self, frame: InternalStateMachineTriggerFrame) -> None:
        """Handle modality availability event."""
        action_handler = self._get_action_handler(frame)
        if action_handler.state == "scheduled":  # type: ignore
            await action_handler.trigger(frame.trigger, frame)  # type: ignore

    async def on_finished(self, frame: InternalStateMachineTriggerFrame) -> None:
        """Handle when an action is finished."""
        action_handler = self._get_action_handler(frame)
        if not action_handler:
            logger.warning(
                f"Action with ID={frame.action_id} not currently running. Ignoring {type(frame)} frame {frame}"
            )
            return

        if action_handler.state != "stopping":  # type: ignore
            logger.warning(f"Got frame {frame} when not in 'stopping' state")
            logger.warning(f"On action state {action_handler.action_state}")

        await action_handler.finished(frame)  # type: ignore
        await self._remove_action_handler(frame.action_id)

    async def _add_action_handler(self, frame: ActionFrame | InternalStateMachineTriggerFrame) -> ActionHandler:
        raise NotImplementedError()

    def _get_action_handler(self, frame: ActionFrame | InternalStateMachineTriggerFrame) -> ActionHandler | None:
        raise NotImplementedError()

    def _get_failure_action_finished(self, reason: str, frame: ActionFrame) -> ActionFrame:
        raise NotImplementedError()

    def _get_internal_frame(
        self, trigger: str, *, action_id: str, data: dict[str, Any] | None = None
    ) -> InternalStateMachineTriggerFrame:
        return InternalStateMachineTriggerFrame(
            trigger, action_id=action_id, action_name=self.action_name, data=data or {}
        )

    async def _remove_action_handler(self, action_id: str) -> None:
        raise NotImplementedError()

    async def _manage_modality_policy(self) -> None:
        """Manage the modality policy.

        Depending on the modality policy, this will do different things.
        """
        raise NotImplementedError()


class BaseActionService(FrameProcessor):
    """Base class for all action services.

    Provides core functionality for processing action frames and managing modalities
    in the system.


    Attributes:
        name: Service name identifier.
        frame_types_to_process: Frame types this service processes.
        modality_managers: List of modality managers.
        stream_id: Current stream identifier.
    """

    name: str
    frame_types_to_process: tuple[type[Frame], ...] = tuple()

    def __init__(self, modality_managers: list[ModalityManager], **kwargs):
        """Initialize the action service.

        Args:
            modality_managers: List of modality managers for different action types.
            **kwargs: Additional arguments passed to parent FrameProcessor.
        """
        super().__init__(**kwargs)
        self.modality_managers = modality_managers
        self.stream_id: str | None = None
        self._processing_queue: asyncio.Queue[Frame] = asyncio.Queue()
        self._processing_task: asyncio.Task | None = None
        self._should_terminate = False

    async def start(self, frame: StartFrame) -> None:
        """Called during pipeline start."""
        if "stream_id" not in frame.metadata:
            raise ValueError(
                "No 'stream_id' found in frame.metadata. "
                "You need to populate this with PipelineParams(start_metadata={'stream_id': 'id'})"
            )
        self.stream_id = frame.metadata["stream_id"]
        self._processing_task = self.create_task(self.processing_task())

    async def stop(self, frame: EndFrame) -> None:
        """Called during pipeline end."""

    async def cancel(self, frame: CancelFrame) -> None:
        """Called during pipeline cancelled."""

    async def cleanup(self):
        """Called during pipeline cleanup."""
        await super().cleanup()
        self._should_terminate = True
        if self._processing_task:
            await self.cancel_task(self._processing_task)
            self._processing_task = None

    async def queue_for_internal_processing(self, frame: Frame):
        """Queue a frame for internal processing."""
        # If we are cancelling we don't want to process any other frame.
        if self._cancelling:
            return

        await self._processing_queue.put(frame)

    def can_handle_frame(self, frame: Frame, direction: FrameDirection) -> bool:
        """Check if the service can handle a given frame."""
        return (
            isinstance(frame, InternalStateMachineTriggerFrame)
            or isinstance(frame, self.frame_types_to_process)
            and direction == FrameDirection.DOWNSTREAM
        )

    async def processing_task(self):
        """Processing task for the action service."""
        while not self._should_terminate:
            try:
                frame = await self._processing_queue.get()
                for manager in self.modality_managers:
                    if manager.can_handle_frame(frame):
                        if isinstance(frame, StartedActionFrame):
                            await manager.on_started(frame)
                        elif isinstance(frame, StartActionFrame):
                            await manager.on_start(frame)
                        elif isinstance(frame, FinishedActionFrame):
                            await manager.on_finished(frame)
                        elif isinstance(frame, UpdatedActionFrame):
                            await manager.on_updated(frame)
                        elif isinstance(frame, ChangeActionFrame):
                            await manager.on_change(frame)
                        elif isinstance(frame, StopActionFrame):
                            await manager.on_stop(frame)
                        elif isinstance(frame, InternalStateMachineTriggerFrame | InternalStateMachineAbortFrame):
                            if frame.trigger == "check_modality":
                                await manager.on_check_modality(frame)
                            elif frame.trigger == "modality_available":
                                await manager.on_modality_available(frame)
                            else:
                                await manager.on_internal_frame(frame)

                if isinstance(frame, StartedActionFrame | FinishedActionFrame | UpdatedActionFrame):
                    await self.push_frame(frame, FrameDirection.UPSTREAM)
                    await self.push_frame(frame, FrameDirection.DOWNSTREAM)

            except Exception as e:
                logger.exception(f"Error in {self.name} processing task: {e}")

    async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
        """Process a frame."""
        await super().process_frame(frame, direction)

        # Internal frames are not propagated all other frames are immediately pushed
        if isinstance(frame, InternalStateMachineTriggerFrame):
            logger.error(f"Received internal frame {frame}")
            return

        await self.push_frame(frame, direction)

        if isinstance(frame, StartFrame):
            await self.start(frame)
        elif isinstance(frame, CancelFrame):
            await self.cancel(frame)
        elif isinstance(frame, EndFrame):
            await self.stop(frame)
        elif self.can_handle_frame(frame, direction):
            await self.queue_for_internal_processing(frame)


class OverrideModalityManager(ModalityManager):
    """Manages actions using override policy.

    The override policy implements an action stack. Multiple actions can run at the same time.
    If an action is already running when a new action should be started, the current
    action will be paused ("overwritten"). When the overwriting action finishes,
    the other action will be resumed.

    Attributes:
        action_stack: Stack of currently active actions.
    """

    def __init__(
        self,
        frame_types_to_process: tuple[type[Frame], ...],
        action_name: str,
        service: "BaseActionService",
        action_handler_factory: Callable[[Frame], ActionHandler],
        failure_action_factory: Callable[[str, Frame], ActionFrame],
    ):
        """Initialize the override modality manager.

        Args:
            frame_types_to_process: Tuple of frame types this manager should handle.
            action_name: Name of the action type managed by this instance.
            service: Reference to the parent action service.
            action_handler_factory: Factory function to create action handlers.
            failure_action_factory: Factory function to create failure action frames.
        """
        super().__init__(
            frame_types_to_process,
            action_name,
            service,
            action_handler_factory,
            failure_action_factory,
        )
        self.action_stack: list[ActionHandler] = []

    @property
    def focus_action_index(self) -> int | None:
        """Get the index of the focus action."""
        if len(self.action_stack) == 0:
            return None

        for i, a in enumerate(self.action_stack):
            if a.state == "scheduled":  # type: ignore
                return i - 1 if i > 0 else None

        return len(self.action_stack) - 1

    def _get_action_handler(self, frame: ActionFrame | InternalStateMachineTriggerFrame) -> ActionHandler | None:
        """Get the action handler for a given frame."""
        for action in reversed(self.action_stack):
            if action.action_id == frame.action_id:
                return action

        return None

    async def on_internal_frame(self, frame: InternalStateMachineTriggerFrame) -> None:
        """Handle when an internal frame is received."""
        if frame.trigger == "resume" and self.action_stack[-1].action_id != frame.action_id:
            logger.info(
                f"Cannot resume Action({short_id(frame.action_id)}) because "
                "it is no longer the topmost action on the stack."
            )
            return

        if frame.trigger == "pause" and self._get_action_handler(frame).state == "paused":
            # If the action is already paused, we don't need to do anything
            return

        await super().on_internal_frame(frame)

    async def on_modality_available(self, frame: InternalStateMachineTriggerFrame) -> None:
        """Handle modality availability event."""
        for i, action in enumerate(self.action_stack):
            if action.action_id == frame.action_id:
                if self.focus_action_index and i == self.focus_action_index:
                    # If the action is already in focus, we don't need to do anything
                    return
                elif self.focus_action_index and i - self.focus_action_index != 1:
                    logger.warning(
                        "Focus action and next focus action differ in index != 1. "
                        f"Difference: {i - self.focus_action_index}"
                    )
                break

        await super().on_modality_available(frame)

    async def _manage_modality_policy(self) -> None:
        """Manage the modality policy.

        Manages the modality policy for the action stack. Actions can be scheduled (s), starting (t), paused (p),
        running (r) or finished (f). In general the stack at any point can look like this:
        [p, ... , p | r | t , s, s, ...]
        """
        if len(self.action_stack) == 0:
            return

        if self.focus_action_index is None:
            await self.service.queue_for_internal_processing(
                self._get_internal_frame("modality_available", action_id=self.action_stack[0].action_id)
            )
        else:
            focus_action = self.action_stack[self.focus_action_index]
            if self.focus_action_index < len(self.action_stack) - 1:
                action_state = focus_action.state  # type: ignore
                short_action_id = short_id(focus_action.action_id)
                next_action_id = self.action_stack[self.focus_action_index + 1].action_id

                logger.info(f"Modality check: Focused Action({short_action_id}) is in state '{action_state}'")
                if action_state == "paused" or action_state == "finished":
                    await self.service.queue_for_internal_processing(
                        self._get_internal_frame("modality_available", action_id=next_action_id)
                    )

                elif action_state == "running":
                    await self.service.queue_for_internal_processing(
                        self._get_internal_frame("pause", action_id=focus_action.action_id)
                    )

                # Allow for other tasks to run
                await asyncio.sleep(0)
                await self.service.queue_for_internal_processing(
                    self._get_internal_frame("check_modality", action_id=next_action_id)
                )

    async def _add_action_handler(self, frame: ActionFrame | InternalStateMachineTriggerFrame) -> ActionHandler:
        action_handler = self.action_handler_factory(frame)
        self.action_stack.append(action_handler)
        return action_handler

    async def _remove_action_handler(self, action_id: str) -> None:
        top_action: ActionHandler | None = self.action_stack[-1] if len(self.action_stack) > 0 else None
        action_to_remove: ActionHandler | None = None

        for action in reversed(self.action_stack):
            if action.action_id == action_id:
                action_to_remove = action

        if action_to_remove:
            self.action_stack.remove(action_to_remove)

            if len(self.action_stack) == 0:
                await action_to_remove.clear_modality()
            elif action_to_remove == top_action:
                await self.service.queue_for_internal_processing(
                    self._get_internal_frame(
                        "resume",
                        action_id=self.action_stack[-1].action_id,
                        data={"previous_action_state": action_to_remove.action_state},
                    )
                )


class ParallelModalityManager(ModalityManager):
    """Manages actions using parallel policy.

    Allows multiple actions to run simultaneously without interference.

    Attributes:
        actions: Dictionary mapping action IDs to handlers.
    """

    def __init__(
        self,
        frame_types_to_process: tuple[type[Frame], ...],
        action_name: str,
        service: "BaseActionService",
        action_handler_factory: Callable[[Frame], ActionHandler],
        failure_action_factory: Callable[[str, Frame], ActionFrame],
    ):
        """Initialize the parallel modality manager.

        Args:
            frame_types_to_process: Tuple of frame types this manager should handle.
            action_name: Name of the action type managed by this instance.
            service: Reference to the parent action service.
            action_handler_factory: Factory function to create action handlers.
            failure_action_factory: Factory function to create failure action frames.
        """
        super().__init__(
            frame_types_to_process,
            action_name,
            service,
            action_handler_factory,
            failure_action_factory,
        )
        self.actions: dict[str, ActionHandler] = {}

    def _get_action_handler(self, frame: ActionFrame | InternalStateMachineTriggerFrame) -> ActionHandler | None:
        return self.actions.get(frame.action_id, None)

    async def _manage_modality_policy(self) -> None:
        for action in self.actions.values():
            if action.state == "scheduled":  # type: ignore
                await self.service.queue_for_internal_processing(
                    self._get_internal_frame("modality_available", action_id=action.action_id)
                )

    async def _add_action_handler(self, frame: ActionFrame | InternalStateMachineTriggerFrame) -> ActionHandler:
        self.actions[frame.action_id] = self.action_handler_factory(frame)
        return self.actions[frame.action_id]

    async def _remove_action_handler(self, action_id: str) -> None:
        if action_id in self.actions:
            del self.actions[action_id]


class ReplaceModalityManager(ModalityManager):
    """Manages actions using replace policy.

    If a new action should be started, any ongoing action is stopped and the new action
    is started after that.

    Attributes:
        action: Currently active action handler.
        replaced_actions: Queue of actions that have been replaced.
    """

    def __init__(
        self,
        frame_types_to_process: tuple[type[Frame], ...],
        action_name: str,
        service: "BaseActionService",
        action_handler_factory: Callable[[Frame], ActionHandler],
        failure_action_factory: Callable[[str, Frame], ActionFrame],
    ):
        """Initialize the replace modality manager.

        Args:
            frame_types_to_process: Tuple of frame types this manager should handle.
            action_name: Name of the action type managed by this instance.
            service: Reference to the parent action service.
            action_handler_factory: Factory function to create action handlers.
            failure_action_factory: Factory function to create failure action frames.
        """
        super().__init__(
            frame_types_to_process,
            action_name,
            service,
            action_handler_factory,
            failure_action_factory,
        )
        self.action: ActionHandler | None = None
        self.replaced_actions: deque[ActionHandler] = deque()

    def _get_action_handler(self, frame: ActionFrame | InternalStateMachineTriggerFrame) -> ActionHandler | None:
        # If action_id matches current action
        if self.action and self.action.action_id == frame.action_id:
            return self.action

        # otherwise check all replaced actions...
        for replaced_action in self.replaced_actions:
            if replaced_action and replaced_action.action_id == frame.action_id:
                return replaced_action

        # ... or return None if no match was found
        return None

    async def _manage_modality_policy(self) -> None:
        if self.action is None:
            return
        if len(self.replaced_actions) > 0:
            newest_action_to_be_replaced = self.replaced_actions[-1]
            replaced_action_state: str = newest_action_to_be_replaced.state  # type: ignore
            if replaced_action_state == "finished":
                await self.service.queue_for_internal_processing(
                    self._get_internal_frame("modality_available", action_id=self.action.action_id)
                )
            elif replaced_action_state == "stopping":
                # Allow for other tasks to run
                await asyncio.sleep(0)
                await self.service.queue_for_internal_processing(
                    self._get_internal_frame("check_modality", action_id=self.action.action_id)
                )
            else:
                await self.service.queue_for_internal_processing(
                    InternalStateMachineAbortFrame(
                        action_name=self.action_name,
                        action_id=newest_action_to_be_replaced.action_id,
                        reason="Action replaced.",
                    )
                )
                # Allow for other tasks to run
                await asyncio.sleep(0)
                await self.service.queue_for_internal_processing(
                    self._get_internal_frame("check_modality", action_id=self.action.action_id)
                )

        else:
            await self.service.queue_for_internal_processing(
                self._get_internal_frame("modality_available", action_id=self.action.action_id)
            )

    async def _add_action_handler(self, frame: ActionFrame | InternalStateMachineTriggerFrame) -> ActionHandler:
        if self.action is not None:
            self.replaced_actions.append(self.action)
        self.action = self.action_handler_factory(frame)
        return self.action

    async def _remove_action_handler(self, action_id: str) -> None:
        if self.action and action_id == self.action.action_id:
            await self.action.clear_modality()
            self.action = None
        else:
            to_remove = None
            for replaced_action in self.replaced_actions:
                if replaced_action and action_id == replaced_action.action_id:
                    to_remove = replaced_action
            if to_remove:
                self.replaced_actions.remove(to_remove)