File size: 8,416 Bytes

854994d

use crate::endpoint::realtime_websocket::protocol_frameless_bidi::parse_frameless_bidi_event;
use crate::endpoint::realtime_websocket::protocol_v1::parse_realtime_event_v1;
use crate::endpoint::realtime_websocket::protocol_v2::parse_realtime_event_v2;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ConversationTextRole;
pub use codex_protocol::protocol::RealtimeAudioFrame;
pub use codex_protocol::protocol::RealtimeEvent;
pub use codex_protocol::protocol::RealtimeOutputModality;
pub use codex_protocol::protocol::RealtimeTranscriptEntry;
pub use codex_protocol::protocol::RealtimeVoice;
use serde::Serialize;
use serde_json::Value;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RealtimeEventParser {
    V1,
    FramelessBidi,
    RealtimeV2,
}

pub type RealtimeWireAdapter = RealtimeEventParser;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RealtimeSessionMode {
    Conversational,
    Transcription,
}

/// Selects the semantic stream used for Frameless Bidi context appends.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum RealtimeContextAppendChannel {
    Speakable,
    Commentary,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RealtimeSessionConfig {
    pub instructions: String,
    pub initial_items: Vec<ConversationTextParams>,
    pub delegation_ack_filler: Option<bool>,
    pub model: Option<String>,
    pub session_id: Option<String>,
    pub event_parser: RealtimeEventParser,
    pub session_mode: RealtimeSessionMode,
    pub output_modality: RealtimeOutputModality,
    pub voice: RealtimeVoice,
}

#[derive(Debug, Clone, Serialize)]
#[serde(tag = "type")]
pub(super) enum RealtimeOutboundMessage {
    #[serde(rename = "input_audio_buffer.append")]
    InputAudioBufferAppend { audio: String },
    #[serde(rename = "conversation.handoff.append")]
    ConversationHandoffAppend {
        handoff_id: String,
        output_text: String,
    },
    #[serde(rename = "input_audio.append")]
    InputAudioAppend { audio: String },
    #[serde(rename = "delegation.context.append")]
    DelegationContextAppend {
        delegation_item_id: String,
        #[serde(skip_serializing_if = "Option::is_none")]
        channel: Option<RealtimeContextAppendChannel>,
        content: Vec<FramelessInputTextContent>,
    },
    #[serde(rename = "session.context.append")]
    SessionContextAppend {
        #[serde(skip_serializing_if = "Option::is_none")]
        channel: Option<RealtimeContextAppendChannel>,
        content: Vec<FramelessInputTextContent>,
    },
    #[serde(rename = "session.close")]
    SessionClose,
    #[serde(rename = "response.create")]
    ResponseCreate,
    #[serde(rename = "session.update")]
    SessionUpdate { session: SessionUpdateSession },
    #[serde(rename = "session.update")]
    FramelessSessionUpdate { session: Value },
    #[serde(rename = "conversation.item.create")]
    ConversationItemCreate { item: ConversationItemPayload },
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct FramelessInputTextContent {
    #[serde(rename = "type")]
    pub(super) r#type: FramelessContentType,
    pub(super) text: String,
}

#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum FramelessContentType {
    InputText,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionUpdateSession {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) id: Option<String>,
    #[serde(rename = "type")]
    pub(super) r#type: SessionType,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) model: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) instructions: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) output_modalities: Option<Vec<String>>,
    pub(super) audio: SessionAudio,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) tools: Option<Vec<SessionFunctionTool>>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) tool_choice: Option<String>,
}

#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum SessionType {
    Quicksilver,
    Realtime,
    Transcription,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionAudio {
    pub(super) input: SessionAudioInput,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) output: Option<SessionAudioOutput>,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionAudioInput {
    pub(super) format: SessionAudioFormat,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) noise_reduction: Option<SessionNoiseReduction>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) transcription: Option<SessionInputAudioTranscription>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) turn_detection: Option<SessionTurnDetection>,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionInputAudioTranscription {
    pub(super) model: String,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionAudioFormat {
    #[serde(rename = "type")]
    pub(super) r#type: AudioFormatType,
    pub(super) rate: u32,
}

#[derive(Debug, Clone, Copy, Serialize)]
pub(super) enum AudioFormatType {
    #[serde(rename = "audio/pcm")]
    AudioPcm,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionAudioOutput {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub(super) format: Option<SessionAudioOutputFormat>,
    pub(super) voice: RealtimeVoice,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionNoiseReduction {
    #[serde(rename = "type")]
    pub(super) r#type: NoiseReductionType,
}

#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum NoiseReductionType {
    NearField,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionTurnDetection {
    #[serde(rename = "type")]
    pub(super) r#type: TurnDetectionType,
    pub(super) interrupt_response: bool,
    pub(super) create_response: bool,
    pub(super) silence_duration_ms: u32,
}

#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum TurnDetectionType {
    ServerVad,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionAudioOutputFormat {
    #[serde(rename = "type")]
    pub(super) r#type: AudioFormatType,
    pub(super) rate: u32,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct ConversationMessageItem {
    #[serde(rename = "type")]
    pub(super) r#type: ConversationItemType,
    pub(super) role: ConversationTextRole,
    pub(super) content: Vec<ConversationItemContent>,
}

#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum ConversationItemType {
    Message,
    FunctionCallOutput,
}

#[derive(Debug, Clone, Serialize)]
#[serde(untagged)]
pub(super) enum ConversationItemPayload {
    Message(ConversationMessageItem),
    FunctionCallOutput(ConversationFunctionCallOutputItem),
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct ConversationFunctionCallOutputItem {
    #[serde(rename = "type")]
    pub(super) r#type: ConversationItemType,
    pub(super) call_id: String,
    pub(super) output: String,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct ConversationItemContent {
    #[serde(rename = "type")]
    pub(super) r#type: ConversationContentType,
    pub(super) text: String,
}

#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum ConversationContentType {
    InputText,
    OutputText,
}

#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionFunctionTool {
    #[serde(rename = "type")]
    pub(super) r#type: SessionToolType,
    pub(super) name: String,
    pub(super) description: String,
    pub(super) parameters: Value,
}

#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum SessionToolType {
    Function,
}

pub(super) fn parse_realtime_event(
    payload: &str,
    event_parser: RealtimeEventParser,
) -> Option<RealtimeEvent> {
    match event_parser {
        RealtimeEventParser::V1 => parse_realtime_event_v1(payload),
        RealtimeEventParser::FramelessBidi => parse_frameless_bidi_event(payload),
        RealtimeEventParser::RealtimeV2 => parse_realtime_event_v2(payload),
    }
}