Download codex-rs/core/tests/suite/code_mode.rs from SaylorTwift/codex: direct link, hf CLI and curl.
- Browser
- Download file 275 kB
-
https://huggingface.co/SaylorTwift/codex/resolve/main/codex-rs/core/tests/suite/code_mode.rs
- Command line
-
hf download hf://SaylorTwift/codex/codex-rs/core/tests/suite/code_mode.rs
-
curl -L -o code_mode.rs https://huggingface.co/SaylorTwift/codex/resolve/main/codex-rs/core/tests/suite/code_mode.rs
275 kB
| use anyhow::Result; | |
| use base64::Engine; | |
| use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; | |
| use codex_config::types::McpServerConfig; | |
| use codex_config::types::McpServerTransportConfig; | |
| use codex_core::StartThreadOptions; | |
| use codex_core::TurnInputRequest; | |
| use codex_core::config::Config; | |
| use codex_core::config::Constrained; | |
| use codex_core::config::CurrentTimeReminderConfig; | |
| use codex_core::context::NodeReplReviewEvidence; | |
| use codex_extension_api::ExtensionData; | |
| use codex_extension_api::ExtensionFuture; | |
| use codex_extension_api::ExtensionRegistryBuilder; | |
| use codex_extension_api::McpServerContribution; | |
| use codex_extension_api::McpServerContributionContext; | |
| use codex_extension_api::McpServerContributor; | |
| use codex_extension_api::ToolCallOutcome; | |
| use codex_extension_api::ToolContributor; | |
| use codex_extension_api::ToolFinishInput; | |
| use codex_extension_api::ToolLifecycleContributor; | |
| use codex_extension_api::ToolLifecycleFuture; | |
| use codex_extension_api::ToolStartInput; | |
| use codex_features::CurrentTimeSource; | |
| use codex_features::Feature; | |
| use codex_login::CodexAuth; | |
| use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; | |
| use codex_mcp::codex_apps_mcp_server_config; | |
| use codex_models_manager::bundled_models_response; | |
| use codex_protocol::config_types::ApprovalsReviewer; | |
| use codex_protocol::config_types::CollaborationMode; | |
| use codex_protocol::config_types::ModeKind; | |
| use codex_protocol::config_types::Settings; | |
| use codex_protocol::config_types::WebSearchMode; | |
| use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem; | |
| use codex_protocol::dynamic_tools::DynamicToolFunctionSpec; | |
| use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec; | |
| use codex_protocol::dynamic_tools::DynamicToolNamespaceTool; | |
| use codex_protocol::dynamic_tools::DynamicToolResponse; | |
| use codex_protocol::dynamic_tools::DynamicToolSpec; | |
| use codex_protocol::models::ContentItem; | |
| use codex_protocol::models::ImageDetail; | |
| use codex_protocol::models::ImageReference; | |
| use codex_protocol::models::PermissionProfile; | |
| use codex_protocol::models::ResponseItem; | |
| use codex_protocol::openai_models::InputModality; | |
| use codex_protocol::openai_models::ToolMode; | |
| use codex_protocol::protocol::AskForApproval; | |
| use codex_protocol::protocol::EventMsg; | |
| use codex_protocol::protocol::Op; | |
| use codex_protocol::protocol::ThreadHistoryMode; | |
| use codex_protocol::protocol::ThreadSettingsOverrides; | |
| use codex_protocol::user_input::UserInput; | |
| use codex_tools::FreeformTool; | |
| use codex_tools::FreeformToolFormat; | |
| use codex_tools::FunctionCallError; | |
| use codex_tools::JsonToolOutput; | |
| use codex_tools::ResponsesApiNamespace; | |
| use codex_tools::ResponsesApiNamespaceTool; | |
| use codex_tools::ToolCall; | |
| use codex_tools::ToolExecutor; | |
| use codex_tools::ToolExecutorFuture; | |
| use codex_tools::ToolName; | |
| use codex_tools::ToolOutput; | |
| use codex_tools::ToolPayload; | |
| use codex_tools::ToolSpec; | |
| use codex_web_search_extension::install as install_web_search_extension; | |
| use core_test_support::apps_test_server::AppsTestServer; | |
| use core_test_support::apps_test_server::AppsTestToolLoading; | |
| use core_test_support::apps_test_server::DIRECT_CALENDAR_APP_ONLY_TOOL; | |
| use core_test_support::apps_test_server::recorded_apps_tool_calls; | |
| use core_test_support::apps_test_server::search_capable_apps_builder; | |
| use core_test_support::assert_regex_match; | |
| use core_test_support::responses; | |
| use core_test_support::responses::ResponseMock; | |
| use core_test_support::responses::ResponsesRequest; | |
| use core_test_support::responses::ev_assistant_message; | |
| use core_test_support::responses::ev_completed; | |
| use core_test_support::responses::ev_custom_tool_call; | |
| use core_test_support::responses::ev_response_created; | |
| use core_test_support::responses::namespace_child_tool; | |
| use core_test_support::responses::sse; | |
| use core_test_support::skip_if_no_network; | |
| use core_test_support::skip_if_sandbox; | |
| use core_test_support::skip_if_wine_exec; | |
| use core_test_support::stdio_server_bin; | |
| use core_test_support::test_codex::TestCodex; | |
| use core_test_support::test_codex::TestCodexBuilder; | |
| use core_test_support::test_codex::test_codex; | |
| use core_test_support::test_codex::turn_permission_fields; | |
| use core_test_support::wait_for_event; | |
| use core_test_support::wait_for_event_match; | |
| use core_test_support::wait_for_mcp_server; | |
| use image::DynamicImage; | |
| use image::GenericImageView; | |
| use image::ImageBuffer; | |
| use image::ImageDecoder; | |
| use image::ImageEncoder; | |
| use image::ImageFormat; | |
| use image::ImageReader; | |
| use image::Rgba; | |
| use image::codecs::png::PngEncoder; | |
| use image::metadata::Orientation; | |
| use pretty_assertions::assert_eq; | |
| use serde_json::Value; | |
| use std::collections::HashMap; | |
| use std::fs; | |
| use std::io::Cursor; | |
| use std::path::Path; | |
| use std::sync::Arc; | |
| use std::sync::Mutex; | |
| use std::sync::atomic::AtomicUsize; | |
| use std::sync::atomic::Ordering; | |
| use std::time::Duration; | |
| use std::time::Instant; | |
| use test_case::test_case; | |
| use tokio::sync::oneshot; | |
| use wiremock::Mock; | |
| use wiremock::MockServer; | |
| use wiremock::Request; | |
| use wiremock::ResponseTemplate; | |
| use wiremock::matchers::body_partial_json; | |
| use wiremock::matchers::method; | |
| use wiremock::matchers::path; | |
| use super::rmcp_client::remote_aware_environment_id; | |
| use super::rmcp_client::remote_aware_stdio_server_bin; | |
| fn custom_tool_output_items(req: &ResponsesRequest, call_id: &str) -> Vec<Value> { | |
| match req.custom_tool_call_output(call_id).get("output") { | |
| Some(Value::Array(items)) => items.clone(), | |
| Some(Value::String(text)) => { | |
| vec![serde_json::json!({ "type": "input_text", "text": text })] | |
| } | |
| _ => panic!("custom tool output should be serialized as text or content items"), | |
| } | |
| } | |
| fn tool_names(body: &Value) -> Vec<String> { | |
| body.get("tools") | |
| .and_then(Value::as_array) | |
| .map(|tools| { | |
| tools | |
| .iter() | |
| .filter_map(|tool| { | |
| tool.get("name") | |
| .or_else(|| tool.get("type")) | |
| .and_then(Value::as_str) | |
| .map(str::to_string) | |
| }) | |
| .collect() | |
| }) | |
| .unwrap_or_default() | |
| } | |
| fn function_tool_output_items(req: &ResponsesRequest, call_id: &str) -> Vec<Value> { | |
| match req.function_call_output(call_id).get("output") { | |
| Some(Value::Array(items)) => items.clone(), | |
| Some(Value::String(text)) => { | |
| vec![serde_json::json!({ "type": "input_text", "text": text })] | |
| } | |
| _ => panic!("function tool output should be serialized as text or content items"), | |
| } | |
| } | |
| fn text_item(items: &[Value], index: usize) -> &str { | |
| items[index] | |
| .get("text") | |
| .and_then(Value::as_str) | |
| .expect("content item should be input_text") | |
| } | |
| fn extract_running_cell_id(text: &str) -> String { | |
| text.strip_prefix("Script running with cell ID ") | |
| .and_then(|rest| rest.split('\n').next()) | |
| .expect("running header should contain a cell ID") | |
| .to_string() | |
| } | |
| fn wait_for_file_source(path: &Path) -> Result<String> { | |
| let quoted_path = shlex::try_join([path.to_string_lossy().as_ref()])?; | |
| let command = format!("if [ -f {quoted_path} ]; then printf ready; fi"); | |
| Ok(format!( | |
| r#"while ((await tools.exec_command({{ cmd: {command:?} }})).output !== "ready") {{ | |
| }}"# | |
| )) | |
| } | |
| fn custom_tool_output_body_and_success( | |
| req: &ResponsesRequest, | |
| call_id: &str, | |
| ) -> (String, Option<bool>) { | |
| let (content, success) = req | |
| .custom_tool_call_output_content_and_success(call_id) | |
| .expect("custom tool output should be present"); | |
| let items = custom_tool_output_items(req, call_id); | |
| let text_items = items | |
| .iter() | |
| .filter_map(|item| item.get("text").and_then(Value::as_str)) | |
| .collect::<Vec<_>>(); | |
| let output = match text_items.as_slice() { | |
| [] => content.unwrap_or_default(), | |
| [only] => (*only).to_string(), | |
| [_, rest @ ..] => rest.concat(), | |
| }; | |
| (output, success) | |
| } | |
| fn custom_tool_output_last_non_empty_text(req: &ResponsesRequest, call_id: &str) -> Option<String> { | |
| match req.custom_tool_call_output(call_id).get("output") { | |
| Some(Value::String(text)) if !text.trim().is_empty() => Some(text.clone()), | |
| Some(Value::Array(items)) => items | |
| .iter() | |
| .filter_map(|item| item.get("text").and_then(Value::as_str)) | |
| .rfind(|text| !text.trim().is_empty()) | |
| .map(str::to_string), | |
| Some(Value::String(_)) | |
| | Some(Value::Object(_)) | |
| | Some(Value::Number(_)) | |
| | Some(Value::Bool(_)) | |
| | Some(Value::Null) | |
| | None => None, | |
| } | |
| } | |
| async fn run_code_mode_turn( | |
| server: &MockServer, | |
| prompt: &str, | |
| code: &str, | |
| ) -> Result<(TestCodex, ResponseMock)> { | |
| run_code_mode_turn_with_config(server, prompt, code, |_| {}).await | |
| } | |
| async fn run_code_mode_turn_with_config( | |
| server: &MockServer, | |
| prompt: &str, | |
| code: &str, | |
| configure: impl FnOnce(&mut Config) + Send + 'static, | |
| ) -> Result<(TestCodex, ResponseMock)> { | |
| run_code_mode_turn_with_model_and_config(server, prompt, code, "test-gpt-5.1-codex", configure) | |
| .await | |
| } | |
| async fn run_code_mode_turn_with_model_and_config( | |
| server: &MockServer, | |
| prompt: &str, | |
| code: &str, | |
| model: &'static str, | |
| configure: impl FnOnce(&mut Config) + Send + 'static, | |
| ) -> Result<(TestCodex, ResponseMock)> { | |
| let builder = test_codex().with_model(model).with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| let _ = config.features.enable(Feature::ExecutedToolCallMetadata); | |
| configure(config); | |
| }); | |
| run_code_mode_turn_with_builder(server, prompt, code, builder).await | |
| } | |
| async fn run_code_mode_turn_with_builder( | |
| server: &MockServer, | |
| prompt: &str, | |
| code: &str, | |
| mut builder: TestCodexBuilder, | |
| ) -> Result<(TestCodex, ResponseMock)> { | |
| let test = builder.build(server).await?; | |
| responses::mount_sse_once( | |
| server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn(prompt).await?; | |
| Ok((test, second_mock)) | |
| } | |
| async fn run_unavailable_code_mode_turn( | |
| server: &MockServer, | |
| test: &TestCodex, | |
| ) -> Result<(Value, Vec<String>)> { | |
| let response_mock = responses::mount_sse_once( | |
| server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| test.codex | |
| .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { | |
| text: "list available tools".to_string(), | |
| text_elements: Vec::new(), | |
| }])) | |
| .await?; | |
| let mut warnings = Vec::new(); | |
| loop { | |
| match wait_for_event(&test.codex, |_| true).await { | |
| EventMsg::Warning(warning) => warnings.push(warning.message), | |
| EventMsg::TurnComplete(_) => break, | |
| _ => {} | |
| } | |
| } | |
| Ok((response_mock.single_request().body_json(), warnings)) | |
| } | |
| async fn missing_process_host_falls_back_to_direct_tools_and_warns_once() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_code_mode_host_program("codex-code-mode-host-does-not-exist".into()) | |
| .with_config(|config| { | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("code mode should be enabled"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| let (first_body, first_warnings) = run_unavailable_code_mode_turn(&server, &test).await?; | |
| let first_tools = tool_names(&first_body); | |
| assert!( | |
| first_tools | |
| .iter() | |
| .all(|name| name != "exec" && name != "wait"), | |
| "unavailable code mode must not expose code-mode tools: {first_tools:?}" | |
| ); | |
| assert!( | |
| first_warnings.iter().any(|warning| { | |
| warning.contains("Code Mode is unavailable") | |
| && warning.contains("codex-code-mode-host-does-not-exist") | |
| }), | |
| "missing host should produce an actionable warning: {first_warnings:?}" | |
| ); | |
| let (_, second_warnings) = run_unavailable_code_mode_turn(&server, &test).await?; | |
| assert!( | |
| second_warnings | |
| .iter() | |
| .all(|warning| !warning.contains("Code Mode is unavailable")), | |
| "host availability warning should be emitted once per thread: {second_warnings:?}" | |
| ); | |
| Ok(()) | |
| } | |
| async fn missing_process_host_keeps_code_mode_only_and_fails_closed() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_code_mode_host_program("codex-code-mode-host-does-not-exist".into()) | |
| .with_config(|config| { | |
| config | |
| .features | |
| .enable(Feature::CodeModeOnly) | |
| .expect("code mode should be enabled"); | |
| }); | |
| let (_test, follow_up_mock) = run_code_mode_turn_with_builder( | |
| &server, | |
| "Run required code mode", | |
| "text('unreachable')", | |
| builder, | |
| ) | |
| .await?; | |
| let request = follow_up_mock.single_request(); | |
| let tools = tool_names(&request.body_json()); | |
| assert!( | |
| tools.iter().any(|name| name == "exec") && tools.iter().any(|name| name == "wait"), | |
| "code-mode-only must retain code-mode tools: {tools:?}" | |
| ); | |
| assert!( | |
| tools | |
| .iter() | |
| .all(|name| !matches!(name.as_str(), "shell" | "exec_command")), | |
| "code-mode-only must never expose direct shell tools: {tools:?}" | |
| ); | |
| let (output, _) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert!( | |
| output.contains("codex-code-mode-host-does-not-exist"), | |
| "code-mode-only must report the host failure: {output}" | |
| ); | |
| Ok(()) | |
| } | |
| async fn missing_process_host_fails_closed_when_direct_fallback_is_disabled() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_code_mode_host_program("codex-code-mode-host-does-not-exist".into()) | |
| .with_config(|config| { | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("code mode should be enabled"); | |
| config.code_mode.disable_in_process_fallback = true; | |
| }); | |
| let (_test, follow_up_mock) = run_code_mode_turn_with_builder( | |
| &server, | |
| "Run required code mode", | |
| "text('unreachable')", | |
| builder, | |
| ) | |
| .await?; | |
| let request = follow_up_mock.single_request(); | |
| let tools = tool_names(&request.body_json()); | |
| assert!( | |
| tools.iter().any(|name| name == "exec") && tools.iter().any(|name| name == "wait"), | |
| "disabled fallback must retain code-mode tools: {tools:?}" | |
| ); | |
| let (output, _) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert!( | |
| output.contains("codex-code-mode-host-does-not-exist"), | |
| "disabled fallback must report the host failure: {output}" | |
| ); | |
| Ok(()) | |
| } | |
| async fn disabled_process_host_with_fallback_disabled_attempts_the_host() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_config(|config| { | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("code mode should be enabled"); | |
| config | |
| .features | |
| .disable(Feature::CodeModeHost) | |
| .expect("code-mode host should be disabled"); | |
| config.code_mode.disable_in_process_fallback = true; | |
| }); | |
| let (_test, follow_up_mock) = run_code_mode_turn_with_builder( | |
| &server, | |
| "Run required code mode", | |
| "text('unreachable')", | |
| builder, | |
| ) | |
| .await?; | |
| let request = follow_up_mock.single_request(); | |
| let (output, _) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert!( | |
| output.contains("failed to spawn code-mode host"), | |
| "disabled fallback must still attempt the standalone host: {output}" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_call_standalone_web_search() -> Result<()> { | |
| assert_code_mode_standalone_web_search(WebSearchMode::Live, serde_json::json!(true)).await | |
| } | |
| async fn code_mode_can_call_indexed_standalone_web_search() -> Result<()> { | |
| assert_code_mode_standalone_web_search(WebSearchMode::Indexed, serde_json::json!("indexed")) | |
| .await | |
| } | |
| async fn assert_code_mode_standalone_web_search( | |
| web_search_mode: WebSearchMode, | |
| expected_external_web_access: Value, | |
| ) -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| Mock::given(method("POST")) | |
| .and(path("/v1/alpha/search")) | |
| .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ | |
| "output": "Search result", | |
| }))) | |
| .expect(1) | |
| .mount(&server) | |
| .await; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| r#" | |
| const result = await tools.web__run({ | |
| search_query: [{ q: "standalone web search" }], | |
| }); | |
| text(result); | |
| "#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let follow_up_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| let auth = CodexAuth::from_api_key("dummy"); | |
| let auth_manager = codex_core::test_support::auth_manager_from_auth(auth.clone()); | |
| let mut extension_builder = ExtensionRegistryBuilder::<Config>::new(); | |
| install_web_search_extension(&mut extension_builder, auth_manager); | |
| let mut builder = test_codex() | |
| .with_auth(auth) | |
| .with_extensions(Arc::new(extension_builder.build())) | |
| .with_model("test-gpt-5.1-codex") | |
| .with_config(move |config| { | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("code mode should be enabled"); | |
| config | |
| .features | |
| .enable(Feature::StandaloneWebSearch) | |
| .expect("standalone web search should be enabled"); | |
| config | |
| .web_search_mode | |
| .set(web_search_mode) | |
| .expect("web search mode should be accepted"); | |
| }); | |
| let test = builder.build(&server).await?; | |
| test.submit_turn("Search the web from code mode").await?; | |
| let search_request = server | |
| .received_requests() | |
| .await | |
| .expect("received requests should be available") | |
| .into_iter() | |
| .find(|request| request.url.path() == "/v1/alpha/search") | |
| .expect("standalone search request should be sent"); | |
| let search_body = search_request | |
| .body_json::<Value>() | |
| .expect("search request body should be JSON"); | |
| assert_eq!( | |
| search_body["model"], | |
| serde_json::json!("test-gpt-5.1-codex") | |
| ); | |
| assert_eq!( | |
| search_body["commands"], | |
| serde_json::json!({ | |
| "search_query": [{"q": "standalone web search"}], | |
| }) | |
| ); | |
| assert_eq!( | |
| search_body["settings"], | |
| serde_json::json!({ | |
| "allowed_callers": ["direct"], | |
| "external_web_access": expected_external_web_access, | |
| }) | |
| ); | |
| assert_eq!( | |
| custom_tool_output_last_non_empty_text(&follow_up_mock.single_request(), "call-1"), | |
| Some("Search result".to_string()) | |
| ); | |
| Ok(()) | |
| } | |
| async fn run_code_mode_turn_with_rmcp( | |
| server: &MockServer, | |
| prompt: &str, | |
| code: &str, | |
| ) -> Result<(TestCodex, ResponseMock)> { | |
| run_code_mode_turn_with_rmcp_model(server, prompt, code, "test-gpt-5.1-codex").await | |
| } | |
| async fn run_code_mode_turn_with_rmcp_model( | |
| server: &MockServer, | |
| prompt: &str, | |
| code: &str, | |
| model: &'static str, | |
| ) -> Result<(TestCodex, ResponseMock)> { | |
| run_code_mode_turn_with_rmcp_config( | |
| server, prompt, code, model, /*code_mode_only*/ false, | |
| /*non_prefixed_mcp_tool_names*/ false, | |
| ) | |
| .await | |
| } | |
| async fn run_code_mode_turn_with_rmcp_mode( | |
| server: &MockServer, | |
| prompt: &str, | |
| code: &str, | |
| code_mode_only: bool, | |
| ) -> Result<(TestCodex, ResponseMock)> { | |
| run_code_mode_turn_with_rmcp_config( | |
| server, | |
| prompt, | |
| code, | |
| "test-gpt-5.1-codex", | |
| code_mode_only, | |
| /*non_prefixed_mcp_tool_names*/ false, | |
| ) | |
| .await | |
| } | |
| async fn run_code_mode_turn_with_rmcp_config( | |
| server: &MockServer, | |
| prompt: &str, | |
| code: &str, | |
| model: &'static str, | |
| code_mode_only: bool, | |
| non_prefixed_mcp_tool_names: bool, | |
| ) -> Result<(TestCodex, ResponseMock)> { | |
| let rmcp_test_server_bin = stdio_server_bin()?; | |
| let mut builder = test_codex().with_model(model).with_config(move |config| { | |
| let _ = if code_mode_only { | |
| config.features.enable(Feature::CodeModeOnly) | |
| } else { | |
| config.features.enable(Feature::CodeMode) | |
| }; | |
| if non_prefixed_mcp_tool_names { | |
| let _ = config.features.enable(Feature::NonPrefixedMcpToolNames); | |
| } | |
| let mut servers = config.mcp_servers.get().clone(); | |
| servers.insert( | |
| "rmcp".to_string(), | |
| McpServerConfig { | |
| auth: Default::default(), | |
| transport: McpServerTransportConfig::Stdio { | |
| command: rmcp_test_server_bin, | |
| args: Vec::new(), | |
| env: Some(HashMap::from([( | |
| "MCP_TEST_VALUE".to_string(), | |
| "propagated-env".to_string(), | |
| )])), | |
| env_vars: Vec::new(), | |
| cwd: None, | |
| }, | |
| environment_id: "local".to_string(), | |
| enabled: true, | |
| required: false, | |
| supports_parallel_tool_calls: false, | |
| omit_tools_from: None, | |
| disabled_reason: None, | |
| startup_timeout_sec: Some(Duration::from_secs(10)), | |
| tool_timeout_sec: None, | |
| default_tools_approval_mode: None, | |
| enabled_tools: None, | |
| disabled_tools: None, | |
| scopes: None, | |
| oauth: None, | |
| oauth_resource: None, | |
| tools: HashMap::new(), | |
| }, | |
| ); | |
| config | |
| .mcp_servers | |
| .set(servers) | |
| .expect("test mcp servers should accept any configuration"); | |
| }); | |
| let test = builder.build(server).await?; | |
| wait_for_mcp_server(&test.codex, "rmcp").await?; | |
| responses::mount_sse_once( | |
| server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn(prompt).await?; | |
| Ok((test, second_mock)) | |
| } | |
| async fn code_mode_can_return_exec_command_output() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "use exec to run exec_command", | |
| r#" | |
| text(JSON.stringify(await tools.exec_command({ cmd: "printf code_mode_exec_marker" }))); | |
| "#, | |
| ) | |
| .await?; | |
| let items = custom_tool_output_items(&second_mock.single_request(), "call-1"); | |
| assert_eq!(items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&items, /*index*/ 0), | |
| ); | |
| let parsed: Value = serde_json::from_str(text_item(&items, /*index*/ 1))?; | |
| assert!( | |
| parsed | |
| .get("chunk_id") | |
| .and_then(Value::as_str) | |
| .is_some_and(|chunk_id| !chunk_id.is_empty()) | |
| ); | |
| assert_eq!( | |
| parsed.get("output").and_then(Value::as_str), | |
| Some("code_mode_exec_marker"), | |
| ); | |
| assert_eq!(parsed.get("exit_code").and_then(Value::as_i64), Some(0)); | |
| assert!(parsed.get("wall_time_seconds").is_some()); | |
| assert!(parsed.get("session_id").is_none()); | |
| Ok(()) | |
| } | |
| async fn code_mode_exec_holds_captured_result_during_elicitation() -> Result<()> { | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let first_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", "text('captured');"), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| assert_eq!( | |
| test.codex.increment_out_of_band_elicitation_count().await?, | |
| 1 | |
| ); | |
| assert_eq!( | |
| test.codex.increment_out_of_band_elicitation_count().await?, | |
| 2 | |
| ); | |
| let release_elicitation = async { | |
| tokio::time::timeout(Duration::from_secs(5), async { | |
| while first_mock.requests().is_empty() { | |
| tokio::time::sleep(Duration::from_millis(10)).await; | |
| } | |
| }) | |
| .await | |
| .expect("initial response request should arrive"); | |
| tokio::time::sleep(Duration::from_millis(250)).await; | |
| assert!( | |
| second_mock.requests().is_empty(), | |
| "captured exec result should not return during an elicitation" | |
| ); | |
| assert_eq!( | |
| test.codex.decrement_out_of_band_elicitation_count().await?, | |
| 1 | |
| ); | |
| tokio::time::sleep(Duration::from_millis(100)).await; | |
| assert!( | |
| second_mock.requests().is_empty(), | |
| "captured exec result should wait for every elicitation" | |
| ); | |
| assert_eq!( | |
| test.codex.decrement_out_of_band_elicitation_count().await?, | |
| 0 | |
| ); | |
| Ok::<(), anyhow::Error>(()) | |
| }; | |
| tokio::try_join!(test.submit_turn("run code mode"), release_elicitation)?; | |
| second_mock.single_request(); | |
| Ok(()) | |
| } | |
| async fn code_mode_only_restricts_prompt_tools() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let resp_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let mut builder = test_codex().with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeModeOnly); | |
| }); | |
| let test = builder.build(&server).await?; | |
| test.submit_turn("list tools in code mode only").await?; | |
| let first_body = resp_mock.single_request().body_json(); | |
| assert_eq!( | |
| tool_names(&first_body), | |
| vec![ | |
| "exec".to_string(), | |
| "wait".to_string(), | |
| "request_user_input".to_string(), | |
| "web_search".to_string() | |
| ] | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_excludes_mcp_servers_using_their_configured_identity() -> Result<()> { | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "requires a Windows test_stdio_server in the Wine-exec environment" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| for omit_tools_from in [ | |
| &["code_mode"][..], | |
| &["deferred"][..], | |
| &["direct"][..], | |
| &["code_mode", "deferred"][..], | |
| &["code_mode", "direct"][..], | |
| &["deferred", "direct"][..], | |
| &["code_mode", "deferred", "direct"][..], | |
| &[][..], | |
| ] { | |
| for (code_mode_only, supports_search_tool, non_prefixed_mcp_tool_names) in [ | |
| (false, false, false), | |
| (false, false, true), | |
| (false, true, false), | |
| (false, true, true), | |
| (true, false, false), | |
| (true, false, true), | |
| (true, true, false), | |
| (true, true, true), | |
| ] { | |
| let server = responses::start_mock_server().await; | |
| let response = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let rmcp_test_server_bin = remote_aware_stdio_server_bin()?; | |
| let environment_id = remote_aware_environment_id(); | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.4", move |model| { | |
| model.supports_search_tool = supports_search_tool; | |
| }) | |
| .with_config(move |config| { | |
| config | |
| .features | |
| .enable(if code_mode_only { | |
| Feature::CodeModeOnly | |
| } else { | |
| Feature::CodeMode | |
| }) | |
| .expect("test config should allow Code Mode"); | |
| if non_prefixed_mcp_tool_names { | |
| config | |
| .features | |
| .enable(Feature::NonPrefixedMcpToolNames) | |
| .expect("test config should allow unprefixed MCP tools"); | |
| } | |
| let mut servers = config.mcp_servers.get().clone(); | |
| servers.insert( | |
| "rmcp".to_string(), | |
| serde_json::from_value(serde_json::json!({ | |
| "command": rmcp_test_server_bin, | |
| "environment_id": environment_id, | |
| "cwd": config.cwd, | |
| "omit_tools_from": omit_tools_from, | |
| })) | |
| .expect("test MCP server config should be valid"), | |
| ); | |
| config | |
| .mcp_servers | |
| .set(servers) | |
| .expect("test config should allow MCP servers"); | |
| }); | |
| let base_test = builder.build_with_auto_env(&server).await?; | |
| let namespace = if non_prefixed_mcp_tool_names { | |
| "rmcp" | |
| } else { | |
| "mcp__rmcp" | |
| }; | |
| let new_thread = base_test | |
| .thread_manager | |
| .start_thread(StartThreadOptions { | |
| dynamic_tools: vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec { | |
| name: namespace.to_string(), | |
| description: "An unrelated dynamic tool sharing the MCP namespace." | |
| .to_string(), | |
| tools: vec![DynamicToolNamespaceTool::Function( | |
| DynamicToolFunctionSpec { | |
| name: "unrelated".to_string(), | |
| description: "A dynamic tool that must remain in Code Mode." | |
| .to_string(), | |
| input_schema: serde_json::json!({ | |
| "type": "object", | |
| "properties": {}, | |
| "additionalProperties": false, | |
| }), | |
| defer_loading: false, | |
| }, | |
| )], | |
| })], | |
| ..StartThreadOptions::new(base_test.config.clone()) | |
| }) | |
| .await?; | |
| let mut test = base_test; | |
| test.codex = new_thread.thread; | |
| test.session_configured = new_thread.session_configured; | |
| wait_for_mcp_server(&test.codex, "rmcp").await?; | |
| test.submit_turn("inspect the directly callable MCP tool") | |
| .await?; | |
| let body = response.single_request().body_json(); | |
| let omit_code_mode = omit_tools_from.contains(&"code_mode"); | |
| let omit_deferred = omit_tools_from.contains(&"deferred"); | |
| let omit_direct = omit_tools_from.contains(&"direct"); | |
| let available_directly = !omit_direct | |
| && (!supports_search_tool || omit_deferred || code_mode_only && omit_code_mode); | |
| let visible_directly = available_directly && (!code_mode_only || omit_code_mode); | |
| let echo_tool = namespace_child_tool(&body, namespace, "echo"); | |
| assert_eq!( | |
| echo_tool.is_some(), | |
| visible_directly, | |
| "MCP omissions must determine initial prompt exposure; \ | |
| omit_tools_from={omit_tools_from:?}, code_mode_only={code_mode_only}, \ | |
| supports_search_tool={supports_search_tool}, namespace={namespace}, tools={:?}", | |
| tool_names(&body) | |
| ); | |
| if let Some(echo_tool) = echo_tool { | |
| assert_eq!( | |
| echo_tool["description"] | |
| .as_str() | |
| .is_some_and(|description| description.contains("exec tool declaration:")), | |
| !omit_code_mode, | |
| "the MCP tool description must agree with Code Mode availability; \ | |
| omit_tools_from={omit_tools_from:?}, code_mode_only={code_mode_only}, \ | |
| namespace={namespace}" | |
| ); | |
| } | |
| let exec_description = body["tools"] | |
| .as_array() | |
| .expect("request should contain tools") | |
| .iter() | |
| .find_map(|tool| { | |
| (tool["name"].as_str() == Some("exec")) | |
| .then(|| tool["description"].as_str()) | |
| .flatten() | |
| }) | |
| .expect("Code Mode exec should remain available"); | |
| let nested_tool_name = format!("{namespace}__echo"); | |
| if code_mode_only { | |
| assert_eq!( | |
| exec_description.contains(&nested_tool_name), | |
| !omit_code_mode && (!supports_search_tool || omit_deferred), | |
| "the Code Mode declaration must agree with MCP tool exposure; \ | |
| omit_tools_from={omit_tools_from:?}, \ | |
| supports_search_tool={supports_search_tool}, \ | |
| nested_tool_name={nested_tool_name}" | |
| ); | |
| } | |
| let unrelated_nested_tool_name = format!("{namespace}__unrelated"); | |
| if code_mode_only { | |
| assert!( | |
| exec_description.contains(&unrelated_nested_tool_name), | |
| "MCP omissions must not exclude unrelated tools sharing its namespace; \ | |
| omit_tools_from={omit_tools_from:?}, \ | |
| supports_search_tool={supports_search_tool}, \ | |
| nested_tool_name={unrelated_nested_tool_name}" | |
| ); | |
| } else { | |
| let unrelated_tool = namespace_child_tool(&body, namespace, "unrelated") | |
| .unwrap_or_else(|| { | |
| panic!( | |
| "MCP omissions must not hide an unrelated same-namespace tool; \ | |
| omit_tools_from={omit_tools_from:?}, \ | |
| supports_search_tool={supports_search_tool}, namespace={namespace}, \ | |
| tools={:?}", | |
| tool_names(&body) | |
| ) | |
| }); | |
| assert!( | |
| unrelated_tool["description"] | |
| .as_str() | |
| .is_some_and(|description| { | |
| description.contains(&unrelated_nested_tool_name) | |
| && description.contains("exec tool declaration:") | |
| }), | |
| "an unrelated same-namespace tool must remain available inside Code Mode; \ | |
| omit_tools_from={omit_tools_from:?}, \ | |
| supports_search_tool={supports_search_tool}, \ | |
| nested_tool_name={unrelated_nested_tool_name}" | |
| ); | |
| } | |
| } | |
| } | |
| Ok(()) | |
| } | |
| async fn mcp_code_mode_exclusion_does_not_change_direct_mode_tool_exposure() -> Result<()> { | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "requires a Windows test_stdio_server in the Wine-exec environment" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| for omit_tools_from in [ | |
| &["code_mode"][..], | |
| &["deferred"][..], | |
| &["direct"][..], | |
| &["code_mode", "deferred"][..], | |
| &["code_mode", "direct"][..], | |
| &["deferred", "direct"][..], | |
| &["code_mode", "deferred", "direct"][..], | |
| &[][..], | |
| ] { | |
| for (supports_search_tool, non_prefixed_mcp_tool_names, direct_only_namespace) in [ | |
| (false, false, false), | |
| (false, false, true), | |
| (false, true, false), | |
| (false, true, true), | |
| (true, false, false), | |
| (true, false, true), | |
| (true, true, false), | |
| (true, true, true), | |
| ] { | |
| let namespace = if non_prefixed_mcp_tool_names { | |
| "rmcp" | |
| } else { | |
| "mcp__rmcp" | |
| }; | |
| let visible_directly = !omit_tools_from.contains(&"direct") | |
| && (!supports_search_tool | |
| || omit_tools_from.contains(&"deferred") | |
| || direct_only_namespace); | |
| let server = responses::start_mock_server().await; | |
| let response = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| if visible_directly { | |
| responses::ev_function_call_with_namespace( | |
| "call-1", | |
| namespace, | |
| "echo", | |
| r#"{"message":"ping"}"#, | |
| ) | |
| } else { | |
| ev_assistant_message("msg-1", "done") | |
| }, | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let follow_up_response = if visible_directly { | |
| Some( | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await, | |
| ) | |
| } else { | |
| None | |
| }; | |
| let rmcp_test_server_bin = remote_aware_stdio_server_bin()?; | |
| let environment_id = remote_aware_environment_id(); | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.4", move |model| { | |
| model.supports_search_tool = supports_search_tool; | |
| }) | |
| .with_config(move |config| { | |
| if non_prefixed_mcp_tool_names { | |
| config | |
| .features | |
| .enable(Feature::NonPrefixedMcpToolNames) | |
| .expect("test config should allow unprefixed MCP tools"); | |
| } | |
| if direct_only_namespace { | |
| config.code_mode.direct_only_tool_namespaces = vec![ | |
| if non_prefixed_mcp_tool_names { | |
| "rmcp" | |
| } else { | |
| "mcp__rmcp" | |
| } | |
| .to_string(), | |
| ]; | |
| } | |
| let mut servers = config.mcp_servers.get().clone(); | |
| servers.insert( | |
| "rmcp".to_string(), | |
| serde_json::from_value(serde_json::json!({ | |
| "command": rmcp_test_server_bin, | |
| "environment_id": environment_id, | |
| "cwd": config.cwd, | |
| "omit_tools_from": omit_tools_from, | |
| })) | |
| .expect("test MCP server config should be valid"), | |
| ); | |
| config | |
| .mcp_servers | |
| .set(servers) | |
| .expect("test config should allow MCP servers"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| wait_for_mcp_server(&test.codex, "rmcp").await?; | |
| test.submit_turn("inspect ordinary direct-mode MCP tool exposure") | |
| .await?; | |
| let body = response.single_request().body_json(); | |
| assert_eq!( | |
| namespace_child_tool(&body, namespace, "echo").is_some(), | |
| visible_directly, | |
| "MCP omissions must determine direct-mode tool exposure; \ | |
| omit_tools_from={omit_tools_from:?}, \ | |
| supports_search_tool={supports_search_tool}, \ | |
| direct_only_namespace={direct_only_namespace}, namespace={namespace}, tools={:?}", | |
| tool_names(&body) | |
| ); | |
| assert_eq!( | |
| tool_names(&body).iter().any(|name| name == "tool_search"), | |
| supports_search_tool, | |
| "MCP omissions must not disable tool search for other tools; \ | |
| omit_tools_from={omit_tools_from:?}, \ | |
| supports_search_tool={supports_search_tool}, \ | |
| direct_only_namespace={direct_only_namespace}, namespace={namespace}, tools={:?}", | |
| tool_names(&body) | |
| ); | |
| if let Some(follow_up_response) = follow_up_response { | |
| let output = follow_up_response | |
| .single_request() | |
| .function_call_output("call-1"); | |
| assert!( | |
| output["output"] | |
| .as_str() | |
| .is_some_and(|output| output.contains("ECHOING: ping")), | |
| "the advertised MCP tool must remain callable; \ | |
| omit_tools_from={omit_tools_from:?}, \ | |
| supports_search_tool={supports_search_tool}, \ | |
| direct_only_namespace={direct_only_namespace}, \ | |
| namespace={namespace}, output={output:?}" | |
| ); | |
| } | |
| } | |
| } | |
| Ok(()) | |
| } | |
| async fn code_mode_tool_call_completeness_is_private_and_opt_in( | |
| metadata_enabled: bool, | |
| yielded: bool, | |
| oversized: bool, | |
| ) -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = if oversized { | |
| r#" | |
| const args = { barrier: { id: "", participants: 1 } }; | |
| args.barrier.id = "x".repeat(8192 - JSON.stringify(args).length); | |
| for (let index = 0; index < 4; index++) await tools.test_sync_tool(args); | |
| text("done"); | |
| yield_control(); | |
| await new Promise(() => {}); | |
| "# | |
| } else if yielded { | |
| r#"await tools.test_sync_tool({}); text("done"); yield_control(); await new Promise(() => {});"# | |
| } else { | |
| r#"await tools.test_sync_tool({}); text("done");"# | |
| }; | |
| // run_code_mode_turn_with_config enables this feature; production keeps it off by default. | |
| let (test, follow_up) = | |
| run_code_mode_turn_with_config(&server, "Record a nested tool call", code, move |config| { | |
| if !metadata_enabled { | |
| config | |
| .features | |
| .disable(Feature::ExecutedToolCallMetadata) | |
| .expect("tool call metadata should be disabled"); | |
| } | |
| }) | |
| .await?; | |
| let request = follow_up.single_request(); | |
| let first_output = request.custom_tool_call_output("call-1"); | |
| let metadata = &first_output["internal_chat_message_metadata_passthrough"]; | |
| let calls = serde_json::json!([{ "name": "test_sync_tool", "arguments": {} }]); | |
| let complete = Value::Bool(true); | |
| if oversized { | |
| assert!( | |
| metadata["executed_tool_calls"] | |
| .as_array() | |
| .is_some_and(|calls| calls.iter().any(|call| call["arguments"] | |
| .get("_codex_executed_tool_call_truncated") | |
| .is_some())) | |
| ); | |
| } else { | |
| assert_eq!( | |
| metadata.get("executed_tool_calls"), | |
| metadata_enabled.then_some(&calls), | |
| ); | |
| } | |
| assert_eq!( | |
| metadata.get("tool_calls_complete"), | |
| (metadata_enabled && !yielded && !oversized).then_some(&complete), | |
| ); | |
| assert_eq!( | |
| metadata.get("cell_id").and_then(Value::as_str), | |
| metadata_enabled.then_some("call-1"), | |
| ); | |
| if yielded || oversized { | |
| let first_items = custom_tool_output_items(&request, "call-1"); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| if oversized { | |
| let compact = responses::mount_sse_once( | |
| &server, | |
| responses::sse(vec![ | |
| serde_json::json!({ | |
| "type": "response.output_item.done", | |
| "item": { | |
| "type": "compaction", | |
| "encrypted_content": "compacted history", | |
| }, | |
| }), | |
| responses::ev_completed("resp-compact"), | |
| ]), | |
| ) | |
| .await; | |
| test.codex.submit(Op::Compact).await?; | |
| wait_for_event(&test.codex, |event| { | |
| matches!(event, EventMsg::TurnComplete(_)) | |
| }) | |
| .await; | |
| assert_eq!( | |
| compact | |
| .single_request() | |
| .inputs_of_type("compaction_trigger") | |
| .len(), | |
| 1 | |
| ); | |
| let wait = responses::mount_function_call_agent_response( | |
| &server, | |
| "call-2", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id, | |
| "terminate": true, | |
| }))?, | |
| "wait", | |
| ) | |
| .await; | |
| test.submit_turn("Finish the compacted cell").await?; | |
| let final_request = wait.completion.single_request(); | |
| assert!( | |
| final_request | |
| .input() | |
| .iter() | |
| .all(|item| item["call_id"] != "call-1") | |
| ); | |
| assert!( | |
| final_request.function_call_output("call-2") | |
| ["internal_chat_message_metadata_passthrough"] | |
| .get("tool_calls_complete") | |
| .is_none() | |
| ); | |
| } | |
| } | |
| Ok(()) | |
| } | |
| async fn code_mode_wait_id_stays_known_after_compaction( | |
| terminal_call_id: &str, | |
| expected_complete: Option<bool>, | |
| ) -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| let _ = config.features.enable(Feature::CodeModeHost); | |
| let _ = config.features.enable(Feature::ExecutedToolCallMetadata); | |
| config.code_mode.disable_in_process_fallback = true; | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| let started = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_custom_tool_call( | |
| "call-exec", | |
| "exec", | |
| r#"await tools.test_sync_tool({}); text("started"); yield_control(); await new Promise(() => {});"#, | |
| ), | |
| ev_completed("resp-start"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-start", "running"), | |
| ev_completed("resp-start-done"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_turn("Start a cell").await?; | |
| let requests = started.requests(); | |
| assert_eq!(requests.len(), 2); | |
| let first_request = &requests[1]; | |
| let first_items = custom_tool_output_items(first_request, "call-exec"); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| assert_eq!(text_item(&first_items, /*index*/ 1), "started"); | |
| let first_output = first_request.custom_tool_call_output("call-exec"); | |
| let metadata = &first_output["internal_chat_message_metadata_passthrough"]; | |
| assert_eq!(metadata["cell_id"], "call-exec"); | |
| assert_eq!( | |
| metadata["executed_tool_calls"], | |
| serde_json::json!([{"name": "test_sync_tool", "arguments": {}}]), | |
| ); | |
| assert!(metadata.get("tool_calls_complete").is_none()); | |
| let yielded = responses::mount_function_call_agent_response( | |
| &server, | |
| "call-wait", | |
| &serde_json::json!({"cell_id": cell_id, "yield_time_ms": 1}).to_string(), | |
| "wait", | |
| ) | |
| .await; | |
| test.submit_turn("Wait once").await?; | |
| yielded.function_call.single_request(); | |
| let yielded_items = | |
| function_tool_output_items(&yielded.completion.single_request(), "call-wait"); | |
| assert_eq!( | |
| extract_running_cell_id(text_item(&yielded_items, /*index*/ 0)), | |
| cell_id, | |
| ); | |
| let compact = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| serde_json::json!({ | |
| "type": "response.output_item.done", | |
| "item": {"type": "compaction", "encrypted_content": "compacted history"}, | |
| }), | |
| ev_completed("resp-compact"), | |
| ]), | |
| ) | |
| .await; | |
| test.codex.submit(Op::Compact).await?; | |
| wait_for_event(&test.codex, |event| { | |
| matches!(event, EventMsg::TurnComplete(_)) | |
| }) | |
| .await; | |
| assert_eq!( | |
| compact | |
| .single_request() | |
| .inputs_of_type("compaction_trigger") | |
| .len(), | |
| 1, | |
| ); | |
| let terminal = responses::mount_function_call_agent_response( | |
| &server, | |
| terminal_call_id, | |
| &serde_json::json!({"cell_id": cell_id, "terminate": true}).to_string(), | |
| "wait", | |
| ) | |
| .await; | |
| test.submit_turn("Finish the compacted cell").await?; | |
| assert!( | |
| terminal | |
| .function_call | |
| .single_request() | |
| .input() | |
| .iter() | |
| .all(|item| { item["call_id"] != "call-exec" && item["call_id"] != "call-wait" }) | |
| ); | |
| let final_request = terminal.completion.single_request(); | |
| let final_items = function_tool_output_items(&final_request, terminal_call_id); | |
| assert_eq!(final_items.len(), 1); | |
| assert_regex_match( | |
| r"^Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z", | |
| text_item(&final_items, /*index*/ 0), | |
| ); | |
| let output = final_request.function_call_output(terminal_call_id); | |
| let metadata = &output["internal_chat_message_metadata_passthrough"]; | |
| assert_eq!( | |
| metadata.get("tool_calls_complete").and_then(Value::as_bool), | |
| expected_complete, | |
| ); | |
| assert!(metadata.get("executed_tool_calls").is_none()); | |
| test.codex.shutdown_and_wait().await?; | |
| Ok(()) | |
| } | |
| const RESULT_METADATA_TOOL: &str = "message_search"; | |
| const RESULT_METADATA_PRIVATE_RESULT: &str = "connector result text is not result metadata"; | |
| struct ResultMetadataTestControl { | |
| server: Mutex<McpServerContribution>, | |
| gate: Mutex<Option<(oneshot::Sender<()>, oneshot::Receiver<()>)>>, | |
| } | |
| impl McpServerContributor<Config> for ResultMetadataTestControl { | |
| fn id(&self) -> &'static str { | |
| "result_metadata_apps" | |
| } | |
| fn contribute<'a>( | |
| &'a self, | |
| _context: McpServerContributionContext<'a, Config>, | |
| ) -> ExtensionFuture<'a, Vec<McpServerContribution>> { | |
| let server = self.server.lock().unwrap().clone(); | |
| Box::pin(async move { vec![server] }) | |
| } | |
| } | |
| impl ToolLifecycleContributor for ResultMetadataTestControl { | |
| fn on_tool_finish<'a>(&'a self, input: ToolFinishInput<'a>) -> ToolLifecycleFuture<'a> { | |
| Box::pin(async move { | |
| if !input.tool_name.name.ends_with(RESULT_METADATA_TOOL) { | |
| return; | |
| } | |
| let gate = self.gate.lock().unwrap().take(); | |
| if let Some((reached, release)) = gate { | |
| reached.send(()).unwrap(); | |
| // Finish notification precedes accepted-result metadata capture. | |
| release | |
| .await | |
| .expect("test should release the accepted result"); | |
| } | |
| }) | |
| } | |
| } | |
| fn result_metadata_fixture_calls(input: &[Value]) -> impl Iterator<Item = &Value> { | |
| input.iter().flat_map(|output| { | |
| output | |
| .pointer("/internal_chat_message_metadata_passthrough/executed_tool_calls") | |
| .and_then(Value::as_array) | |
| .into_iter() | |
| .flatten() | |
| .map(move |_| output) | |
| }) | |
| } | |
| async fn mount_result_metadata_app( | |
| server: &MockServer, | |
| result_metadata: Option<Value>, | |
| is_error: bool, | |
| ) -> Result<AppsTestServer> { | |
| let apps_server = AppsTestServer::mount(server).await?; | |
| let mut tool_result = serde_json::json!({ | |
| "content": [{ "type": "text", "text": RESULT_METADATA_PRIVATE_RESULT }], | |
| "isError": is_error, | |
| }); | |
| if let Some(metadata) = result_metadata { | |
| tool_result["_meta"] = metadata; | |
| } | |
| for (method_name, result) in [ | |
| ( | |
| "tools/list", | |
| serde_json::json!({ | |
| "tools": [{ | |
| "name": RESULT_METADATA_TOOL, | |
| "annotations": { "readOnlyHint": true }, | |
| "inputSchema": { "type": "object" }, | |
| "_meta": { | |
| "connector_id": "test_connector", | |
| "connector_name": "MessageSearch", | |
| }, | |
| }], | |
| "nextCursor": null, | |
| }), | |
| ), | |
| ("tools/call", tool_result), | |
| ] { | |
| Mock::given(method("POST")) | |
| .and(path("/api/codex/ps/mcp")) | |
| .and(body_partial_json( | |
| serde_json::json!({ "method": method_name }), | |
| )) | |
| .respond_with(move |request: &Request| { | |
| let request: Value = serde_json::from_slice(&request.body).unwrap(); | |
| ResponseTemplate::new(200).set_body_json(serde_json::json!({ | |
| "jsonrpc": "2.0", | |
| "id": request["id"], | |
| "result": result, | |
| })) | |
| }) | |
| .with_priority(/*p*/ 1) | |
| .mount(server) | |
| .await; | |
| } | |
| Ok(apps_server) | |
| } | |
| fn result_metadata_apps_builder(base_url: String, account_email: &str) -> TestCodexBuilder { | |
| let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode( | |
| serde_json::json!({ | |
| "email": account_email, | |
| "https://api.openai.com/auth": { "chatgpt_account_id": "account_id" }, | |
| }) | |
| .to_string(), | |
| ); | |
| let auth = CodexAuth::from_external_chatgpt_tokens( | |
| &format!("e30.{payload}.signature"), | |
| "account_id", | |
| /*chatgpt_plan_type*/ None, | |
| ) | |
| .unwrap(); | |
| search_capable_apps_builder(base_url) | |
| .with_auth(auth) | |
| .with_config(|config| { | |
| for feature in [ | |
| Feature::CodeMode, | |
| Feature::CodeModeOnly, | |
| Feature::ExecutedToolCallMetadata, | |
| ] { | |
| config.features.enable(feature).unwrap(); | |
| } | |
| }) | |
| } | |
| async fn code_mode_mcp_metadata_keeps_originating_window_after_compaction() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let apps_server = mount_result_metadata_app( | |
| &server, /*result_metadata*/ None, /*is_error*/ false, | |
| ) | |
| .await?; | |
| let (reached_tx, reached_rx) = oneshot::channel(); | |
| let (release_tx, release_rx) = oneshot::channel(); | |
| let control = Arc::new(ResultMetadataTestControl { | |
| server: Mutex::new(McpServerContribution::HostedApps { | |
| config: Box::new(codex_apps_mcp_server_config( | |
| &apps_server.chatgpt_base_url, | |
| /*apps_mcp_product_sku*/ None, | |
| /*originator*/ None, | |
| )), | |
| }), | |
| gate: Mutex::new(Some((reached_tx, release_rx))), | |
| }); | |
| let mut extensions = ExtensionRegistryBuilder::<Config>::new(); | |
| extensions.tool_lifecycle_contributor(control); | |
| let mut builder = | |
| result_metadata_apps_builder(apps_server.chatgpt_base_url, "user@example.com") | |
| .with_model("test-gpt-5.1-codex") | |
| .with_extensions(Arc::new(extensions.build())); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| let originating_item_id = "ctc_before_compaction"; | |
| let mut exec_call = ev_custom_tool_call( | |
| "call-exec", | |
| "exec", | |
| &format!( | |
| r#"const tool = ALL_TOOLS.find(({{ name }}) => name.endsWith("{RESULT_METADATA_TOOL}")); | |
| const pending = tools[tool.name]({{}}); | |
| yield_control(); | |
| await pending; | |
| await tools[tool.name]({{}}); | |
| text("done");"#, | |
| ), | |
| ); | |
| exec_call["item"]["id"] = serde_json::json!(originating_item_id); | |
| let started = | |
| responses::mount_sse_once(&server, sse(vec![exec_call, ev_completed("resp-start")])).await; | |
| let yielded = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-start", "waiting"), | |
| ev_completed("resp-yield"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("Start an MCP call in code mode").await?; | |
| tokio::time::timeout(Duration::from_secs(10), reached_rx).await??; | |
| let first_items = custom_tool_output_items(&yielded.single_request(), "call-exec"); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| let originating_window_id = | |
| started.single_request().body_json()["client_metadata"]["x-codex-window-id"].clone(); | |
| assert!(originating_window_id.is_string()); | |
| let compact = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| serde_json::json!({ | |
| "type": "response.output_item.done", | |
| "item": {"type": "compaction", "encrypted_content": "compacted history"}, | |
| }), | |
| ev_completed("resp-compact"), | |
| ]), | |
| ) | |
| .await; | |
| test.codex.submit(Op::Compact).await?; | |
| wait_for_event(&test.codex, |event| { | |
| matches!(event, EventMsg::TurnComplete(_)) | |
| }) | |
| .await; | |
| assert_eq!( | |
| compact | |
| .single_request() | |
| .inputs_of_type("compaction_trigger") | |
| .len(), | |
| 1 | |
| ); | |
| let resumed = responses::mount_function_call_agent_response( | |
| &server, | |
| "call-wait", | |
| &serde_json::json!({"cell_id": cell_id, "yield_time_ms": 10_000}).to_string(), | |
| "wait", | |
| ) | |
| .await; | |
| release_tx.send(()).unwrap(); | |
| test.submit_turn("Finish the code-mode cell").await?; | |
| let resumed_request = resumed.function_call.single_request(); | |
| assert_ne!( | |
| resumed_request.body_json()["client_metadata"]["x-codex-window-id"], | |
| originating_window_id | |
| ); | |
| assert!( | |
| resumed_request | |
| .input() | |
| .iter() | |
| .all(|item| item["call_id"] != "call-exec") | |
| ); | |
| assert!( | |
| function_tool_output_items(&resumed.completion.single_request(), "call-wait") | |
| .iter() | |
| .any(|item| item["text"] == "done") | |
| ); | |
| let calls = recorded_apps_tool_calls(&server).await; | |
| assert_eq!(calls.len(), 2); | |
| for call in calls { | |
| let meta = &call["params"]["_meta"]; | |
| assert_eq!( | |
| serde_json::json!({ | |
| "sessionId": meta["sessionId"], | |
| "threadId": meta["threadId"], | |
| "itemId": meta["itemId"], | |
| "windowId": meta["windowId"], | |
| }), | |
| serde_json::json!({ | |
| "sessionId": test.session_configured.session_id, | |
| "threadId": test.session_configured.thread_id, | |
| "itemId": originating_item_id, | |
| "windowId": originating_window_id, | |
| }), | |
| ); | |
| } | |
| test.codex.shutdown_and_wait().await?; | |
| Ok(()) | |
| } | |
| fn assert_result_metadata_call( | |
| output: &Value, | |
| arguments: &Value, | |
| expected_metadata: Option<Value>, | |
| ) { | |
| let metadata = &output["internal_chat_message_metadata_passthrough"]; | |
| let call_name = metadata["executed_tool_calls"][0]["name"].as_str().unwrap(); | |
| assert!(call_name.ends_with(RESULT_METADATA_TOOL)); | |
| let mut expected_call = serde_json::json!({ "name": call_name, "arguments": arguments }); | |
| if let Some(result_metadata) = expected_metadata { | |
| expected_call["tool_result_metadata"] = result_metadata; | |
| } | |
| assert_eq!( | |
| metadata["executed_tool_calls"], | |
| serde_json::json!([expected_call]) | |
| ); | |
| assert!(metadata.get("tool_result_metadata").is_none()); | |
| } | |
| async fn result_metadata_follows_call_binding( | |
| metadata_enabled: bool, | |
| host_owned: bool, | |
| has_metadata: bool, | |
| is_error: bool, | |
| account_email: &str, | |
| tool_mode: ToolMode, | |
| ) -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let direct = matches!(tool_mode, ToolMode::Direct); | |
| let server = responses::start_mock_server().await; | |
| let result_metadata = has_metadata.then(|| { | |
| serde_json::json!({ | |
| "openai/resource_access": { | |
| "resource_coverage": "incomplete", | |
| "coverage_reasons": ["tool_error"], | |
| }, | |
| "provider_state": { | |
| "items": [{ "id": "room-42", "labels": ["one", "two"] }], | |
| "ready": true, | |
| "count": 2, | |
| "missing": null, | |
| }, | |
| }) | |
| }); | |
| let apps_server = mount_result_metadata_app(&server, result_metadata.clone(), is_error).await?; | |
| let mut builder = | |
| result_metadata_apps_builder(apps_server.chatgpt_base_url.clone(), account_email) | |
| .with_config(move |config| { | |
| if direct { | |
| config.features.disable(Feature::CodeMode).unwrap(); | |
| config.features.disable(Feature::CodeModeOnly).unwrap(); | |
| } | |
| if !metadata_enabled { | |
| config | |
| .features | |
| .disable(Feature::ExecutedToolCallMetadata) | |
| .unwrap(); | |
| } | |
| }); | |
| if !host_owned { | |
| let mut extensions = ExtensionRegistryBuilder::<Config>::new(); | |
| extensions.mcp_server_contributor(Arc::new(ResultMetadataTestControl { | |
| server: Mutex::new(McpServerContribution::Set { | |
| name: CODEX_APPS_MCP_SERVER_NAME.to_string(), | |
| config: Box::new(codex_apps_mcp_server_config( | |
| &apps_server.chatgpt_base_url, | |
| /*apps_mcp_product_sku*/ None, | |
| /*originator*/ None, | |
| )), | |
| }), | |
| gate: Mutex::new(None), | |
| })); | |
| builder = builder.with_extensions(Arc::new(extensions.build())); | |
| } | |
| let arguments = serde_json::json!({ "search": "launch plan" }); | |
| let (test, follow_up) = if direct { | |
| let test = builder.build(&server).await?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| responses::ev_function_call_with_namespace( | |
| "call-1", | |
| "mcp__codex_apps__messagesearch", | |
| RESULT_METADATA_TOOL, | |
| &arguments.to_string(), | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let follow_up = responses::mount_sse_once(&server, sse(vec![ev_completed("resp-2")])).await; | |
| test.submit_turn("Search a connected app").await?; | |
| (test, follow_up) | |
| } else { | |
| let code = format!( | |
| "const tool = ALL_TOOLS.find(({{ name }}) => name.endsWith(\"{RESULT_METADATA_TOOL}\")); \ | |
| const result = await tools[tool.name]({arguments}); \ | |
| text(JSON.stringify({{ isError: Boolean(result.isError), hasMeta: Object.hasOwn(result, \"_meta\") }}));" | |
| ); | |
| run_code_mode_turn_with_builder(&server, "Search a connected app", &code, builder).await? | |
| }; | |
| let request = follow_up.single_request(); | |
| assert_eq!(recorded_apps_tool_calls(&server).await.len(), 1); | |
| let output = if direct { | |
| let output = request.function_call_output("call-1"); | |
| assert!( | |
| output["output"] | |
| .to_string() | |
| .contains(RESULT_METADATA_PRIVATE_RESULT) | |
| ); | |
| assert!(!output["output"].to_string().contains("provider_state")); | |
| output | |
| } else { | |
| assert!( | |
| !request | |
| .body_json() | |
| .to_string() | |
| .contains(RESULT_METADATA_PRIVATE_RESULT) | |
| ); | |
| let (body, success) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert_ne!(success, Some(false), "Code Mode failed: {body}"); | |
| assert_eq!( | |
| serde_json::from_str::<Value>(&body)?, | |
| serde_json::json!({ "isError": is_error, "hasMeta": false }), | |
| ); | |
| request.custom_tool_call_output("call-1") | |
| }; | |
| assert_eq!( | |
| result_metadata_fixture_calls(&request.input()).count(), | |
| usize::from(metadata_enabled), | |
| ); | |
| let captured = codex_core::test_support::history_with_tool_call_metadata(&test.codex).await; | |
| if direct { | |
| let result = captured | |
| .iter() | |
| .find_map(|item| match item { | |
| codex_protocol::models::ResponseItem::FunctionCallOutput { | |
| call_id, | |
| output, | |
| .. | |
| } if call_id.as_deref() == Some("call-1") => Some(output), | |
| _ => None, | |
| }) | |
| .expect("captured direct output"); | |
| assert_eq!(result.success, Some(!is_error)); | |
| } | |
| if metadata_enabled { | |
| // The custom inference endpoint gets no raw metadata; inspect capture independently. | |
| assert_result_metadata_call(&output, &arguments, /*expected_metadata*/ None); | |
| let captured = serde_json::to_value(captured)?; | |
| let captured_output = captured | |
| .as_array() | |
| .unwrap() | |
| .iter() | |
| .find(|item| item["type"] == output["type"] && item["call_id"] == "call-1") | |
| .expect("captured tool output"); | |
| // The public build never captures result metadata, even with employee test credentials. | |
| let expected_metadata = None; | |
| assert_result_metadata_call(captured_output, &arguments, expected_metadata); | |
| assert_eq!( | |
| output["internal_chat_message_metadata_passthrough"]["tool_calls_complete"], | |
| true | |
| ); | |
| } else { | |
| assert!( | |
| output["internal_chat_message_metadata_passthrough"] | |
| .get("tool_calls_complete") | |
| .is_none() | |
| ); | |
| } | |
| Ok(()) | |
| } | |
| async fn code_mode_result_metadata_follows_runtime_recording_enablement() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let result_metadata = serde_json::json!({ "provider": { "id": "refresh" } }); | |
| let apps_server = mount_result_metadata_app( | |
| &server, | |
| Some(result_metadata.clone()), | |
| /*is_error*/ false, | |
| ) | |
| .await?; | |
| let mut builder = | |
| result_metadata_apps_builder(apps_server.chatgpt_base_url, "employee@openai.com") | |
| .with_config(|config| { | |
| config | |
| .features | |
| .disable(Feature::ExecutedToolCallMetadata) | |
| .unwrap(); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| let arguments = serde_json::json!({ "search": "launch plan" }); | |
| let code = format!( | |
| "const tool = ALL_TOOLS.find(({{ name }}) => name.endsWith(\"{RESULT_METADATA_TOOL}\")); \ | |
| const result = await tools[tool.name]({arguments}); \ | |
| text(JSON.stringify({{ isError: Boolean(result.isError), hasMeta: Object.hasOwn(result, \"_meta\") }}));" | |
| ); | |
| for (call_id, enabled) in [("call-off", false), ("call-on", true)] { | |
| if enabled { | |
| let mut config = test.config.clone(); | |
| config | |
| .features | |
| .enable(Feature::ExecutedToolCallMetadata) | |
| .unwrap(); | |
| test.codex.refresh_runtime_config(config).await; | |
| // Runtime recording changes without updating the session's execution features. | |
| assert!( | |
| !test | |
| .codex | |
| .config() | |
| .await | |
| .features | |
| .enabled(Feature::ExecutedToolCallMetadata) | |
| ); | |
| } | |
| let initial = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_custom_tool_call(call_id, "exec", &code), | |
| ev_completed(call_id), | |
| ]), | |
| ) | |
| .await; | |
| let follow_up = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message(&format!("{call_id}-message"), "done"), | |
| ev_completed(&format!("{call_id}-done")), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("Search a connected app").await?; | |
| initial.single_request(); | |
| let request = follow_up.single_request(); | |
| let (body, success) = custom_tool_output_body_and_success(&request, call_id); | |
| assert_ne!(success, Some(false), "Code Mode failed: {body}"); | |
| assert_eq!( | |
| serde_json::from_str::<Value>(&body)?, | |
| serde_json::json!({ "isError": false, "hasMeta": false }), | |
| ); | |
| assert_eq!( | |
| result_metadata_fixture_calls(&request.input()).count(), | |
| usize::from(enabled) | |
| ); | |
| let output = request.custom_tool_call_output(call_id); | |
| assert!( | |
| output["internal_chat_message_metadata_passthrough"] | |
| .get("tool_calls_complete") | |
| .is_none() | |
| ); | |
| if enabled { | |
| assert_result_metadata_call(&output, &arguments, /*expected_metadata*/ None); | |
| } | |
| } | |
| assert_eq!(recorded_apps_tool_calls(&server).await.len(), 2); | |
| let captured = codex_core::test_support::history_with_tool_call_metadata(&test.codex).await; | |
| let captured = serde_json::to_value(captured)?; | |
| let captured_output = captured | |
| .as_array() | |
| .unwrap() | |
| .iter() | |
| .find(|item| item["type"] == "custom_tool_call_output" && item["call_id"] == "call-on") | |
| .expect("captured exec output after runtime enablement"); | |
| let expected_metadata = None; | |
| assert_result_metadata_call(captured_output, &arguments, expected_metadata); | |
| Ok(()) | |
| } | |
| async fn code_mode_result_metadata_keeps_prepared_call_binding_across_runtime_refresh() -> Result<()> | |
| { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let original_metadata = | |
| serde_json::json!({ "provider": { "origin": "original", "items": [] } }); | |
| let apps_server = mount_result_metadata_app( | |
| &server, | |
| Some(original_metadata.clone()), | |
| /*is_error*/ false, | |
| ) | |
| .await?; | |
| let refreshed_server = responses::start_mock_server().await; | |
| let refreshed_apps = mount_result_metadata_app( | |
| &refreshed_server, | |
| Some(serde_json::json!({ "provider": { "origin": "refreshed" } })), | |
| /*is_error*/ false, | |
| ) | |
| .await?; | |
| let (reached_tx, reached_rx) = oneshot::channel(); | |
| let (release_tx, release_rx) = oneshot::channel(); | |
| let control = Arc::new(ResultMetadataTestControl { | |
| server: Mutex::new(McpServerContribution::HostedApps { | |
| config: Box::new(codex_apps_mcp_server_config( | |
| &apps_server.chatgpt_base_url, | |
| /*apps_mcp_product_sku*/ None, | |
| /*originator*/ None, | |
| )), | |
| }), | |
| gate: Mutex::new(Some((reached_tx, release_rx))), | |
| }); | |
| let mut extensions = ExtensionRegistryBuilder::<Config>::new(); | |
| extensions.mcp_server_contributor(control.clone()); | |
| extensions.tool_lifecycle_contributor(control.clone()); | |
| let builder = result_metadata_apps_builder(apps_server.chatgpt_base_url, "employee@openai.com") | |
| .with_extensions(Arc::new(extensions.build())); | |
| let arguments = serde_json::json!({ | |
| "query": "launch plan", | |
| "response_format": "detailed", | |
| "content_types": "messages", | |
| }); | |
| let code = format!( | |
| "const tool = ALL_TOOLS.find(({{ name }}) => name.endsWith(\"{RESULT_METADATA_TOOL}\")); \ | |
| const pending = tools[tool.name]({arguments}); yield_control(); await pending; \ | |
| await tools[tool.name]({arguments}); text(\"done\");" | |
| ); | |
| let (test, follow_up) = | |
| run_code_mode_turn_with_builder(&server, "Search a connected app", &code, builder).await?; | |
| let first_items = custom_tool_output_items(&follow_up.single_request(), "call-1"); | |
| assert!( | |
| text_item(&first_items, /*index*/ 0).starts_with("Script running with cell ID "), | |
| "expected the held call to yield: {first_items:?}", | |
| ); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| tokio::time::timeout(Duration::from_secs(10), reached_rx).await??; | |
| // Yield may precede call admission. Drain inventory after the lifecycle gate is reached; | |
| // the short wait is only a poll budget while the gate keeps acceptance blocked. | |
| let held_wait = responses::mount_function_call_agent_response( | |
| &server, | |
| "call-2", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id, | |
| "yield_time_ms": 1, | |
| }))?, | |
| "wait", | |
| ) | |
| .await; | |
| test.submit_turn("Read the pending call inventory").await?; | |
| let held_request = held_wait.completion.single_request(); | |
| let held_items = function_tool_output_items(&held_request, "call-2"); | |
| assert_eq!( | |
| extract_running_cell_id(text_item(&held_items, /*index*/ 0)), | |
| cell_id | |
| ); | |
| let held_input = held_request.input(); | |
| let emitted_calls = result_metadata_fixture_calls(&held_input).collect::<Vec<_>>(); | |
| assert_eq!( | |
| emitted_calls.len(), | |
| 1, | |
| "held wait must not duplicate inventory" | |
| ); | |
| let original_output = emitted_calls[0]; | |
| assert_result_metadata_call(original_output, &arguments, /*expected_metadata*/ None); | |
| assert_ne!( | |
| original_output["internal_chat_message_metadata_passthrough"]["tool_calls_complete"], | |
| true | |
| ); | |
| // The next call uses an extension-owned binding, but the held call keeps its host proof. | |
| *control.server.lock().unwrap() = McpServerContribution::Set { | |
| name: CODEX_APPS_MCP_SERVER_NAME.to_string(), | |
| config: Box::new(codex_apps_mcp_server_config( | |
| &refreshed_apps.chatgpt_base_url, | |
| /*apps_mcp_product_sku*/ None, | |
| /*originator*/ None, | |
| )), | |
| }; | |
| test.codex.refresh_runtime_config(test.config.clone()).await; | |
| release_tx.send(()).unwrap(); | |
| let wait = responses::mount_function_call_agent_response( | |
| &server, | |
| "call-3", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id, | |
| "yield_time_ms": 10_000, | |
| }))?, | |
| "wait", | |
| ) | |
| .await; | |
| test.submit_turn("Wait for the accepted app result").await?; | |
| let request = wait.completion.single_request(); | |
| assert_eq!(recorded_apps_tool_calls(&server).await.len(), 1); | |
| assert_eq!(recorded_apps_tool_calls(&refreshed_server).await.len(), 1); | |
| assert!( | |
| !request | |
| .body_json() | |
| .to_string() | |
| .contains(RESULT_METADATA_PRIVATE_RESULT) | |
| ); | |
| let input = request.input(); | |
| assert_eq!( | |
| result_metadata_fixture_calls(&input).count(), | |
| 2, | |
| "late results must not duplicate the call" | |
| ); | |
| let captured = codex_core::test_support::history_with_tool_call_metadata(&test.codex).await; | |
| let captured = serde_json::to_value(captured)?; | |
| // A's accepted result must update the output that first reported it, not the final wait. | |
| let expected_metadata = None; | |
| for (call_id, call_type, expected_metadata) in [ | |
| ( | |
| original_output["call_id"].as_str().unwrap(), | |
| original_output["type"].as_str().unwrap(), | |
| expected_metadata, | |
| ), | |
| ("call-3", "function_call_output", None), | |
| ] { | |
| let output = request.call_output(call_id, call_type); | |
| assert_result_metadata_call(&output, &arguments, /*expected_metadata*/ None); | |
| let captured_output = captured | |
| .as_array() | |
| .unwrap() | |
| .iter() | |
| .find(|item| item["type"] == call_type && item["call_id"] == call_id) | |
| .expect("captured tool output"); | |
| assert_result_metadata_call(captured_output, &arguments, expected_metadata); | |
| } | |
| let terminal_output = request.function_call_output("call-3"); | |
| assert_eq!( | |
| terminal_output["internal_chat_message_metadata_passthrough"]["tool_calls_complete"], | |
| true | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_resumed_wait_does_not_certify_a_reused_runtime_cell() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let configure = |config: &mut Config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| let _ = config.features.enable(Feature::CodeModeHost); | |
| let _ = config.features.enable(Feature::ExecutedToolCallMetadata); | |
| config.code_mode.disable_in_process_fallback = true; | |
| }; | |
| let mut builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_history_mode(ThreadHistoryMode::Legacy) | |
| .with_config(configure); | |
| let initial = builder.build(&server).await?; | |
| let seed = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_custom_tool_call( | |
| "call-a", | |
| "exec", | |
| r#"text("A"); yield_control(); await new Promise(() => {});"#, | |
| ), | |
| ev_completed("resp-a"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-a", "A is running"), | |
| ev_completed("resp-a-done"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| initial.submit_turn("Start A").await?; | |
| let old_output = seed.requests()[1].custom_tool_call_output("call-a"); | |
| let old_items = custom_tool_output_items(&seed.requests()[1], "call-a"); | |
| let old_cell_id = extract_running_cell_id(text_item(&old_items, /*index*/ 0)); | |
| assert_eq!(old_cell_id, "1"); | |
| assert_eq!(text_item(&old_items, /*index*/ 1), "A"); | |
| // Restart loads the old running-cell output, but starts a new host whose IDs reset. | |
| let resumed = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_config(configure) | |
| .restart(&server, &initial) | |
| .await?; | |
| assert_eq!( | |
| resumed.session_configured.thread_id, | |
| initial.session_configured.thread_id | |
| ); | |
| let gate = resumed.workspace_path("release-cell-b"); | |
| let quoted_gate = shlex::try_join([gate.to_string_lossy().as_ref()])?; | |
| let arguments = serde_json::json!({ | |
| "cmd": format!( | |
| "attempt=0; while [ ! -f {quoted_gate} ] && [ \"$attempt\" -lt 1000 ]; \ | |
| do sleep 0.01; attempt=$((attempt + 1)); done; [ -f {quoted_gate} ] && printf B" | |
| ), | |
| "login": false, | |
| "yield_time_ms": 30_000, | |
| "max_output_tokens": 1000, | |
| }); | |
| let code = format!("yield_control(); text((await tools.exec_command({arguments})).output);"); | |
| let continued = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_custom_tool_call("call-b", "exec", &code), | |
| ev_completed("resp-b"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-b", "B is running"), | |
| ev_completed("resp-b-done"), | |
| ]), | |
| sse(vec![ | |
| responses::ev_function_call( | |
| "call-stale-wait", | |
| "wait", | |
| &serde_json::json!({"cell_id": old_cell_id, "yield_time_ms": 10_000}) | |
| .to_string(), | |
| ), | |
| ev_completed("resp-wait"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-wait", "done"), | |
| ev_completed("resp-wait-done"), | |
| ]), | |
| sse(vec![ | |
| ev_custom_tool_call( | |
| "call-fresh", | |
| "exec", | |
| r#"await tools.test_sync_tool({}); text("fresh");"#, | |
| ), | |
| ev_completed("resp-fresh"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-fresh", "done"), | |
| ev_completed("resp-fresh-done"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| resumed.submit_turn("Start B").await?; | |
| let requests = continued.requests(); | |
| assert_eq!( | |
| requests[0].custom_tool_call_output("call-a")["output"], | |
| old_output["output"] | |
| ); | |
| let new_items = custom_tool_output_items(&requests[1], "call-b"); | |
| assert_eq!( | |
| extract_running_cell_id(text_item(&new_items, /*index*/ 0)), | |
| old_cell_id | |
| ); | |
| fs::write(&gate, "ready")?; | |
| resumed.submit_turn("Wait using A's old cell ID").await?; | |
| let requests = continued.requests(); | |
| assert_eq!(requests.len(), 4); | |
| let items = function_tool_output_items(&requests[3], "call-stale-wait"); | |
| assert_eq!(items.len(), 2); | |
| assert!(text_item(&items, /*index*/ 0).starts_with("Script completed\n")); | |
| assert_eq!(text_item(&items, /*index*/ 1), "B"); | |
| let wait = requests[3].function_call_output("call-stale-wait"); | |
| let metadata = &wait["internal_chat_message_metadata_passthrough"]; | |
| assert!(metadata.get("tool_calls_complete").is_none()); | |
| let mut recorded = Vec::new(); | |
| for output in [requests[3].custom_tool_call_output("call-b"), wait] { | |
| let metadata = &output["internal_chat_message_metadata_passthrough"]; | |
| if let Some(calls) = metadata | |
| .get("executed_tool_calls") | |
| .and_then(Value::as_array) | |
| && !calls.is_empty() | |
| { | |
| assert_eq!(metadata["cell_id"], "call-b"); | |
| recorded.extend_from_slice(calls); | |
| } | |
| } | |
| assert_eq!( | |
| recorded, | |
| vec![serde_json::json!({"name": "exec_command", "arguments": arguments})] | |
| ); | |
| resumed.submit_turn("Run a fresh immediate exec").await?; | |
| let requests = continued.requests(); | |
| assert_eq!(requests.len(), 6); | |
| assert_eq!( | |
| custom_tool_output_last_non_empty_text(&requests[5], "call-fresh").as_deref(), | |
| Some("fresh") | |
| ); | |
| let fresh = requests[5].custom_tool_call_output("call-fresh"); | |
| let metadata = &fresh["internal_chat_message_metadata_passthrough"]; | |
| assert_eq!( | |
| metadata["executed_tool_calls"], | |
| serde_json::json!([{"name": "test_sync_tool", "arguments": {}}]) | |
| ); | |
| assert_eq!(metadata["cell_id"], "call-fresh"); | |
| assert_eq!(metadata["tool_calls_complete"], true); | |
| resumed.codex.shutdown_and_wait().await?; | |
| Ok(()) | |
| } | |
| async fn code_mode_complete_call_survives_unrelated_truncation() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (test, _) = run_code_mode_turn_with_config( | |
| &server, | |
| "Record an oversized call", | |
| r#"await tools.test_sync_tool({ barrier: { id: "x".repeat(8192), participants: 1 } }); text("done");"#, | |
| |config| { | |
| let _ = config.features.disable(Feature::RemoteCompactionV2); | |
| }, | |
| ) | |
| .await?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| ev_custom_tool_call( | |
| "call-2", | |
| "exec", | |
| r#"await tools.test_sync_tool({ wait_for_git_enrichment: false }); text("done");"#, | |
| ), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let follow_up = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "done"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("Record a complete call").await?; | |
| test.codex.shutdown_and_wait().await?; | |
| let request = follow_up.single_request(); | |
| let overflow = | |
| &request.custom_tool_call_output("call-1")["internal_chat_message_metadata_passthrough"]; | |
| assert!( | |
| overflow["executed_tool_calls"] | |
| .as_array() | |
| .is_some_and(|calls| calls.iter().any(|call| call["arguments"] | |
| .get("_codex_executed_tool_call_truncated") | |
| .is_some())) | |
| ); | |
| assert!(overflow.get("tool_calls_complete").is_none()); | |
| let complete = | |
| &request.custom_tool_call_output("call-2")["internal_chat_message_metadata_passthrough"]; | |
| assert_eq!( | |
| complete["executed_tool_calls"], | |
| serde_json::json!([{ "name": "test_sync_tool", "arguments": { "wait_for_git_enrichment": false } }]), | |
| ); | |
| assert_eq!(complete["cell_id"], "call-2"); | |
| assert_eq!(complete["tool_calls_complete"], true); | |
| Ok(()) | |
| } | |
| async fn code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let apps_server = AppsTestServer::mount_searchable(&server).await?; | |
| let originating_item_id = "ctc_code_mode_origin"; | |
| let mut exec_call = ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| r#" | |
| const tool = ALL_TOOLS.find( | |
| ({ name }) => name === "mcp__codex_apps__calendar_timezone_option_99" | |
| ); | |
| if (!tool) { | |
| text(JSON.stringify({ found: false })); | |
| } else { | |
| const result = await tools[tool.name]({ timezone: "UTC" }); | |
| text(JSON.stringify({ | |
| found: true, | |
| isError: Boolean(result.isError), | |
| text: result.content?.[0]?.text ?? "", | |
| })); | |
| } | |
| "#, | |
| ); | |
| exec_call["item"]["id"] = serde_json::json!(originating_item_id); | |
| let resp_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| exec_call, | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let follow_up_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| let apps_base_url = apps_server.chatgpt_base_url.clone(); | |
| let mut builder = test_codex() | |
| .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) | |
| .with_config(move |config| { | |
| config | |
| .features | |
| .enable(Feature::Apps) | |
| .expect("test config should allow feature update"); | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("test config should allow feature update"); | |
| config | |
| .features | |
| .enable(Feature::CodeModeOnly) | |
| .expect("test config should allow feature update"); | |
| let mut model_catalog = | |
| bundled_models_response().expect("bundled models.json should parse"); | |
| let model = model_catalog | |
| .models | |
| .iter_mut() | |
| .find(|model| model.slug == "gpt-5.5") | |
| .expect("gpt-5.5 exists in bundled models.json"); | |
| config.chatgpt_base_url = apps_base_url; | |
| config.model = Some("gpt-5.5".to_string()); | |
| model.supports_search_tool = true; | |
| config.model_catalog = Some(model_catalog); | |
| }); | |
| let test = builder.build(&server).await?; | |
| test.submit_turn("inspect tools in code mode only").await?; | |
| let first_body = resp_mock.single_request().body_json(); | |
| assert_eq!( | |
| tool_names(&first_body), | |
| vec![ | |
| "exec".to_string(), | |
| "wait".to_string(), | |
| "request_user_input".to_string(), | |
| "web_search".to_string() | |
| ] | |
| ); | |
| let exec_description = first_body | |
| .get("tools") | |
| .and_then(Value::as_array) | |
| .and_then(|tools| { | |
| tools.iter().find_map(|tool| { | |
| if tool | |
| .get("name") | |
| .or_else(|| tool.get("type")) | |
| .and_then(Value::as_str) | |
| == Some("exec") | |
| { | |
| tool.get("description").and_then(Value::as_str) | |
| } else { | |
| None | |
| } | |
| }) | |
| }) | |
| .expect("exec description should be present"); | |
| assert!(exec_description.contains("filter `ALL_TOOLS` by `name` and `description`")); | |
| assert!(exec_description.contains("Shared MCP Types:")); | |
| assert!(!exec_description.contains("calendar_timezone_option_99")); | |
| let request = follow_up_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "code_mode_only deferred app tool call failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str(&output)?; | |
| assert_eq!( | |
| parsed, | |
| serde_json::json!({ | |
| "found": true, | |
| "isError": false, | |
| "text": "called calendar_timezone_option_99 for at with ", | |
| }) | |
| ); | |
| let apps_tool_calls = recorded_apps_tool_calls(&server).await; | |
| assert!( | |
| apps_tool_calls.iter().any(|call| { | |
| call.pointer("/params/_meta/itemId") == Some(&serde_json::json!(originating_item_id)) | |
| && call.pointer("/params/_meta/sessionId") | |
| == Some(&serde_json::json!(test.session_configured.session_id)) | |
| && call.pointer("/params/_meta/windowId") | |
| == Some(&first_body["client_metadata"]["x-codex-window-id"]) | |
| }), | |
| "the nested MCP call should inherit its code cell's originating Responses item" | |
| ); | |
| Ok(()) | |
| } | |
| async fn app_only_tools_are_not_visible_or_runnable_by_code_mode_model() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let apps_server = | |
| AppsTestServer::mount_with_app_only_tool(&server, AppsTestToolLoading::Searchable).await?; | |
| let code = format!( | |
| r#" | |
| const visibleTool = ALL_TOOLS.find(({{ name }}) => name === {visible_tool_name:?}); | |
| const tool = ALL_TOOLS.find(({{ name }}) => name === {tool_name:?}); | |
| let error = null; | |
| try {{ | |
| await tools[{tool_name:?}]({{}}); | |
| }} catch (caught) {{ | |
| error = String(caught); | |
| }} | |
| text(JSON.stringify({{ | |
| visibleListed: visibleTool !== undefined, | |
| listed: tool !== undefined, | |
| callable: typeof tools[{tool_name:?}] === "function", | |
| error, | |
| }})); | |
| "#, | |
| visible_tool_name = "mcp__codex_apps__calendar_timezone_option_99", | |
| tool_name = DIRECT_CALENDAR_APP_ONLY_TOOL, | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| let mut builder = search_capable_apps_builder(apps_server.chatgpt_base_url.clone()) | |
| .with_config(|config| { | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("test config should allow feature update"); | |
| config | |
| .features | |
| .enable(Feature::CodeModeOnly) | |
| .expect("test config should allow feature update"); | |
| }); | |
| let test = builder.build(&server).await?; | |
| test.submit_turn("try to call the app-only calendar tool through exec") | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "code mode visibility check should complete successfully: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str(&output)?; | |
| assert_eq!(parsed["visibleListed"], true); | |
| assert_eq!(parsed["listed"], false); | |
| assert_eq!(parsed["callable"], false); | |
| assert!( | |
| parsed["error"] | |
| .as_str() | |
| .is_some_and(|error| error.contains("is not a function")), | |
| "app-only code mode call should fail before MCP dispatch: {parsed:?}" | |
| ); | |
| assert!( | |
| recorded_apps_tool_calls(&server).await.is_empty(), | |
| "app-only code mode call should not reach the MCP server" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_only_can_call_nested_tools() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| r#" | |
| const output = await tools.exec_command({ cmd: "printf code_mode_only_nested_tool_marker" }); | |
| text(output.output); | |
| "#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let follow_up_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| let mut builder = test_codex().with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeModeOnly); | |
| }); | |
| let test = builder.build(&server).await?; | |
| test.submit_turn("use exec to run nested tool in code mode only") | |
| .await?; | |
| let request = follow_up_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "code_mode_only nested tool call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!(output, "code_mode_only_nested_tool_marker"); | |
| assert!( | |
| request | |
| .custom_tool_call_output("call-1") | |
| .pointer("/internal_chat_message_metadata_passthrough/executed_tool_calls") | |
| .is_none(), | |
| "nested tool calls must not be recorded while executed-tool-call metadata is disabled", | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_does_not_expose_update_plan_by_default() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "inspect the available tools", | |
| r#" | |
| text(JSON.stringify({ | |
| callable: typeof tools.update_plan === "function", | |
| listed: ALL_TOOLS.some(({ name }) => name === "update_plan"), | |
| })); | |
| "#, | |
| ) | |
| .await?; | |
| let (output, _) = custom_tool_output_body_and_success(&second_mock.single_request(), "call-1"); | |
| assert_eq!( | |
| serde_json::from_str::<Value>(&output)?, | |
| serde_json::json!({ "callable": false, "listed": false }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_update_plan_nested_tool_result_is_empty_object() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_config( | |
| &server, | |
| "use exec to run update_plan", | |
| r#" | |
| const result = await tools.update_plan({ | |
| plan: [{ step: "Run update_plan from code mode", status: "in_progress" }], | |
| }); | |
| text(JSON.stringify(result)); | |
| "#, | |
| |config| config.update_plan_enabled = true, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec update_plan call failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str(&output)?; | |
| assert_eq!(parsed, serde_json::json!({})); | |
| Ok(()) | |
| } | |
| async fn code_mode_get_context_remaining_returns_structured_result() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_config( | |
| &server, | |
| "use exec to get remaining context", | |
| r#" | |
| const result = await tools.get_context_remaining({}); | |
| text(JSON.stringify(result)); | |
| "#, | |
| |config| { | |
| config.model_context_window = Some(10_000); | |
| config | |
| .features | |
| .enable(Feature::TokenBudget) | |
| .expect("test config should allow token budget"); | |
| }, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec get_context_remaining call failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str(&output)?; | |
| assert_eq!( | |
| parsed, | |
| serde_json::json!({ | |
| "tokens_left": 9000, | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_current_time_returns_structured_result(argument: &str) -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_config( | |
| &server, | |
| "use exec to get the current time", | |
| &format!( | |
| r#" | |
| const result = await tools.clock__curr_time({argument}); | |
| text(JSON.stringify(result)); | |
| "#, | |
| ), | |
| |config| { | |
| config | |
| .features | |
| .enable(Feature::CurrentTimeReminder) | |
| .expect("test config should allow current-time reminders"); | |
| config.current_time_reminder = Some(CurrentTimeReminderConfig { | |
| reminder_interval_seconds: 3_000, | |
| clock_source: CurrentTimeSource::System, | |
| ..CurrentTimeReminderConfig::default() | |
| }); | |
| }, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec clock.curr_time call failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str(&output)?; | |
| let current_time = parsed | |
| .get("current_time") | |
| .and_then(Value::as_str) | |
| .expect("clock.curr_time should return current_time"); | |
| assert_regex_match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} UTC$", current_time); | |
| Ok(()) | |
| } | |
| async fn code_mode_nested_tool_calls_can_run_in_parallel() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| let _ = config.features.enable(Feature::ExecutedToolCallMetadata); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let warmup_code = r#" | |
| const args = { | |
| sleep_after_ms: 10, | |
| barrier: { | |
| id: "code-mode-parallel-tools-warmup", | |
| participants: 2, | |
| timeout_ms: 1_000, | |
| }, | |
| }; | |
| await Promise.all([ | |
| tools.test_sync_tool(args), | |
| tools.test_sync_tool(args), | |
| ]); | |
| "#; | |
| let code = r#" | |
| const args = { | |
| sleep_after_ms: 300, | |
| barrier: { | |
| id: "code-mode-parallel-tools", | |
| participants: 2, | |
| timeout_ms: 1_000, | |
| }, | |
| }; | |
| const results = await Promise.all([ | |
| tools.test_sync_tool(args), | |
| tools.test_sync_tool(args), | |
| ]); | |
| text(JSON.stringify(results)); | |
| "#; | |
| let response_mock = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_response_created("resp-warm-1"), | |
| ev_custom_tool_call("call-warm-1", "exec", warmup_code), | |
| ev_completed("resp-warm-1"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-warm-1", "warmup done"), | |
| ev_completed("resp-warm-2"), | |
| ]), | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", code), | |
| ev_completed("resp-1"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_turn("warm up nested tools in parallel").await?; | |
| let start = Instant::now(); | |
| test.submit_turn("run nested tools in parallel").await?; | |
| let duration = start.elapsed(); | |
| assert!( | |
| duration < Duration::from_millis(1_600), | |
| "expected nested tools to finish in parallel, got {duration:?}", | |
| ); | |
| let req = response_mock | |
| .last_request() | |
| .expect("parallel code mode run should send a completion request"); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| assert_eq!(items.len(), 2); | |
| assert_eq!(text_item(&items, /*index*/ 1), "[\"ok\",\"ok\"]"); | |
| Ok(()) | |
| } | |
| async fn code_mode_write_stdin_calls_run_in_parallel_across_sessions() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "write to independent terminal sessions in parallel", | |
| r#" | |
| const sessions = await Promise.all([ | |
| tools.exec_command({ cmd: "bash --noprofile --norc", tty: true, yield_time_ms: 250 }), | |
| tools.exec_command({ cmd: "bash --noprofile --norc", tty: true, yield_time_ms: 250 }), | |
| ]); | |
| const results = await Promise.all([ | |
| tools.write_stdin({ | |
| session_id: sessions[0].session_id, | |
| chars: ": > .code-mode-a; while [ ! -e .code-mode-b ]; do sleep 0.01; done; printf 'code-alpha-%s\\n' ready; exit\n", | |
| yield_time_ms: 5000, | |
| }), | |
| tools.write_stdin({ | |
| session_id: sessions[1].session_id, | |
| chars: ": > .code-mode-b; while [ ! -e .code-mode-a ]; do sleep 0.01; done; printf 'code-beta-%s\\n' ready; exit\n", | |
| yield_time_ms: 5000, | |
| }), | |
| ]); | |
| text(JSON.stringify([results[0].output.includes("code-alpha-ready"), results[1].output.includes("code-beta-ready")])); | |
| "#, | |
| ) | |
| .await?; | |
| let items = custom_tool_output_items(&second_mock.single_request(), "call-1"); | |
| let result: Value = serde_json::from_str(text_item(&items, /*index*/ 1))?; | |
| assert_eq!(result, serde_json::json!([true, true])); | |
| Ok(()) | |
| } | |
| // This model uses token-based tool-output truncation, giving the downstream | |
| // history assertions a stable `…N tokens truncated…` marker. | |
| const TOKEN_POLICY_TEST_MODEL: &str = "gpt-5.5"; | |
| // A nested `exec_command` limit applies to `result.output` inside JavaScript. | |
| // The outer code-mode and history budgets apply after the script calls `text`. | |
| async fn code_mode_exec_nested_limit_formats_truncated_result_with_warning() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "use exec_command from code mode", | |
| r#" | |
| const result = await tools.exec_command({ | |
| cmd: "printf '0123456789012345678901234567890123456789'", | |
| max_output_tokens: 5 | |
| }); | |
| text(result.output); | |
| "#, | |
| ) | |
| .await?; | |
| assert_eq!( | |
| text_item( | |
| &custom_tool_output_items(&second_mock.single_request(), "call-1"), | |
| /*index*/ 1 | |
| ), | |
| "Warning: truncated output (original token count: 10)\nTotal output lines: 1\n\n0123456789…5 tokens truncated…0123456789" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exec_nested_limit_preserves_result_variable_before_default_history_truncation() | |
| -> Result<()> { | |
| // TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode. | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "only part of nested exec_command stdout reaches the code-mode result" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_model_and_config( | |
| &server, | |
| "use exec_command from code mode", | |
| r#"// @exec: {"max_output_tokens": 20000} | |
| const result = await tools.exec_command({ | |
| cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"", | |
| max_output_tokens: 20000 | |
| }); | |
| const resultVariableWasTruncated = result.output.length !== 50000; | |
| text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`); | |
| "#, | |
| TOKEN_POLICY_TEST_MODEL, | |
| |_| {}, | |
| ) | |
| .await?; | |
| let items = custom_tool_output_items(&second_mock.single_request(), "call-1"); | |
| let output = text_item(&items, /*index*/ 1); | |
| assert_regex_match( | |
| r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$", | |
| output, | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exec_nested_limit_truncates_result_variable_when_exceeded() -> Result<()> { | |
| // TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode. | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "only part of nested exec_command stdout reaches the code-mode result" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_model_and_config( | |
| &server, | |
| "use exec_command from code mode", | |
| r#"// @exec: {"max_output_tokens": 25000} | |
| const result = await tools.exec_command({ | |
| cmd: "python3 -c \"import sys; sys.stdout.write('A' * 90000)\"", | |
| max_output_tokens: 20000 | |
| }); | |
| const resultVariableWasTruncated = result.output.includes("…2500 tokens truncated…"); | |
| text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`); | |
| "#, | |
| TOKEN_POLICY_TEST_MODEL, | |
| |_| {}, | |
| ) | |
| .await?; | |
| let items = custom_tool_output_items(&second_mock.single_request(), "call-1"); | |
| let output = text_item(&items, /*index*/ 1); | |
| // The nested 20,000-token budget leaves about 80,000 characters. This | |
| // ceiling independently proves that history applied its smaller cap. | |
| assert!( | |
| output.len() < 60_000, | |
| "expected history to truncate the emitted value, got {} bytes", | |
| output.len() | |
| ); | |
| // The boolean describes the nested result; the marker below comes from | |
| // history truncating the value emitted with `text` afterward. | |
| assert_regex_match( | |
| r"(?s)^Variable truncated: True\. Variable: .*…\d+ tokens truncated…A+$", | |
| output, | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exec_nested_limit_preserves_result_variable_before_configured_history_truncation() | |
| -> Result<()> { | |
| // TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode. | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "only part of nested exec_command stdout reaches the code-mode result" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_model_and_config( | |
| &server, | |
| "use exec_command from code mode", | |
| r#"// @exec: {"max_output_tokens": 20000} | |
| const result = await tools.exec_command({ | |
| cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"", | |
| max_output_tokens: 20000 | |
| }); | |
| const resultVariableWasTruncated = result.output.length !== 50000; | |
| text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`); | |
| "#, | |
| TOKEN_POLICY_TEST_MODEL, | |
| |config| { | |
| config.tool_output_token_limit = Some(50); | |
| }, | |
| ) | |
| .await?; | |
| let items = custom_tool_output_items(&second_mock.single_request(), "call-1"); | |
| let output = text_item(&items, /*index*/ 1); | |
| // The 50-token override must shrink this 50,000-character value far below | |
| // what the default 10,000-token history cap would retain. | |
| assert!( | |
| output.len() < 1_000, | |
| "expected configured history cap to truncate the emitted value, got {} bytes", | |
| output.len() | |
| ); | |
| assert_regex_match( | |
| r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$", | |
| output, | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exec_without_nested_limit_preserves_result_variable_before_default_history_truncation() | |
| -> Result<()> { | |
| // TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode. | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "only part of nested exec_command stdout reaches the code-mode result" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_model_and_config( | |
| &server, | |
| "use exec_command from code mode", | |
| r#"// @exec: {"max_output_tokens": 20000} | |
| const result = await tools.exec_command({ | |
| cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"" | |
| }); | |
| const resultVariableWasTruncated = result.output.length !== 50000; | |
| text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`); | |
| "#, | |
| TOKEN_POLICY_TEST_MODEL, | |
| |_| {}, | |
| ) | |
| .await?; | |
| let items = custom_tool_output_items(&second_mock.single_request(), "call-1"); | |
| let output = text_item(&items, /*index*/ 1); | |
| assert_regex_match( | |
| r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$", | |
| output, | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exec_without_nested_limit_preserves_result_variable_before_configured_history_truncation() | |
| -> Result<()> { | |
| // TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode. | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "only part of nested exec_command stdout reaches the code-mode result" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_model_and_config( | |
| &server, | |
| "use exec_command from code mode", | |
| r#"// @exec: {"max_output_tokens": 20000} | |
| const result = await tools.exec_command({ | |
| cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"" | |
| }); | |
| const resultVariableWasTruncated = result.output.length !== 50000; | |
| text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`); | |
| "#, | |
| TOKEN_POLICY_TEST_MODEL, | |
| |config| { | |
| config.tool_output_token_limit = Some(50); | |
| }, | |
| ) | |
| .await?; | |
| let items = custom_tool_output_items(&second_mock.single_request(), "call-1"); | |
| let output = text_item(&items, /*index*/ 1); | |
| // The 50-token override must shrink this 50,000-character value far below | |
| // what the default 10,000-token history cap would retain. | |
| assert!( | |
| output.len() < 1_000, | |
| "expected configured history cap to truncate the emitted value, got {} bytes", | |
| output.len() | |
| ); | |
| assert_regex_match( | |
| r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$", | |
| output, | |
| ); | |
| Ok(()) | |
| } | |
| // The outer directive limits output after JavaScript emits it; it does not | |
| // limit `result.output` returned by the nested command. | |
| async fn code_mode_exec_outer_limit_truncates_emitted_output() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "use exec_command from code mode", | |
| r#"// @exec: {"max_output_tokens": 5} | |
| const result = await tools.exec_command({ | |
| cmd: "printf '0123456789012345678901234567890123456789'" | |
| }); | |
| text(result.output); | |
| "#, | |
| ) | |
| .await?; | |
| assert_eq!( | |
| text_item( | |
| &custom_tool_output_items(&second_mock.single_request(), "call-1"), | |
| /*index*/ 1 | |
| ), | |
| "Warning: truncated output (original token count: 10)\nTotal output lines: 1\n\n0123456789…5 tokens truncated…0123456789" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_returns_accumulated_output_when_script_fails() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "use code_mode to surface script failures", | |
| r#" | |
| text("before crash"); | |
| text("still before crash"); | |
| throw new Error("boom"); | |
| "#, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| assert_eq!(items.len(), 4); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&items, /*index*/ 1), "before crash"); | |
| assert_eq!(text_item(&items, /*index*/ 2), "still before crash"); | |
| assert_regex_match( | |
| r#"(?sx) | |
| \A | |
| Script\ error:\n | |
| Error:\ boom\n | |
| (?:\s+at\ .+\n?)+ | |
| \z | |
| "#, | |
| text_item(&items, /*index*/ 3), | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exec_surfaces_handler_errors_as_exceptions() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "surface nested tool handler failures as script exceptions", | |
| r#" | |
| try { | |
| await tools.exec_command({}); | |
| text("no-exception"); | |
| } catch (error) { | |
| text(`caught:${error?.message ?? String(error)}`); | |
| } | |
| "#, | |
| ) | |
| .await?; | |
| let request = second_mock.single_request(); | |
| let metadata = | |
| &request.custom_tool_call_output("call-1")["internal_chat_message_metadata_passthrough"]; | |
| assert_eq!( | |
| metadata["executed_tool_calls"], | |
| serde_json::json!([ | |
| {"name": "exec_command", "arguments": {}}, | |
| ]), | |
| "failed nested tool attempts remain private request metadata", | |
| ); | |
| assert_eq!(metadata["tool_calls_complete"], Value::Bool(true)); | |
| let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "script should catch the nested tool error: {output}" | |
| ); | |
| assert!( | |
| output.contains("caught:"), | |
| "expected caught exception text in output: {output}" | |
| ); | |
| assert!( | |
| !output.contains("no-exception"), | |
| "nested tool error should not allow success path: {output}" | |
| ); | |
| Ok(()) | |
| } | |
| /// A stalled host wait must return its timeout to the model and reconnect for the next exec. | |
| async fn code_mode_wait_timeout_reconnects_on_next_exec() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_config(|config| { | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("code mode should be enabled"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| let first_turn = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| "yield_control(); await new Promise(() => {});", | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "waiting"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_turn("start a stalled code-mode cell").await?; | |
| let first_request = first_turn | |
| .last_request() | |
| .expect("initial exec should be returned to the model"); | |
| let first_items = custom_tool_output_items(&first_request, "call-1"); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| let timeout_completion = responses::mount_function_call_agent_response( | |
| &server, | |
| "call-2", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id, | |
| "yield_time_ms": 60_000, | |
| }))?, | |
| "wait", | |
| ) | |
| .await | |
| .completion; | |
| test.codex | |
| .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { | |
| text: "wait for the stalled cell".to_string(), | |
| text_elements: Vec::new(), | |
| }])) | |
| .await?; | |
| wait_for_event_match(&test.codex, |event| match event { | |
| EventMsg::RawResponseItem(raw) => match &raw.item { | |
| ResponseItem::FunctionCall { call_id, .. } if call_id == "call-2" => Some(()), | |
| _ => None, | |
| }, | |
| _ => None, | |
| }) | |
| .await; | |
| tokio::time::pause(); | |
| for _ in 0..130 { | |
| tokio::time::advance(Duration::from_secs(1)).await; | |
| if timeout_completion | |
| .function_call_output_text("call-2") | |
| .is_some() | |
| { | |
| break; | |
| } | |
| } | |
| tokio::time::resume(); | |
| wait_for_event(&test.codex, |event| { | |
| matches!(event, EventMsg::TurnComplete(_)) | |
| }) | |
| .await; | |
| let timeout_output = timeout_completion | |
| .function_call_output_text("call-2") | |
| .expect("timed-out wait should be returned to the model"); | |
| assert!( | |
| timeout_output.contains("code-mode host timed out waiting for wait response"), | |
| "unexpected wait output: {timeout_output}" | |
| ); | |
| let reconnect_turn = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_response_created("resp-5"), | |
| ev_custom_tool_call( | |
| "call-3", | |
| "exec", | |
| r#"text("reconnected"); yield_control(); await new Promise(() => {});"#, | |
| ), | |
| ev_completed("resp-5"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-3", "reconnected"), | |
| ev_completed("resp-6"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_turn("run a cell after the timeout").await?; | |
| let reconnect_request = reconnect_turn | |
| .last_request() | |
| .expect("replacement exec should be returned to the model"); | |
| let reconnect_items = custom_tool_output_items(&reconnect_request, "call-3"); | |
| assert_eq!( | |
| extract_running_cell_id(text_item(&reconnect_items, /*index*/ 0)), | |
| "g2:1" | |
| ); | |
| assert_eq!(text_item(&reconnect_items, /*index*/ 1), "reconnected"); | |
| Ok(()) | |
| } | |
| struct ResponseIdObserver { | |
| response_ids: Mutex<Vec<(String, Option<String>)>>, | |
| originating_items: Mutex<Vec<(String, Option<codex_protocol::ResponseItemId>)>>, | |
| wait_started: tokio::sync::Notify, | |
| } | |
| impl ToolLifecycleContributor for ResponseIdObserver { | |
| fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> { | |
| Box::pin(async move { | |
| if matches!(input.tool_name.name.as_str(), "exec" | "exec_command") { | |
| self.originating_items.lock().unwrap().push(( | |
| input.tool_name.name.clone(), | |
| input.originating_item_id.cloned(), | |
| )); | |
| } | |
| if matches!(input.tool_name.name.as_str(), "exec_command" | "wait") { | |
| self.response_ids.lock().unwrap().push(( | |
| input.tool_name.name.clone(), | |
| input | |
| .turn_store | |
| .get::<codex_api::ResponseId>() | |
| .map(|id| id.0.clone()), | |
| )); | |
| if input.tool_name.name == "wait" { | |
| self.wait_started.notify_one(); | |
| } | |
| } | |
| }) | |
| } | |
| } | |
| async fn code_mode_can_yield_and_resume_with_wait() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let observer = Arc::new(ResponseIdObserver::default()); | |
| let mut extensions = ExtensionRegistryBuilder::<Config>::new(); | |
| extensions.tool_lifecycle_contributor(observer.clone()); | |
| let mut builder = test_codex() | |
| .with_extensions(Arc::new(extensions.build())) | |
| .with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let phase_2_gate = test.workspace_path("code-mode-phase-2.ready"); | |
| let phase_3_gate = test.workspace_path("code-mode-phase-3.ready"); | |
| let phase_2_wait = wait_for_file_source(&phase_2_gate)?; | |
| let phase_3_wait = wait_for_file_source(&phase_3_gate)?; | |
| let code = format!( | |
| r#" | |
| text("phase 1"); | |
| yield_control(); | |
| {phase_2_wait} | |
| text((await tools.exec_command({{cmd: "printf 'phase 2'"}})).output); | |
| {phase_3_wait} | |
| text((await tools.exec_command({{cmd: "printf 'phase 3'"}})).output); | |
| "# | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| responses::ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let first_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| responses::ev_response_created("resp-2"), | |
| ev_assistant_message("msg-1", "waiting"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start the long exec").await?; | |
| let first_request = first_completion.single_request(); | |
| let first_items = custom_tool_output_items(&first_request, "call-1"); | |
| assert_eq!(first_items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&first_items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1"); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| responses::ev_response_created("resp-3"), | |
| responses::ev_function_call( | |
| "call-2", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id.clone(), | |
| "yield_time_ms": 1_000, | |
| }))?, | |
| ), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let second_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "still waiting"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| tokio::try_join!(test.submit_turn("wait again"), async { | |
| observer.wait_started.notified().await; | |
| fs::write(&phase_2_gate, "ready")?; | |
| anyhow::Ok(()) | |
| })?; | |
| let second_request = second_completion.single_request(); | |
| let second_items = function_tool_output_items(&second_request, "call-2"); | |
| assert_eq!(second_items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&second_items, /*index*/ 0), | |
| ); | |
| assert_eq!( | |
| extract_running_cell_id(text_item(&second_items, /*index*/ 0)), | |
| cell_id | |
| ); | |
| assert_eq!(text_item(&second_items, /*index*/ 1), "phase 2"); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| responses::ev_response_created("resp-5"), | |
| responses::ev_function_call( | |
| "call-3", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id.clone(), | |
| "yield_time_ms": 1_000, | |
| }))?, | |
| ), | |
| ev_completed("resp-5"), | |
| ]), | |
| ) | |
| .await; | |
| let third_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-3", "done"), | |
| ev_completed("resp-6"), | |
| ]), | |
| ) | |
| .await; | |
| tokio::try_join!(test.submit_turn("wait for completion"), async { | |
| observer.wait_started.notified().await; | |
| fs::write(&phase_3_gate, "ready")?; | |
| anyhow::Ok(()) | |
| })?; | |
| let third_request = third_completion.single_request(); | |
| let third_items = function_tool_output_items(&third_request, "call-3"); | |
| assert_eq!(third_items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&third_items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&third_items, /*index*/ 1), "phase 3"); | |
| // Nested calls retain the original exec item and turn context when later | |
| // turns resume the cell with wait. | |
| let originating_items = observer.originating_items.lock().unwrap(); | |
| let wrapper_item = originating_items[0] | |
| .1 | |
| .clone() | |
| .expect("wrapper item identity"); | |
| assert!(originating_items.len() >= 3); | |
| assert!( | |
| originating_items | |
| .iter() | |
| .all(|(_, item)| item.as_ref() == Some(&wrapper_item)) | |
| ); | |
| let observed = observer.response_ids.lock().unwrap(); | |
| let mut nested_response_ids = observed | |
| .iter() | |
| .skip_while(|(tool, _)| tool != "wait") | |
| .filter(|(tool, _)| tool == "exec_command") | |
| .map(|(_, response_id)| response_id.clone()) | |
| .collect::<Vec<_>>(); | |
| nested_response_ids.dedup(); | |
| assert_eq!(nested_response_ids, vec![Some("resp-2".to_owned())]); | |
| assert_eq!( | |
| observed | |
| .iter() | |
| .filter(|(tool, _)| tool == "wait") | |
| .cloned() | |
| .collect::<Vec<_>>(), | |
| vec![ | |
| ("wait".to_owned(), Some("resp-3".to_owned())), | |
| ("wait".to_owned(), Some("resp-5".to_owned())), | |
| ] | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_yield_and_termination_are_not_starved_by_runtime_output() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let termination_gate = test.workspace_path("code-mode-output-termination.ready"); | |
| let termination_wait = wait_for_file_source(&termination_gate)?; | |
| // Exact controller arbitration is covered by deterministic code-mode contract tests. Keep | |
| // this end-to-end load bounded while exercising a substantial runtime output backlog. | |
| let code = format!( | |
| r#"// @exec: {{"yield_time_ms": 0, "max_output_tokens": 16}} | |
| for (let index = 0; index < 256; index++) {{ | |
| text(`event ${{index}}`); | |
| }} | |
| {termination_wait} | |
| "# | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let first_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "waiting"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start the bounded output backlog").await?; | |
| let first_request = first_completion.single_request(); | |
| let first_items = custom_tool_output_items(&first_request, "call-1"); | |
| assert_eq!(first_items.len(), 1); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&first_items, /*index*/ 0), | |
| ); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| responses::ev_function_call( | |
| "call-2", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id.clone(), | |
| "terminate": true, | |
| }))?, | |
| ), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let second_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "terminated"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("terminate it").await?; | |
| let second_request = second_completion.single_request(); | |
| let second_items = function_tool_output_items(&second_request, "call-2"); | |
| assert!(!second_items.is_empty()); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&second_items, /*index*/ 0), | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_run_multiple_yielded_sessions() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| let _ = config.features.enable(Feature::ExecutedToolCallMetadata); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let session_a_gate = test.workspace_path("code-mode-session-a.ready"); | |
| let session_b_gate = test.workspace_path("code-mode-session-b.ready"); | |
| let session_a_wait = wait_for_file_source(&session_a_gate)?; | |
| let session_b_wait = wait_for_file_source(&session_b_gate)?; | |
| let session_a_code = format!( | |
| r#" | |
| await tools.exec_command({{ cmd: "printf cell-a" }}); | |
| text("session a start"); | |
| yield_control(); | |
| {session_a_wait} | |
| text("session a done"); | |
| "# | |
| ); | |
| let session_b_code = format!( | |
| r#" | |
| await tools.exec_command({{ cmd: "printf cell-b" }}); | |
| text("session b start"); | |
| yield_control(); | |
| {session_b_wait} | |
| text("session b done"); | |
| "# | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &session_a_code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let first_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "session a waiting"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start session a").await?; | |
| let first_request = first_completion.single_request(); | |
| let first_items = custom_tool_output_items(&first_request, "call-1"); | |
| assert_eq!(first_items.len(), 2); | |
| let session_a_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| assert_eq!(text_item(&first_items, /*index*/ 1), "session a start"); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| ev_custom_tool_call("call-2", "exec", &session_b_code), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let second_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "session b waiting"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start session b").await?; | |
| let second_request = second_completion.single_request(); | |
| let second_items = custom_tool_output_items(&second_request, "call-2"); | |
| assert_eq!(second_items.len(), 2); | |
| let session_b_id = extract_running_cell_id(text_item(&second_items, /*index*/ 0)); | |
| assert_eq!(text_item(&second_items, /*index*/ 1), "session b start"); | |
| assert_ne!(session_a_id, session_b_id); | |
| fs::write(&session_a_gate, "ready")?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-5"), | |
| responses::ev_function_call( | |
| "call-3", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": session_a_id.clone(), | |
| "yield_time_ms": 5_000, | |
| }))?, | |
| ), | |
| ev_completed("resp-5"), | |
| ]), | |
| ) | |
| .await; | |
| let third_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-3", "session a done"), | |
| ev_completed("resp-6"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("wait session a").await?; | |
| let third_request = third_completion.single_request(); | |
| let third_items = function_tool_output_items(&third_request, "call-3"); | |
| assert_eq!(third_items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&third_items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&third_items, /*index*/ 1), "session a done"); | |
| fs::write(&session_b_gate, "ready")?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-7"), | |
| responses::ev_function_call( | |
| "call-4", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": session_b_id.clone(), | |
| "yield_time_ms": 5_000, | |
| }))?, | |
| ), | |
| ev_completed("resp-7"), | |
| ]), | |
| ) | |
| .await; | |
| let fourth_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-4", "session b done"), | |
| ev_completed("resp-8"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("wait session b").await?; | |
| let fourth_request = fourth_completion.single_request(); | |
| let fourth_items = function_tool_output_items(&fourth_request, "call-4"); | |
| assert_eq!(fourth_items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&fourth_items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&fourth_items, /*index*/ 1), "session b done"); | |
| for (output, originating_call_id, expected_command) in [ | |
| ( | |
| first_request.custom_tool_call_output("call-1"), | |
| "call-1", | |
| Some("printf cell-a"), | |
| ), | |
| ( | |
| second_request.custom_tool_call_output("call-2"), | |
| "call-2", | |
| Some("printf cell-b"), | |
| ), | |
| (third_request.function_call_output("call-3"), "call-1", None), | |
| ( | |
| fourth_request.function_call_output("call-4"), | |
| "call-2", | |
| None, | |
| ), | |
| ] { | |
| let metadata = &output["internal_chat_message_metadata_passthrough"]; | |
| assert_eq!(metadata["cell_id"], serde_json::json!(originating_call_id)); | |
| assert_eq!( | |
| metadata.get("tool_calls_complete").and_then(Value::as_bool), | |
| expected_command.is_none().then_some(true), | |
| ); | |
| assert_eq!( | |
| metadata | |
| .get("executed_tool_calls") | |
| .and_then(Value::as_array) | |
| .and_then(|calls| { | |
| calls.iter().find_map(|call| { | |
| call["arguments"]["cmd"] | |
| .as_str() | |
| .filter(|command| command.starts_with("printf cell-")) | |
| }) | |
| }), | |
| expected_command, | |
| ); | |
| } | |
| Ok(()) | |
| } | |
| async fn code_mode_concurrent_cells_merge_only_the_stored_values_they_write() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let first_gate = test.workspace_path("code-mode-first-store.ready"); | |
| let first_wait = wait_for_file_source(&first_gate)?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-init", | |
| "exec", | |
| r#" | |
| store("a", 1); | |
| store("b", 2); | |
| "#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "initialized"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("initialize stored values").await?; | |
| let first_code = format!( | |
| r#" | |
| store("a", 3); | |
| yield_control(); | |
| {first_wait} | |
| "# | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| ev_custom_tool_call("call-first", "exec", &first_code), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let first_started = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "first pending"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start first store").await?; | |
| let first_request = first_started.single_request(); | |
| let first_items = custom_tool_output_items(&first_request, "call-first"); | |
| let first_cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-5"), | |
| ev_custom_tool_call("call-second", "exec", r#"store("b", 4);"#), | |
| ev_completed("resp-5"), | |
| ]), | |
| ) | |
| .await; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-3", "second complete"), | |
| ev_completed("resp-6"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("write the second key").await?; | |
| fs::write(&first_gate, "ready")?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-7"), | |
| responses::ev_function_call( | |
| "call-wait", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": first_cell_id, | |
| "yield_time_ms": 1_000, | |
| }))?, | |
| ), | |
| ev_completed("resp-7"), | |
| ]), | |
| ) | |
| .await; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-4", "first completed"), | |
| ev_completed("resp-8"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("complete the first store").await?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-9"), | |
| ev_custom_tool_call( | |
| "call-check", | |
| "exec", | |
| r#"text(JSON.stringify({ a: load("a"), b: load("b") }));"#, | |
| ), | |
| ev_completed("resp-9"), | |
| ]), | |
| ) | |
| .await; | |
| let check_response = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-5", "checked"), | |
| ev_completed("resp-10"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("check merged stored values").await?; | |
| let check_request = check_response.single_request(); | |
| let stored_values: Value = serde_json::from_str( | |
| &custom_tool_output_last_non_empty_text(&check_request, "call-check") | |
| .expect("checking stored values should emit JSON"), | |
| )?; | |
| assert_eq!(stored_values, serde_json::json!({ "a": 3, "b": 4 })); | |
| Ok(()) | |
| } | |
| async fn code_mode_wait_can_terminate_and_continue() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let termination_gate = test.workspace_path("code-mode-terminate.ready"); | |
| let termination_wait = wait_for_file_source(&termination_gate)?; | |
| let code = format!( | |
| r#" | |
| text("phase 1"); | |
| yield_control(); | |
| {termination_wait} | |
| text("phase 2"); | |
| "# | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let first_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "waiting"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start the long exec").await?; | |
| let first_request = first_completion.single_request(); | |
| let first_items = custom_tool_output_items(&first_request, "call-1"); | |
| assert_eq!(first_items.len(), 2); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1"); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| responses::ev_function_call( | |
| "call-2", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id.clone(), | |
| "terminate": true, | |
| }))?, | |
| ), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let second_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "terminated"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("terminate it").await?; | |
| let second_request = second_completion.single_request(); | |
| let second_items = function_tool_output_items(&second_request, "call-2"); | |
| assert_eq!(second_items.len(), 1); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&second_items, /*index*/ 0), | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-5"), | |
| ev_custom_tool_call( | |
| "call-3", | |
| "exec", | |
| r#" | |
| text("after terminate"); | |
| "#, | |
| ), | |
| ev_completed("resp-5"), | |
| ]), | |
| ) | |
| .await; | |
| let third_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-3", "done"), | |
| ev_completed("resp-6"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("run another exec").await?; | |
| let third_request = third_completion.single_request(); | |
| let third_items = custom_tool_output_items(&third_request, "call-3"); | |
| assert_eq!(third_items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&third_items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&third_items, /*index*/ 1), "after terminate"); | |
| Ok(()) | |
| } | |
| async fn code_mode_wait_returns_error_for_unknown_session() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| responses::ev_function_call( | |
| "call-1", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": "999999", | |
| "yield_time_ms": 1_000, | |
| }))?, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("wait on an unknown exec cell").await?; | |
| let request = completion.single_request(); | |
| let (_, success) = request | |
| .function_call_output_content_and_success("call-1") | |
| .expect("function tool output should be present"); | |
| assert_ne!(success, Some(true)); | |
| let items = function_tool_output_items(&request, "call-1"); | |
| assert_eq!(items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&items, /*index*/ 0), | |
| ); | |
| assert_eq!( | |
| text_item(&items, /*index*/ 1), | |
| "Script error:\nexec cell 999999 not found" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_wait_terminate_returns_completed_session_if_it_finished_after_yield_control() | |
| -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let session_a_gate = test.workspace_path("code-mode-session-a-finished.ready"); | |
| let session_b_gate = test.workspace_path("code-mode-session-b-blocked.ready"); | |
| let session_a_done_marker = test.workspace_path("code-mode-session-a-done.txt"); | |
| let session_a_wait = wait_for_file_source(&session_a_gate)?; | |
| let session_b_wait = wait_for_file_source(&session_b_gate)?; | |
| let session_a_done_marker_quoted = | |
| shlex::try_join([session_a_done_marker.to_string_lossy().as_ref()])?; | |
| let session_a_done_command = format!("printf done > {session_a_done_marker_quoted}"); | |
| let session_a_code = format!( | |
| r#" | |
| text("session a start"); | |
| yield_control(); | |
| {session_a_wait} | |
| text("session a done"); | |
| await tools.exec_command({{ cmd: {session_a_done_command:?} }}); | |
| "# | |
| ); | |
| let session_b_code = format!( | |
| r#" | |
| text("session b start"); | |
| yield_control(); | |
| {session_b_wait} | |
| text("session b done"); | |
| "# | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &session_a_code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let first_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "session a waiting"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start session a").await?; | |
| let first_request = first_completion.single_request(); | |
| let first_items = custom_tool_output_items(&first_request, "call-1"); | |
| assert_eq!(first_items.len(), 2); | |
| let session_a_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| assert_eq!(text_item(&first_items, /*index*/ 1), "session a start"); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| ev_custom_tool_call("call-2", "exec", &session_b_code), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let second_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "session b waiting"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start session b").await?; | |
| let second_request = second_completion.single_request(); | |
| let second_items = custom_tool_output_items(&second_request, "call-2"); | |
| assert_eq!(second_items.len(), 2); | |
| let session_b_id = extract_running_cell_id(text_item(&second_items, /*index*/ 0)); | |
| assert_eq!(text_item(&second_items, /*index*/ 1), "session b start"); | |
| fs::write(&session_a_gate, "ready")?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-5"), | |
| responses::ev_function_call( | |
| "call-3", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": session_b_id.clone(), | |
| "yield_time_ms": 1_000, | |
| }))?, | |
| ), | |
| ev_completed("resp-5"), | |
| ]), | |
| ) | |
| .await; | |
| let third_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-3", "session b still waiting"), | |
| ev_completed("resp-6"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("wait session b").await?; | |
| let third_request = third_completion.single_request(); | |
| let third_items = function_tool_output_items(&third_request, "call-3"); | |
| assert_eq!(third_items.len(), 1); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&third_items, /*index*/ 0), | |
| ); | |
| assert_eq!( | |
| extract_running_cell_id(text_item(&third_items, /*index*/ 0)), | |
| session_b_id | |
| ); | |
| for _ in 0..100 { | |
| if session_a_done_marker.exists() { | |
| break; | |
| } | |
| tokio::time::sleep(Duration::from_millis(50)).await; | |
| } | |
| assert!(session_a_done_marker.exists()); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-7"), | |
| responses::ev_function_call( | |
| "call-4", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": session_a_id.clone(), | |
| "terminate": true, | |
| }))?, | |
| ), | |
| ev_completed("resp-7"), | |
| ]), | |
| ) | |
| .await; | |
| let fourth_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-4", "session a already done"), | |
| ev_completed("resp-8"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("terminate session a").await?; | |
| let fourth_request = fourth_completion.single_request(); | |
| let fourth_items = function_tool_output_items(&fourth_request, "call-4"); | |
| match fourth_items.len() { | |
| 1 => { | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&fourth_items, /*index*/ 0), | |
| ); | |
| } | |
| 2 => { | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script (?:completed|terminated)\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&fourth_items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&fourth_items, /*index*/ 1), "session a done"); | |
| } | |
| other => panic!("unexpected number of content items: {other}"), | |
| } | |
| Ok(()) | |
| } | |
| async fn code_mode_background_keeps_running_on_later_turn_without_wait() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let resumed_file = test.workspace_path("code-mode-yield-resumed.txt"); | |
| let resumed_file_quoted = shlex::try_join([resumed_file.to_string_lossy().as_ref()])?; | |
| let write_file_command = format!("printf resumed > {resumed_file_quoted}"); | |
| let wait_for_file_command = | |
| format!("while [ ! -f {resumed_file_quoted} ]; do sleep 0.01; done; printf ready"); | |
| let code = format!( | |
| r#" | |
| text("before yield"); | |
| yield_control(); | |
| await tools.exec_command({{ cmd: {write_file_command:?} }}); | |
| text("after yield"); | |
| "# | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let first_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "exec yielded"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start yielded exec").await?; | |
| let first_request = first_completion.single_request(); | |
| let first_items = custom_tool_output_items(&first_request, "call-1"); | |
| assert_eq!(first_items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&first_items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&first_items, /*index*/ 1), "before yield"); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| responses::ev_function_call( | |
| "call-2", | |
| "exec_command", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cmd": wait_for_file_command, | |
| }))?, | |
| ), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let second_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "file appeared"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("wait for resumed file").await?; | |
| let second_request = second_completion.single_request(); | |
| assert!( | |
| second_request | |
| .function_call_output_text("call-2") | |
| .is_some_and(|output| output.ends_with("ready")) | |
| ); | |
| assert_eq!(fs::read_to_string(&resumed_file)?, "resumed"); | |
| Ok(()) | |
| } | |
| struct InterruptedNestedToolObserver { | |
| started: Mutex<Option<oneshot::Sender<String>>>, | |
| finished: Mutex<Option<oneshot::Sender<ToolCallOutcome>>>, | |
| } | |
| impl ToolLifecycleContributor for InterruptedNestedToolObserver { | |
| fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> { | |
| Box::pin(async move { | |
| let codex_extension_api::ToolCallSource::CodeMode { cell_id, .. } = input.source else { | |
| return; | |
| }; | |
| if input.tool_name.name != "test_sync_tool" { | |
| return; | |
| } | |
| if let Some(started) = self | |
| .started | |
| .lock() | |
| .unwrap_or_else(std::sync::PoisonError::into_inner) | |
| .take() | |
| { | |
| let _ = started.send(cell_id); | |
| } | |
| }) | |
| } | |
| fn on_tool_finish<'a>(&'a self, input: ToolFinishInput<'a>) -> ToolLifecycleFuture<'a> { | |
| Box::pin(async move { | |
| if input.tool_name.name != "test_sync_tool" | |
| || !matches!( | |
| input.source, | |
| codex_extension_api::ToolCallSource::CodeMode { .. } | |
| ) | |
| { | |
| return; | |
| } | |
| if let Some(finished) = self | |
| .finished | |
| .lock() | |
| .unwrap_or_else(std::sync::PoisonError::into_inner) | |
| .take() | |
| { | |
| let _ = finished.send(input.outcome); | |
| } | |
| }) | |
| } | |
| } | |
| async fn code_mode_interrupt_terminates_active_cells_and_nested_tools() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let (started_tx, started_rx) = oneshot::channel(); | |
| let (finished_tx, finished_rx) = oneshot::channel(); | |
| let mut extensions = ExtensionRegistryBuilder::<Config>::new(); | |
| extensions.tool_lifecycle_contributor(Arc::new(InterruptedNestedToolObserver { | |
| started: Mutex::new(Some(started_tx)), | |
| finished: Mutex::new(Some(finished_tx)), | |
| })); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_extensions(Arc::new(extensions.build())) | |
| .with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| let _ = config.features.enable(Feature::CodeModeInterrupt); | |
| let _ = config.features.enable(Feature::ExecutedToolCallMetadata); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| let setup = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_response_created("resp-store"), | |
| ev_custom_tool_call("call-store", "exec", r#"store("persisted", "preserved");"#), | |
| ev_completed("resp-store"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-store", "stored"), | |
| ev_completed("resp-store-complete"), | |
| ]), | |
| sse(vec![ | |
| ev_response_created("resp-background"), | |
| ev_custom_tool_call( | |
| "call-background", | |
| "exec", | |
| "yield_control(); await new Promise(() => {});", | |
| ), | |
| ev_completed("resp-background"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-background", "running"), | |
| ev_completed("resp-background-complete"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_turn("store a value in the reusable code-mode session") | |
| .await?; | |
| test.submit_turn("start a background code-mode cell") | |
| .await?; | |
| let background_response = setup | |
| .last_request() | |
| .expect("background cell should be returned to the model"); | |
| let background_items = custom_tool_output_items(&background_response, "call-background"); | |
| assert!( | |
| text_item(&background_items, /*index*/ 0).starts_with("Script running with cell ID "), | |
| "background cell should remain active: {background_items:?}" | |
| ); | |
| let background_cell_id = | |
| extract_running_cell_id(text_item(&background_items, /*index*/ 0)); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-interrupted"), | |
| ev_custom_tool_call( | |
| "call-interrupted", | |
| "exec", | |
| "await tools.test_sync_tool({ sleep_after_ms: 60_000 });", | |
| ), | |
| ev_completed("resp-interrupted"), | |
| ]), | |
| ) | |
| .await; | |
| test.codex | |
| .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { | |
| text: "start a long-running nested tool".to_string(), | |
| text_elements: Vec::new(), | |
| }])) | |
| .await?; | |
| let active_cell_id = tokio::time::timeout(Duration::from_secs(10), started_rx).await??; | |
| test.codex.submit(Op::Interrupt).await?; | |
| wait_for_event(&test.codex, |event| { | |
| matches!(event, EventMsg::TurnAborted(_)) | |
| }) | |
| .await; | |
| let nested_outcome = tokio::time::timeout(Duration::from_secs(10), finished_rx).await??; | |
| assert_eq!(nested_outcome, ToolCallOutcome::Aborted); | |
| let recovery = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_response_created("resp-wait-background"), | |
| responses::ev_function_call( | |
| "call-wait-background", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": background_cell_id, | |
| "yield_time_ms": 1, | |
| }))?, | |
| ), | |
| ev_completed("resp-wait-background"), | |
| ]), | |
| sse(vec![ | |
| ev_response_created("resp-wait-active"), | |
| responses::ev_function_call( | |
| "call-wait-active", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": active_cell_id, | |
| "yield_time_ms": 1, | |
| }))?, | |
| ), | |
| ev_completed("resp-wait-active"), | |
| ]), | |
| sse(vec![ | |
| ev_response_created("resp-recovery"), | |
| ev_custom_tool_call("call-recovery", "exec", r#"text(load("persisted"));"#), | |
| ev_completed("resp-recovery"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-recovery", "recovered"), | |
| ev_completed("resp-recovery-complete"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_turn("verify interrupted cells and reuse their session") | |
| .await?; | |
| let requests = recovery.requests(); | |
| let metadata = &requests[0].custom_tool_call_output("call-interrupted")["internal_chat_message_metadata_passthrough"]; | |
| assert_eq!( | |
| metadata["executed_tool_calls"], | |
| serde_json::json!([{"name": "test_sync_tool", "arguments": {"sleep_after_ms": 60_000}}]), | |
| ); | |
| assert_eq!(metadata["tool_calls_complete"], Value::Bool(true)); | |
| let background_output = function_tool_output_items(&requests[1], "call-wait-background"); | |
| assert!(text_item(&background_output, /*index*/ 1).contains("not found")); | |
| let active_output = function_tool_output_items(&requests[2], "call-wait-active"); | |
| assert!(text_item(&active_output, /*index*/ 1).contains("not found")); | |
| let recovery_items = custom_tool_output_items(&requests[3], "call-recovery"); | |
| assert_eq!(text_item(&recovery_items, /*index*/ 1), "preserved"); | |
| Ok(()) | |
| } | |
| async fn code_mode_wait_uses_its_own_max_tokens_budget() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| // An explicit yield separates the output batches without a nested shell call. | |
| let code = r#"// @exec: {"max_output_tokens": 100} | |
| text("phase 1"); | |
| yield_control(); | |
| text("token one token two token three token four token five token six token seven"); | |
| "#; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let first_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "waiting"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start the long exec").await?; | |
| let first_request = first_completion.single_request(); | |
| let first_items = custom_tool_output_items(&first_request, "call-1"); | |
| assert_eq!(first_items.len(), 2, "exec output: {first_items:?}"); | |
| assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1"); | |
| let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| responses::ev_function_call( | |
| "call-2", | |
| "wait", | |
| &serde_json::to_string(&serde_json::json!({ | |
| "cell_id": cell_id.clone(), | |
| "yield_time_ms": 1_000, | |
| "max_tokens": 6, | |
| }))?, | |
| ), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let second_completion = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "done"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("wait for completion").await?; | |
| let second_request = second_completion.single_request(); | |
| let second_items = function_tool_output_items(&second_request, "call-2"); | |
| assert_eq!(second_items.len(), 2, "wait output: {second_items:?}"); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&second_items, /*index*/ 0), | |
| ); | |
| let expected_pattern = r#"(?sx) | |
| \A | |
| Warning:\ truncated\ output\ \(original\ token\ count:\ \d+\)\n | |
| Total\ output\ lines:\ 1\n | |
| \n | |
| .*…\d+\ tokens\ truncated….* | |
| \z | |
| "#; | |
| assert_regex_match(expected_pattern, text_item(&second_items, /*index*/ 1)); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_resume_after_set_timeout() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "use exec to wait for a timeout", | |
| r#" | |
| await new Promise((resolve) => setTimeout(resolve, 10)); | |
| text("timer done"); | |
| "#, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec setTimeout call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!(output, "timer done"); | |
| Ok(()) | |
| } | |
| async fn code_mode_notify_injects_additional_exec_tool_output_into_active_context() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "use exec notify helper", | |
| r#" | |
| notify("code_mode_notify_marker"); | |
| text("done"); | |
| "#, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let has_notify_output = req | |
| .inputs_of_type("custom_tool_call_output") | |
| .iter() | |
| .any(|item| { | |
| item.get("call_id").and_then(serde_json::Value::as_str) == Some("call-1") | |
| && item | |
| .get("output") | |
| .and_then(serde_json::Value::as_str) | |
| .is_some_and(|text| text.contains("code_mode_notify_marker")) | |
| && item.get("name").and_then(serde_json::Value::as_str) == Some("exec") | |
| }); | |
| assert!( | |
| has_notify_output, | |
| "expected notify marker in custom_tool_call_output item: {:?}", | |
| req.input() | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exit_stops_script_immediately() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "use exec to stop script early with exit helper", | |
| r#" | |
| text("before"); | |
| exit(); | |
| text("after"); | |
| "#, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec exit helper call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!(items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&items, /*index*/ 1), "before"); | |
| assert_eq!(output, "before"); | |
| Ok(()) | |
| } | |
| async fn code_mode_omits_short_audio_and_preserves_other_output() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let [short_audio_url, valid_audio_url] = [120_u32, 600].map(|frames| { | |
| let sample_rate = 24_000_u32; | |
| let data_size = frames * 2; | |
| let mut wav = b"RIFF".to_vec(); | |
| wav.extend_from_slice(&(36 + data_size).to_le_bytes()); | |
| wav.extend_from_slice(b"WAVEfmt "); | |
| wav.extend_from_slice(&16_u32.to_le_bytes()); | |
| wav.extend_from_slice(&1_u16.to_le_bytes()); | |
| wav.extend_from_slice(&1_u16.to_le_bytes()); | |
| wav.extend_from_slice(&sample_rate.to_le_bytes()); | |
| wav.extend_from_slice(&(sample_rate * 2).to_le_bytes()); | |
| wav.extend_from_slice(&2_u16.to_le_bytes()); | |
| wav.extend_from_slice(&16_u16.to_le_bytes()); | |
| wav.extend_from_slice(b"data"); | |
| wav.extend_from_slice(&data_size.to_le_bytes()); | |
| wav.resize(44 + data_size as usize, /*value*/ 0); | |
| format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav)) | |
| }); | |
| let code = format!( | |
| r#" | |
| text("before"); | |
| audio({short_audio_url:?}); | |
| text("between"); | |
| audio({valid_audio_url:?}); | |
| text("after"); | |
| "#, | |
| ); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.4", |model| { | |
| model.input_modalities.push(InputModality::Audio); | |
| }) | |
| .with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("use exec to return short and valid audio with text") | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!(success, Some(false)); | |
| assert_eq!( | |
| &items[1..], | |
| &[ | |
| serde_json::json!({ "type": "input_text", "text": "before" }), | |
| serde_json::json!({ | |
| "type": "input_text", | |
| "text": "Audio output omitted because the clip is shorter than 25 ms; use a longer clip." | |
| }), | |
| serde_json::json!({ "type": "input_text", "text": "between" }), | |
| serde_json::json!({ "type": "input_audio", "audio_url": valid_audio_url }), | |
| serde_json::json!({ "type": "input_text", "text": "after" }), | |
| ] | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_replaces_malformed_image() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "use exec to return an image", | |
| r#"image("data:image/png;base64,AAA");"#, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!(success, Some(false)); | |
| assert_eq!(items.len(), 2); | |
| assert_eq!( | |
| items[1], | |
| serde_json::json!({ | |
| "type": "input_text", | |
| "text": "image content omitted because it could not be processed" | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_resizes_explicit_original_image() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let original_dimensions = (6401, 100); | |
| let image = ImageBuffer::from_pixel( | |
| original_dimensions.0, | |
| original_dimensions.1, | |
| Rgba([20, 40, 60, 255]), | |
| ); | |
| let mut encoded = Cursor::new(Vec::new()); | |
| DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?; | |
| let image_data_url = format!( | |
| "data:image/png;base64,{}", | |
| BASE64_STANDARD.encode(encoded.into_inner()) | |
| ); | |
| let code = format!( | |
| "image({}, \"original\");", | |
| serde_json::to_string(&image_data_url)? | |
| ); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_model_and_config( | |
| &server, | |
| "use exec to return a large original-detail image", | |
| &code, | |
| "gpt-5.5", | |
| |_| {}, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!(success, Some(false)); | |
| let resized_url = items[1]["image_url"] | |
| .as_str() | |
| .expect("code mode image output should contain a data URL"); | |
| assert_eq!(items[1]["detail"], "original"); | |
| let (_, resized_base64) = resized_url | |
| .split_once(',') | |
| .expect("resized image should contain a data URL prefix"); | |
| let resized_bytes = BASE64_STANDARD.decode(resized_base64)?; | |
| let resized = image::load_from_memory(&resized_bytes)?; | |
| let resized_dimensions = resized.dimensions(); | |
| assert_eq!(resized_dimensions, (6000, 94)); | |
| Ok(()) | |
| } | |
| async fn code_mode_unified_image_budget_accepts_legacy_detail_hints() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let image = ImageBuffer::from_pixel( | |
| /*width*/ 2304, | |
| /*height*/ 864, | |
| Rgba([20, 40, 60, 255]), | |
| ); | |
| let mut encoded = Cursor::new(Vec::new()); | |
| DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?; | |
| let image_data = BASE64_STANDARD.encode(encoded.into_inner()); | |
| let code = format!( | |
| r#" | |
| const data = {}; | |
| const imageUrl = `data:image/png;base64,${{data}}`; | |
| image(imageUrl); | |
| image(imageUrl, "auto"); | |
| image(imageUrl, "high"); | |
| image({{ image_url: imageUrl, detail: "low" }}); | |
| image({{ | |
| type: "image", | |
| data, | |
| mimeType: "image/png", | |
| _meta: {{ "codex/imageDetail": "original" }} | |
| }}); | |
| "#, | |
| serde_json::to_string(&image_data)? | |
| ); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn_with_model_and_config( | |
| &server, | |
| "emit images with legacy detail arguments and MCP metadata", | |
| &code, | |
| "gpt-5.5", | |
| |config| { | |
| let _ = config.features.enable(Feature::UnifiedImageBudget); | |
| }, | |
| ) | |
| .await?; | |
| let request = second_mock.single_request(); | |
| let items = custom_tool_output_items(&request, "call-1"); | |
| assert_eq!(items.len(), 6, "unexpected code-mode output: {items:?}"); | |
| for item in &items[1..] { | |
| assert_eq!(item["type"], "input_image"); | |
| assert_eq!(item["detail"], "original"); | |
| let image_url = item["image_url"] | |
| .as_str() | |
| .expect("image helper should emit a data URL"); | |
| let (_, payload) = image_url | |
| .split_once(',') | |
| .expect("image data URL should have a base64 payload"); | |
| let image = image::load_from_memory(&BASE64_STANDARD.decode(payload)?)?; | |
| assert_eq!(image.dimensions(), (2304, 864)); | |
| } | |
| let body = request.body_json(); | |
| let exec_description = body["tools"] | |
| .as_array() | |
| .and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec")) | |
| .and_then(|tool| tool["description"].as_str()) | |
| .expect("the model request should contain the code-mode exec tool"); | |
| assert!( | |
| exec_description | |
| .contains("`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`") | |
| ); | |
| assert!(!exec_description.contains("codex/imageDetail")); | |
| assert!(!exec_description.contains("detail?:")); | |
| Ok(()) | |
| } | |
| async fn code_mode_unified_image_budget_preserves_legacy_contract_for_unsupported_model() | |
| -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let builder = test_codex() | |
| .with_model_info_override("image-budget-unsupported-model", |model| { | |
| model.tool_mode = Some(ToolMode::CodeMode); | |
| model.supports_image_detail_original = false; | |
| }) | |
| .with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| let _ = config.features.enable(Feature::ExecutedToolCallMetadata); | |
| let _ = config.features.enable(Feature::UnifiedImageBudget); | |
| }); | |
| let (_test, second_mock) = run_code_mode_turn_with_builder( | |
| &server, | |
| "emit an image on a model without original-detail support", | |
| r#"image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==");"#, | |
| builder, | |
| ) | |
| .await?; | |
| let request = second_mock.single_request(); | |
| let items = custom_tool_output_items(&request, "call-1"); | |
| assert_eq!(items.len(), 2); | |
| assert_eq!(items[1]["detail"], "high"); | |
| let body = request.body_json(); | |
| let exec_description = body["tools"] | |
| .as_array() | |
| .and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec")) | |
| .and_then(|tool| tool["description"].as_str()) | |
| .expect("the model request should contain the code-mode exec tool"); | |
| assert!(exec_description.contains("codex/imageDetail")); | |
| assert!(exec_description.contains("detail?:")); | |
| Ok(()) | |
| } | |
| async fn code_mode_view_image_rejects_invalid_file_without_exposing_contents() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| const INVALID_IMAGE_CONTENTS: &str = "private-file-contents-must-not-be-exposed"; | |
| let server = responses::start_mock_server().await; | |
| let builder = test_codex() | |
| .with_model("gpt-5.5") | |
| .with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }) | |
| .with_workspace_setup(|cwd, _fs| async move { | |
| fs::write( | |
| cwd.join("not-an-image.txt").as_path(), | |
| INVALID_IMAGE_CONTENTS, | |
| )?; | |
| Ok(()) | |
| }); | |
| let (_test, second_mock) = run_code_mode_turn_with_builder( | |
| &server, | |
| "use exec to call view_image on a non-image file", | |
| r#"await tools.view_image({ path: "not-an-image.txt" });"#, | |
| builder, | |
| ) | |
| .await?; | |
| let request = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(true), | |
| "code-mode view_image unexpectedly accepted a non-image file" | |
| ); | |
| assert!( | |
| output.contains("unable to process image: invalid or unsupported image data"), | |
| "unexpected code-mode failure: {output}" | |
| ); | |
| let model_visible_output = serde_json::to_string(&request.custom_tool_call_output("call-1"))?; | |
| assert!( | |
| !model_visible_output.contains(INVALID_IMAGE_CONTENTS), | |
| "invalid file contents leaked into model-visible output: {model_visible_output}" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_use_view_image_result_with_image_helper( | |
| unified_image_budget: bool, | |
| ) -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model("gpt-5.5") | |
| .with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| if unified_image_budget { | |
| let _ = config.features.enable(Feature::UnifiedImageBudget); | |
| } | |
| }); | |
| let test = builder.build(&server).await?; | |
| let image = ImageBuffer::from_pixel( | |
| /*width*/ 2, | |
| /*height*/ 1, | |
| Rgba([255u8, 0, 0, 255]), | |
| ); | |
| let rotate_90_exif = vec![ | |
| 0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x12, 0x01, 0x03, 0x00, 0x01, | |
| 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, | |
| ]; | |
| let mut image_bytes = Vec::new(); | |
| let mut encoder = PngEncoder::new(&mut image_bytes); | |
| encoder.set_exif_metadata(rotate_90_exif.clone())?; | |
| encoder.write_image( | |
| image.as_raw(), | |
| image.width(), | |
| image.height(), | |
| image::ColorType::Rgba8.into(), | |
| )?; | |
| let image_path = test.cwd_path().join("code_mode_view_image.png"); | |
| fs::write(&image_path, &image_bytes)?; | |
| let image_path_json = serde_json::to_string(&image_path.to_string_lossy().to_string())?; | |
| let expected_output_keys = if unified_image_budget { | |
| r#"["image_url"]"# | |
| } else { | |
| r#"["detail","image_url"]"# | |
| }; | |
| let code = format!( | |
| r#" | |
| const out = await tools.view_image({{ path: {image_path_json}, detail: "original" }}); | |
| const keys = Object.keys(out).sort(); | |
| if (JSON.stringify(keys) !== JSON.stringify({expected_output_keys})) {{ | |
| throw new Error(`unexpected view_image output keys: ${{JSON.stringify(keys)}}`); | |
| }} | |
| image(out); | |
| "# | |
| ); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", &code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("use exec to call view_image and emit its image output") | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "code_mode view_image call failed unexpectedly" | |
| ); | |
| assert_eq!(items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&items, /*index*/ 0), | |
| ); | |
| assert_eq!( | |
| items[1].get("type").and_then(Value::as_str), | |
| Some("input_image") | |
| ); | |
| let emitted_image_url = items[1] | |
| .get("image_url") | |
| .and_then(Value::as_str) | |
| .expect("image helper should emit an input_image item with image_url"); | |
| assert!(emitted_image_url.starts_with("data:image/png;base64,")); | |
| let (_, emitted_image_base64) = emitted_image_url | |
| .split_once(',') | |
| .expect("emitted image should contain a data URL prefix"); | |
| let emitted_image_bytes = BASE64_STANDARD.decode(emitted_image_base64)?; | |
| assert_eq!(emitted_image_bytes, image_bytes); | |
| let mut decoder = ImageReader::with_format(Cursor::new(&emitted_image_bytes), ImageFormat::Png) | |
| .into_decoder()?; | |
| assert_eq!( | |
| ( | |
| decoder.dimensions(), | |
| decoder.orientation()?, | |
| decoder.exif_metadata()? | |
| ), | |
| ((2, 1), Orientation::Rotate90, Some(rotate_90_exif)) | |
| ); | |
| assert_eq!( | |
| items[1].get("detail").and_then(Value::as_str), | |
| Some("original") | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_use_mcp_image_result_with_image_helper() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const out = await tools.mcp__rmcp__image_scenario({ | |
| scenario: "image_only_original_detail", | |
| }); | |
| const imageItem = out.content.find((item) => item.type === "image"); | |
| image(imageItem); | |
| "#; | |
| let (_test, second_mock) = run_code_mode_turn_with_rmcp_model( | |
| &server, | |
| "use exec to call the rmcp image scenario tool and emit its image output", | |
| code, | |
| "gpt-5.5", | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "code_mode mcp image scenario call failed unexpectedly" | |
| ); | |
| assert_eq!(items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&items, /*index*/ 0), | |
| ); | |
| assert_eq!( | |
| items[1].get("type").and_then(Value::as_str), | |
| Some("input_image") | |
| ); | |
| let emitted_image_url = items[1] | |
| .get("image_url") | |
| .and_then(Value::as_str) | |
| .expect("image helper should emit an input_image item with image_url"); | |
| assert!(emitted_image_url.starts_with("data:image/png;base64,")); | |
| assert_eq!( | |
| items[1].get("detail").and_then(Value::as_str), | |
| Some("original") | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_apply_patch_via_nested_tool() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let file_name = "code_mode_apply_patch.txt"; | |
| let patch = format!( | |
| "*** Begin Patch\n*** Add File: {file_name}\n+hello from code_mode\n*** End Patch\n" | |
| ); | |
| let code = format!("text(await tools.apply_patch({patch:?}));\n"); | |
| let (test, second_mock) = | |
| run_code_mode_turn(&server, "use exec to run apply_patch", &code).await?; | |
| let req = second_mock.single_request(); | |
| let items = custom_tool_output_items(&req, "call-1"); | |
| let (_, success) = req | |
| .custom_tool_call_output_content_and_success("call-1") | |
| .expect("custom tool output should be present"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec apply_patch call failed unexpectedly: {items:?}" | |
| ); | |
| assert_eq!(items.len(), 2); | |
| assert_regex_match( | |
| concat!( | |
| r"(?s)\A", | |
| r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" | |
| ), | |
| text_item(&items, /*index*/ 0), | |
| ); | |
| assert_eq!(text_item(&items, /*index*/ 1), "{}"); | |
| let file_path = test.cwd_path().join(file_name); | |
| assert_eq!(fs::read_to_string(&file_path)?, "hello from code_mode\n"); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_print_structured_mcp_tool_result_fields() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({ | |
| message: "ping", | |
| }); | |
| text( | |
| `echo=${structuredContent?.echo ?? "missing"}\n` + | |
| `env=${structuredContent?.env ?? "missing"}\n` + | |
| `isError=${String(isError)}\n` + | |
| `contentLength=${content.length}` | |
| ); | |
| "#; | |
| let (_test, second_mock) = | |
| run_code_mode_turn_with_rmcp(&server, "use exec to run the rmcp echo tool", code).await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec rmcp echo call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!( | |
| output, | |
| "echo=ECHOING: ping | |
| env=propagated-env | |
| isError=false | |
| contentLength=0" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_node_repl_screenshots_can_be_captured_without_guardian_transcript_flags( | |
| repl_server: &'static str, | |
| ) -> Result<()> { | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "requires a Windows test_stdio_server in the Wine-exec environment" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| const SCREENSHOT: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; | |
| let server = responses::start_mock_server().await; | |
| let mcp_server_bin = remote_aware_stdio_server_bin()?; | |
| let mut builder = test_codex().with_config(move |config| { | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("enable Code Mode"); | |
| let mcp = serde_json::from_value(serde_json::json!({ | |
| "command": mcp_server_bin, | |
| "environment_id": remote_aware_environment_id(), | |
| "cwd": config.cwd, | |
| "env": { "MCP_TEST_ENABLE_NODE_REPL_JS": "1" }, | |
| "omit_tools_from": ["deferred"], | |
| })) | |
| .expect("valid REPL MCP server config"); | |
| config | |
| .mcp_servers | |
| .set(HashMap::from([(repl_server.to_owned(), mcp)])) | |
| .expect("configure REPL MCP server"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| core_test_support::wait_for_mcp_server(&test.codex, repl_server).await?; | |
| let evidence = test | |
| .codex | |
| .thread_extension_data() | |
| .get_or_init(NodeReplReviewEvidence::default); | |
| evidence.enable_image_capture(); | |
| let response_mock = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_custom_tool_call( | |
| "node-repl-image", | |
| "exec", | |
| &r#"for (let index = 0; index < 2; index++) await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' });"# | |
| .replace("node_repl", repl_server), | |
| ), | |
| ev_completed("response-node-repl"), | |
| ]), | |
| sse(vec![ev_completed("response-done")]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_text_turn("capture a computer-use screenshot") | |
| .await?; | |
| assert_eq!( | |
| evidence.images(), | |
| vec![ContentItem::InputImage { | |
| image: ImageReference::Inline { | |
| image_url: SCREENSHOT.to_owned() | |
| }, | |
| detail: Some(ImageDetail::Low), | |
| }] | |
| ); | |
| let parent_request = response_mock | |
| .requests() | |
| .pop() | |
| .expect("parent turn should complete"); | |
| assert!(!serde_json::to_string(&parent_request.input())?.contains(SCREENSHOT)); | |
| Ok(()) | |
| } | |
| async fn code_mode_node_repl_image_flag_without_enhanced_stays_disabled( | |
| repl_server: &'static str, | |
| ) -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| skip_if_sandbox!(Ok(())); | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "Guardian approval actions require host-native paths" | |
| ); | |
| let server = responses::start_mock_server().await; | |
| let mcp_server_bin = remote_aware_stdio_server_bin()?; | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.5", |model| { | |
| model.node_repl_auto_review_required = false; | |
| }) | |
| .with_config(move |config| { | |
| config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest); | |
| config.approvals_reviewer = ApprovalsReviewer::AutoReview; | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("enable Code Mode"); | |
| config | |
| .features | |
| .set_enabled( | |
| Feature::GuardianEnhancedNodeReplTranscripts, | |
| /*enabled*/ false, | |
| ) | |
| .expect("disable enhanced Guardian transcripts"); | |
| config | |
| .features | |
| .enable(Feature::GuardianNodeReplTranscriptImages) | |
| .expect("enable Guardian transcript images"); | |
| let mcp = serde_json::from_value(serde_json::json!({ | |
| "command": mcp_server_bin, | |
| "environment_id": remote_aware_environment_id(), | |
| "cwd": config.cwd, | |
| "env": { "MCP_TEST_ENABLE_NODE_REPL_JS": "1" }, | |
| "omit_tools_from": ["deferred"], | |
| })) | |
| .expect("valid REPL MCP server config"); | |
| config | |
| .mcp_servers | |
| .set(HashMap::from([(repl_server.to_owned(), mcp)])) | |
| .expect("configure REPL MCP server"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| wait_for_mcp_server(&test.codex, repl_server).await?; | |
| let code = r#" | |
| await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' }); | |
| await tools.exec_command({ cmd: "true", sandbox_permissions: "require_escalated", justification: "review" }); | |
| "# | |
| .replace("node_repl", repl_server); | |
| let response_mock = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_custom_tool_call("code-mode-call", "exec", &code), | |
| ev_completed("resp-parent"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("guardian", r#"{"outcome":"allow"}"#), | |
| ev_completed("resp-guardian"), | |
| ]), | |
| sse(vec![ev_completed("resp-done")]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_text_turn(&format!("review a nested {repl_server} screenshot")) | |
| .await?; | |
| let requests = response_mock.requests(); | |
| assert_eq!(requests.len(), 3); | |
| let guardian_request = requests | |
| .iter() | |
| .find(|request| { | |
| request.body_json()["client_metadata"]["x-openai-subagent"].as_str() == Some("guardian") | |
| }) | |
| .expect("the shell command should trigger Guardian review"); | |
| assert!(guardian_request.message_input_image_urls("user").is_empty()); | |
| let guardian_text = guardian_request.message_input_texts("user").concat(); | |
| let parent_input = serde_json::to_string(&requests.last().unwrap().input())?; | |
| for marker in [ | |
| "guardian-visible-before-image", | |
| "guardian-visible-after-image", | |
| "data:image/png;base64,", | |
| ] { | |
| assert!(!guardian_text.contains(marker)); | |
| assert!(!parent_input.contains(marker)); | |
| } | |
| Ok(()) | |
| } | |
| async fn code_mode_node_repl_text_evidence_is_visible_only_to_guardian( | |
| repl_server: &'static str, | |
| enhanced_transcripts: bool, | |
| transcript_images: bool, | |
| auto_review_required: bool, | |
| reviewer_constraint: Option<&'static str>, | |
| ) -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| skip_if_sandbox!(Ok(())); | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "Guardian approval actions require host-native paths" | |
| ); | |
| const NODE_REPL_DOM_MIDDLE: &str = "guardian-visible-dom-middle"; | |
| const DIRECT_NODE_REPL_MIDDLE: &str = "direct-node-repl-visible-middle"; | |
| const DIRECT_UNRELATED_MIDDLE: &str = "direct-unrelated-hidden-middle"; | |
| const OTHER_NODE_REPL_RESULT: &str = "ECHOING: guardian-visible-other-tool-result"; | |
| const UNRELATED_RESULT: &str = "ECHOING: guardian-hidden-unrelated-result"; | |
| const PRIVATE_IMAGE: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; | |
| const COMPACTION_SUMMARY: &str = "Guardian browser evidence summary"; | |
| let server = responses::start_mock_server().await; | |
| let mcp_server_bin = remote_aware_stdio_server_bin()?; | |
| let reviewer_compaction = reviewer_constraint == Some("compaction"); | |
| let check_detail = enhanced_transcripts && transcript_images && reviewer_constraint.is_none(); | |
| let mut large_image = Cursor::new(Vec::new()); | |
| if check_detail { | |
| DynamicImage::new_rgba8(/*w*/ 2049, /*h*/ 32) | |
| .write_to(&mut large_image, ImageFormat::Png)?; | |
| } | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.5", move |model| { | |
| model.node_repl_auto_review_required = auto_review_required | |
| }) | |
| .with_config(move |config| { | |
| config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest); | |
| config.approvals_reviewer = ApprovalsReviewer::AutoReview; | |
| if reviewer_constraint.is_some_and(|value| value != "large_prompt") || check_detail { | |
| let reviewer = config | |
| .model_catalog | |
| .as_mut() | |
| .expect("bundled model catalog") | |
| .models | |
| .iter_mut() | |
| .find(|model| model.slug == "gpt-5.6-luna") | |
| .expect("API-key Guardian reviewer"); | |
| if reviewer_compaction { | |
| reviewer.context_window = Some(100_000); | |
| reviewer.max_context_window = Some(100_000); | |
| reviewer.auto_compact_token_limit = Some(50_000); | |
| } else if check_detail { | |
| reviewer.use_responses_lite = false; | |
| } else if reviewer_constraint == Some("unsupported") { | |
| reviewer.input_modalities = | |
| vec![codex_protocol::openai_models::InputModality::Text]; | |
| } else { | |
| // Fit the required text and request-only prefix, while leaving | |
| // insufficient room for the additional image reservation. | |
| reviewer.context_window = | |
| (reviewer_constraint == Some("small")).then_some(20_000); | |
| reviewer.max_context_window = reviewer.context_window; | |
| } | |
| } | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("enable Code Mode"); | |
| if reviewer_compaction { | |
| config | |
| .features | |
| .enable(Feature::TokenBudget) | |
| .expect("enable token budget"); | |
| } | |
| config | |
| .features | |
| .set_enabled( | |
| Feature::GuardianEnhancedNodeReplTranscripts, | |
| enhanced_transcripts, | |
| ) | |
| .expect("configure enhanced transcripts"); | |
| config | |
| .features | |
| .set_enabled(Feature::GuardianNodeReplTranscriptImages, transcript_images) | |
| .expect("configure Guardian transcript images"); | |
| let mcp: McpServerConfig = serde_json::from_value(serde_json::json!({ | |
| "command": mcp_server_bin, | |
| "environment_id": remote_aware_environment_id(), | |
| "cwd": config.cwd, | |
| "env": { | |
| "MCP_TEST_ENABLE_NODE_REPL_JS": "1", | |
| "MCP_TEST_IMAGE_DATA_URL": format!("data:image/png;base64,{}", BASE64_STANDARD.encode(large_image.into_inner())), | |
| "MCP_TEST_OVERSIZED_INVALID_IMAGE": u8::from(!core_test_support::is_remote_test_environment()).to_string() | |
| }, | |
| "omit_tools_from": ["deferred"], | |
| })) | |
| .expect("valid MCP server config"); | |
| config | |
| .mcp_servers | |
| .set( | |
| [repl_server.to_owned(), format!("{repl_server}_")] | |
| .into_iter() | |
| .map(|name| (name, mcp.clone())) | |
| .collect(), | |
| ) | |
| .expect("configure MCP servers"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| wait_for_mcp_server(&test.codex, repl_server).await?; | |
| let images_enabled = auto_review_required || (enhanced_transcripts && transcript_images); | |
| let reviewer_images = images_enabled && (reviewer_constraint.is_none() || reviewer_compaction); | |
| let snapshot_padding = if images_enabled && reviewer_constraint != Some("large_prompt") { | |
| 2_500 | |
| } else { | |
| 10_000 | |
| }; | |
| let padding = "a".repeat(snapshot_padding); | |
| let snapshot_args = |marker| { | |
| let padding = if check_detail && marker == DIRECT_NODE_REPL_MIDDLE { | |
| "" | |
| } else { | |
| padding.as_str() | |
| }; | |
| serde_json::json!({ | |
| "message": format!("{padding}{marker}{padding}"), | |
| }) | |
| .to_string() | |
| }; | |
| let code = r#" | |
| await tools.mcp__node_repl_echo({ message: "guardian-hidden-unrelated-result" }); | |
| await tools.mcp__node_repl__js({ code: 'nodeRepl.fail()' }); | |
| await tools.mcp__node_repl__js({ code: `nodeRepl.write(${JSON.stringify("a".repeat(SNAPSHOT_PADDING) + ["guardian-visible", "dom-middle"].join("-") + "b".repeat(SNAPSHOT_PADDING))})` }); | |
| await tools.mcp__node_repl__echo({ message: ["guardian-visible-other-", "tool-result"].join("") }); | |
| await tools.mcp__node_repl__encrypted_output({}); | |
| await tools.mcp__node_repl__js({ code: 'nodeRepl.empty()' }); | |
| await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' }); | |
| if (LARGE_IMAGE) await tools.mcp__node_repl__image_scenario({ scenario: "invalid_image_bytes_then_image" }); | |
| await tools.exec_command({ cmd: "true", sandbox_permissions: "require_escalated", justification: "review" }); | |
| if (!REVIEWER_COMPACTION) await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' }); | |
| if (LARGE_IMAGE) await tools.mcp__node_repl__image({}); | |
| await tools.exec_command({ cmd: "printf second", sandbox_permissions: "require_escalated", justification: "review again" }); | |
| "# | |
| .replace("node_repl", repl_server) | |
| .replace("SNAPSHOT_PADDING", &snapshot_padding.to_string()) | |
| .replace("REVIEWER_COMPACTION", &reviewer_compaction.to_string()) | |
| .replace("LARGE_IMAGE", &check_detail.to_string()); | |
| let response_mock = responses::mount_sse_sequence(&server, { | |
| let mut response_bodies = vec![ | |
| sse(vec![ | |
| responses::ev_function_call_with_namespace( | |
| "node-repl-call", | |
| &format!("mcp__{repl_server}"), | |
| "echo", | |
| &snapshot_args(DIRECT_NODE_REPL_MIDDLE), | |
| ), | |
| ev_completed("resp-node-repl"), | |
| ]), | |
| sse(vec![ | |
| responses::ev_function_call_with_namespace( | |
| "unrelated-call", | |
| &format!("mcp__{repl_server}_"), | |
| "echo", | |
| &snapshot_args(DIRECT_UNRELATED_MIDDLE), | |
| ), | |
| ev_completed("resp-unrelated"), | |
| ]), | |
| sse(vec![ | |
| ev_custom_tool_call("code-mode-call", "exec", &code), | |
| ev_completed("resp-parent"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("guardian", r#"{"outcome":"allow"}"#), | |
| if reviewer_compaction { | |
| responses::ev_completed_with_tokens( | |
| "resp-guardian", | |
| /*total_tokens*/ 70_000, | |
| ) | |
| } else { | |
| ev_completed("resp-guardian") | |
| }, | |
| ]), | |
| ]; | |
| if reviewer_compaction { | |
| response_bodies.push(sse(vec![ | |
| serde_json::json!({ | |
| "type": "response.output_item.done", | |
| "item": {"type": "compaction", "encrypted_content": COMPACTION_SUMMARY}, | |
| }), | |
| ev_completed("resp-guardian-compact"), | |
| ])); | |
| } | |
| response_bodies.extend([ | |
| sse(vec![ | |
| ev_assistant_message("guardian-again", r#"{"outcome":"allow"}"#), | |
| ev_completed("resp-guardian-again"), | |
| ]), | |
| sse(vec![ev_completed("resp-done")]), | |
| ]); | |
| response_bodies | |
| }) | |
| .await; | |
| test.submit_text_turn(&format!("review a nested {repl_server} tool response")) | |
| .await?; | |
| let requests = response_mock.requests(); | |
| let guardian_requests = requests | |
| .iter() | |
| .filter(|request| { | |
| request.body_json()["client_metadata"]["x-openai-subagent"].as_str() == Some("guardian") | |
| && request.inputs_of_type("compaction_trigger").is_empty() | |
| }) | |
| .collect::<Vec<_>>(); | |
| assert_eq!(guardian_requests.len(), 2); | |
| let guardian_text = guardian_requests[0].message_input_texts("user").concat(); | |
| let evidence_enabled = enhanced_transcripts || auto_review_required; | |
| assert_eq!( | |
| guardian_text.contains(&format!("tool={repl_server}.js")), | |
| evidence_enabled, | |
| "private evidence must identify its originating REPL server" | |
| ); | |
| for included in [ | |
| DIRECT_NODE_REPL_MIDDLE, | |
| NODE_REPL_DOM_MIDDLE, | |
| OTHER_NODE_REPL_RESULT, | |
| "Lookup completed", | |
| "<completed without visible text>", | |
| "guardian-visible-before-image", | |
| "guardian-visible-after-image", | |
| ] { | |
| assert_eq!( | |
| guardian_text.contains(included), | |
| evidence_enabled, | |
| "unexpected Guardian evidence visibility for {included}" | |
| ); | |
| } | |
| let guardian_request = guardian_requests[0]; | |
| let reviewer_image_urls = guardian_request.message_input_image_urls("user"); | |
| let expected_images = usize::from(reviewer_images) + usize::from(check_detail); | |
| assert_eq!(reviewer_image_urls.len(), expected_images); | |
| if reviewer_images { | |
| assert_eq!(reviewer_image_urls[0], PRIVATE_IMAGE); | |
| let reviewer_user_content = guardian_request | |
| .inputs_of_type("message") | |
| .into_iter() | |
| .filter(|item| item.get("role").and_then(Value::as_str) == Some("user")) | |
| .filter_map(|item| item.get("content").and_then(Value::as_array).cloned()) | |
| .flatten() | |
| .collect::<Vec<_>>(); | |
| let image_index = reviewer_user_content | |
| .iter() | |
| .position(|item| item.get("image_url").and_then(Value::as_str) == Some(PRIVATE_IMAGE)) | |
| .expect("private reviewer image should be present"); | |
| if check_detail { | |
| assert_eq!(reviewer_user_content[image_index]["detail"], "high"); | |
| let payload = reviewer_image_urls[1].split_once(',').unwrap().1; | |
| let dimensions = | |
| image::load_from_memory(&BASE64_STANDARD.decode(payload)?)?.dimensions(); | |
| assert_eq!(dimensions, (2048, 32)); | |
| } | |
| for (index, marker) in [(image_index - 1, "before"), (image_index + 1, "after")] { | |
| let text = reviewer_user_content[index]["text"].as_str().unwrap(); | |
| assert!(text.contains(marker)); | |
| } | |
| } | |
| for excluded in [ | |
| DIRECT_UNRELATED_MIDDLE, | |
| UNRELATED_RESULT, | |
| "guardian-hidden-failed-result", | |
| "gAAAA-test", | |
| "guardian-hidden-ui-preview", | |
| "guardian-hidden-structured-override", | |
| ] { | |
| assert!( | |
| !guardian_text.contains(excluded), | |
| "protected, failed, unrelated, or duplicate evidence leaked: {excluded}" | |
| ); | |
| } | |
| assert_eq!( | |
| guardian_requests[1] | |
| .message_input_texts("user") | |
| .concat() | |
| .matches(NODE_REPL_DOM_MIDDLE) | |
| .count(), | |
| usize::from(evidence_enabled), | |
| "a reused Guardian session must not append the same evidence twice" | |
| ); | |
| assert_eq!( | |
| guardian_requests[1].message_input_image_urls("user"), | |
| if reviewer_constraint == Some("large_prompt") { | |
| vec![PRIVATE_IMAGE.to_string()] | |
| } else { | |
| reviewer_image_urls | |
| } | |
| ); | |
| if reviewer_compaction { | |
| let compact_requests = requests | |
| .iter() | |
| .filter(|request| !request.inputs_of_type("compaction_trigger").is_empty()) | |
| .collect::<Vec<_>>(); | |
| assert_eq!(compact_requests.len(), 1); | |
| let compact_request = compact_requests[0]; | |
| assert!( | |
| compact_request | |
| .message_input_texts("user") | |
| .concat() | |
| .contains(NODE_REPL_DOM_MIDDLE) | |
| ); | |
| assert_eq!( | |
| guardian_requests[1].inputs_of_type("compaction")[0]["encrypted_content"], | |
| COMPACTION_SUMMARY | |
| ); | |
| for request in &guardian_requests { | |
| assert!(!request.has_content_kinds(&["token_budget.context_window"])); | |
| } | |
| } | |
| let parent_request = requests.last().expect("parent turn should complete"); | |
| let parent_input = serde_json::to_string(&parent_request.input())?; | |
| assert!(!parent_input.contains("data:image/png;base64,")); | |
| assert!(!parent_input.contains("guardian-visible-before-image")); | |
| assert!(!parent_input.contains("guardian-visible-after-image")); | |
| assert!( | |
| !parent_input.contains(NODE_REPL_DOM_MIDDLE) | |
| && !parent_input.contains(OTHER_NODE_REPL_RESULT) | |
| && !parent_input.contains(UNRELATED_RESULT), | |
| "nested MCP responses leaked into parent model history: {parent_input}" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_only_can_call_mcp_tool() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const result = await tools.mcp__rmcp__echo({ message: "ping" }); | |
| text(`echo=${result.structuredContent?.echo ?? "missing"}`); | |
| "#; | |
| let (_test, second_mock) = run_code_mode_turn_with_rmcp_mode( | |
| &server, | |
| "use exec to run the rmcp echo tool in code mode only", | |
| code, | |
| /*code_mode_only*/ true, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "code_mode_only rmcp tool call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!(output, "echo=ECHOING: ping"); | |
| Ok(()) | |
| } | |
| async fn code_mode_exposes_mcp_tools_on_global_tools_object() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({ | |
| message: "ping", | |
| }); | |
| text( | |
| `hasEcho=${String(Object.keys(tools).includes("mcp__rmcp__echo"))}\n` + | |
| `echoType=${typeof tools.mcp__rmcp__echo}\n` + | |
| `echo=${structuredContent?.echo ?? "missing"}\n` + | |
| `isError=${String(isError)}\n` + | |
| `contentLength=${content.length}` | |
| ); | |
| "#; | |
| let (_test, second_mock) = | |
| run_code_mode_turn_with_rmcp(&server, "use exec to inspect the global tools object", code) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec global rmcp access failed unexpectedly: {output}" | |
| ); | |
| assert_eq!( | |
| output, | |
| "hasEcho=true | |
| echoType=function | |
| echo=ECHOING: ping | |
| isError=false | |
| contentLength=0" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_uses_non_prefixed_mcp_tool_names_when_feature_enabled() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const result = await tools.rmcp__echo({ message: "ping" }); | |
| text(JSON.stringify({ | |
| hasNonPrefixedEcho: typeof tools.rmcp__echo === "function", | |
| hasPrefixedEcho: typeof tools.mcp__rmcp__echo === "function", | |
| echo: result.structuredContent?.echo ?? "missing", | |
| })); | |
| "#; | |
| let (_test, second_mock) = run_code_mode_turn_with_rmcp_config( | |
| &server, | |
| "use exec to inspect non-prefixed MCP names", | |
| code, | |
| "test-gpt-5.1-codex", | |
| /*code_mode_only*/ false, | |
| /*non_prefixed_mcp_tool_names*/ true, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec non-prefixed rmcp access failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str(&output)?; | |
| assert_eq!( | |
| parsed, | |
| serde_json::json!({ | |
| "hasNonPrefixedEcho": true, | |
| "hasPrefixedEcho": false, | |
| "echo": "ECHOING: ping", | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| struct NamespacedCustomTool { | |
| generation: usize, | |
| generations: Arc<AtomicUsize>, | |
| } | |
| impl ToolContributor for NamespacedCustomTool { | |
| fn tools( | |
| &self, | |
| _session_store: &ExtensionData, | |
| _thread_store: &ExtensionData, | |
| ) -> Vec<Arc<dyn for<'call> ToolExecutor<ToolCall<'call>>>> { | |
| Vec::new() | |
| } | |
| fn tools_for_step( | |
| &self, | |
| _session_store: &ExtensionData, | |
| _thread_store: &ExtensionData, | |
| _step_store: &ExtensionData, | |
| ) -> Vec<Arc<dyn for<'call> ToolExecutor<ToolCall<'call>>>> { | |
| vec![Arc::new(Self { | |
| generation: self.generations.fetch_add(1, Ordering::Relaxed) + 1, | |
| generations: Arc::clone(&self.generations), | |
| })] | |
| } | |
| } | |
| impl<'call> ToolExecutor<ToolCall<'call>> for NamespacedCustomTool { | |
| fn tool_name(&self) -> ToolName { | |
| ToolName::namespaced("editor", "apply_patch") | |
| } | |
| fn spec(&self) -> ToolSpec { | |
| ToolSpec::Namespace(ResponsesApiNamespace { | |
| name: "editor".to_string(), | |
| description: "Editing tools.".to_string(), | |
| tools: vec![ResponsesApiNamespaceTool::Custom(FreeformTool { | |
| name: "apply_patch".to_string(), | |
| description: format!("Apply a raw editor patch (step {}).", self.generation), | |
| defer_loading: None, | |
| format: FreeformToolFormat { | |
| r#type: "grammar".to_string(), | |
| syntax: "lark".to_string(), | |
| definition: "start: /.+/".to_string(), | |
| }, | |
| })], | |
| }) | |
| } | |
| fn handle<'a>(&'a self, call: ToolCall<'call>) -> ToolExecutorFuture<'a> | |
| where | |
| 'call: 'a, | |
| { | |
| Box::pin(async move { | |
| let ToolPayload::Custom { input } = call.payload else { | |
| return Err(FunctionCallError::Fatal( | |
| "expected custom tool payload".to_string(), | |
| )); | |
| }; | |
| Ok(Box::new(JsonToolOutput::new(serde_json::json!({ | |
| "namespace": call.tool_name.namespace, | |
| "name": call.tool_name.name, | |
| "input": input, | |
| "generation": self.generation, | |
| }))) as Box<dyn ToolOutput>) | |
| }) | |
| } | |
| } | |
| async fn yielded_code_mode_tool_callbacks_keep_their_originating_step() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut extensions = ExtensionRegistryBuilder::<Config>::new(); | |
| extensions.tool_contributor(Arc::new(NamespacedCustomTool { | |
| generation: 0, | |
| generations: Arc::new(AtomicUsize::new(0)), | |
| })); | |
| let mut builder = test_codex() | |
| .with_extensions(Arc::new(extensions.build())) | |
| .with_model_info_override("gpt-5.4", |model| { | |
| model.tool_mode = Some(ToolMode::CodeMode); | |
| model.experimental_supported_tools = vec!["test_sync_tool".to_string()]; | |
| }) | |
| .with_config(|config| { | |
| config.features.enable(Feature::CodeMode).unwrap(); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| // A cannot invoke its editor until B is executing in the next turn. | |
| let barrier = r#"await tools.test_sync_tool({barrier: { | |
| id: "code-mode-origin-turns", participants: 2, timeout_ms: 60000 | |
| }});"#; | |
| responses::mount_sse_once(&server, sse(vec![ | |
| ev_response_created("resp-a"), | |
| ev_custom_tool_call("call-a", "exec", &format!( | |
| "// @exec: {{\"yield_time_ms\": 1}}\n{barrier}\ntext('A_ORIGIN:' + (await tools.editor__apply_patch('A')).generation);" | |
| )), | |
| ev_completed("resp-a"), | |
| ])).await; | |
| let yielded = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-a", "waiting for the next turn"), | |
| ev_completed("resp-a-done"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("start A and leave it running").await?; | |
| let cell_id = extract_running_cell_id(text_item( | |
| &custom_tool_output_items(&yielded.single_request(), "call-a"), | |
| /*index*/ 0, | |
| )); | |
| responses::mount_sse_once(&server, sse(vec![ | |
| ev_response_created("resp-b"), | |
| ev_custom_tool_call("call-b", "exec", &format!( | |
| "// @exec: {{\"yield_time_ms\": 60000}}\n{barrier}\ntext('B_ORIGIN:' + (await tools.editor__apply_patch('B')).generation);" | |
| )), | |
| ev_completed("resp-b"), | |
| ])).await; | |
| let b_finished = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-wait"), | |
| responses::ev_function_call( | |
| "call-wait-a", | |
| "wait", | |
| &serde_json::json!({ | |
| "cell_id": cell_id, "yield_time_ms": 60000, | |
| }) | |
| .to_string(), | |
| ), | |
| ev_completed("resp-wait"), | |
| ]), | |
| ) | |
| .await; | |
| let completed = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-b", "both cells finished"), | |
| ev_completed("resp-b-done"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("run B, then wait for A").await?; | |
| assert_eq!( | |
| custom_tool_output_body_and_success(&b_finished.single_request(), "call-b").0, | |
| "B_ORIGIN:3" | |
| ); | |
| let live = completed.single_request(); | |
| let a_output = function_tool_output_items(&live, "call-wait-a"); | |
| assert!(text_item(&a_output, /*index*/ 0).starts_with("Script completed")); | |
| assert_eq!(text_item(&a_output, /*index*/ 1), "A_ORIGIN:1"); | |
| Ok(()) | |
| } | |
| async fn code_mode_exposes_and_dispatches_namespaced_custom_tools() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut extensions = ExtensionRegistryBuilder::<Config>::new(); | |
| extensions.tool_contributor(Arc::new(NamespacedCustomTool { | |
| generation: 0, | |
| generations: Arc::new(AtomicUsize::new(0)), | |
| })); | |
| let mut builder = test_codex() | |
| .with_model("test-gpt-5.1-codex") | |
| .with_extensions(Arc::new(extensions.build())) | |
| .with_config(|config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| let code = r#" | |
| const tool = ALL_TOOLS.find(({ name }) => name === "editor__apply_patch"); | |
| const result = await tools.editor__apply_patch("nested patch"); | |
| text(JSON.stringify({ | |
| name: tool?.name ?? null, | |
| description: tool?.description ?? null, | |
| result, | |
| })); | |
| "#; | |
| let response_mock = responses::mount_sse_sequence( | |
| &server, | |
| vec![ | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| responses::ev_custom_tool_call_with_namespace( | |
| "call-direct", | |
| "editor", | |
| "apply_patch", | |
| "direct patch", | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| sse(vec![ | |
| ev_response_created("resp-2"), | |
| ev_custom_tool_call("call-exec", "exec", code), | |
| ev_completed("resp-2"), | |
| ]), | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-3"), | |
| ]), | |
| ], | |
| ) | |
| .await; | |
| test.submit_turn("call the namespaced custom editor tool directly and through exec") | |
| .await?; | |
| let requests = response_mock.requests(); | |
| assert_eq!(requests.len(), 3); | |
| let declaration = | |
| "declare const tools: { editor__apply_patch(input: string): Promise<unknown>; };"; | |
| let description = |generation| { | |
| format!( | |
| "Apply a raw editor patch (step {generation}).\n\nexec tool declaration:\n```ts\n{declaration}\n```" | |
| ) | |
| }; | |
| let first_body = requests[0].body_json(); | |
| let namespaced_custom_tool = namespace_child_tool(&first_body, "editor", "apply_patch") | |
| .expect("namespaced custom tool should be included in the model request"); | |
| assert_eq!( | |
| namespaced_custom_tool, | |
| &serde_json::json!({ | |
| "type": "custom", | |
| "name": "apply_patch", | |
| "description": description(1), | |
| "format": { | |
| "type": "grammar", | |
| "syntax": "lark", | |
| "definition": "start: /.+/", | |
| }, | |
| }) | |
| ); | |
| let second_body = requests[1].body_json(); | |
| let second_description = description(2); | |
| assert_eq!( | |
| namespace_child_tool(&second_body, "editor", "apply_patch") | |
| .and_then(|tool| tool.get("description")) | |
| .and_then(Value::as_str), | |
| Some(second_description.as_str()) | |
| ); | |
| let (direct_output, direct_success) = | |
| custom_tool_output_body_and_success(&requests[1], "call-direct"); | |
| assert_ne!(direct_success, Some(false)); | |
| let direct_output = serde_json::from_str::<Value>(&direct_output).unwrap_or_else(|error| { | |
| panic!("invalid direct custom tool output `{direct_output}`: {error}") | |
| }); | |
| assert_eq!( | |
| direct_output, | |
| serde_json::json!({ | |
| "namespace": "editor", | |
| "name": "apply_patch", | |
| "input": "direct patch", | |
| "generation": 1, | |
| }) | |
| ); | |
| let (exec_output, exec_success) = | |
| custom_tool_output_body_and_success(&requests[2], "call-exec"); | |
| assert_ne!(exec_success, Some(false)); | |
| let exec_output = serde_json::from_str::<Value>(&exec_output).unwrap_or_else(|error| { | |
| panic!("invalid code mode custom tool output `{exec_output}`: {error}") | |
| }); | |
| assert_eq!( | |
| exec_output, | |
| serde_json::json!({ | |
| "name": "editor__apply_patch", | |
| "description": format!("Editing tools.\n\n{second_description}"), | |
| "result": { | |
| "namespace": "editor", | |
| "name": "apply_patch", | |
| "input": "nested patch", | |
| "generation": 2, | |
| }, | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exposes_namespaced_mcp_tools_on_global_tools_object() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| text(JSON.stringify({ | |
| hasExecCommand: typeof tools.exec_command === "function", | |
| hasNamespacedEcho: typeof tools.mcp__rmcp__echo === "function", | |
| })); | |
| "#; | |
| let (_test, second_mock) = | |
| run_code_mode_turn_with_rmcp(&server, "use exec to inspect the global tools object", code) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec global tools inspection failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str(&output)?; | |
| assert_eq!( | |
| parsed, | |
| serde_json::json!({ | |
| "hasExecCommand": true, | |
| "hasNamespacedEcho": true, | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exposes_normalized_illegal_mcp_tool_names() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const result = await tools.mcp__rmcp__echo_tool({ message: "ping" }); | |
| text(`echo=${result.structuredContent.echo}`); | |
| "#; | |
| let (_test, second_mock) = run_code_mode_turn_with_rmcp( | |
| &server, | |
| "use exec to call a normalized rmcp tool name", | |
| code, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec normalized rmcp tool call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!(output, "echo=ECHOING: ping"); | |
| Ok(()) | |
| } | |
| async fn code_mode_exports_all_tools_metadata_for_builtin_tools() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const tool = ALL_TOOLS.find(({ name }) => name === "view_image"); | |
| text(JSON.stringify(tool)); | |
| "#; | |
| let (_test, second_mock) = | |
| run_code_mode_turn(&server, "use exec to inspect ALL_TOOLS", code).await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec ALL_TOOLS lookup failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str( | |
| &custom_tool_output_last_non_empty_text(&req, "call-1") | |
| .expect("exec ALL_TOOLS lookup should emit JSON"), | |
| )?; | |
| assert_eq!( | |
| parsed, | |
| serde_json::json!({ | |
| "name": "view_image", | |
| "description": "View a local image file from the filesystem when visual inspection is needed. Use this for images already available on disk.\n\nexec tool declaration:\n```ts\ndeclare const tools: { view_image(args: {\n // Local filesystem path to an image file.\n path: string;\n}): Promise<{\n // Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved.\n detail: \"high\" | \"original\";\n // Data URL for the loaded image.\n image_url: string;\n}>; };\n```", | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_exports_all_tools_metadata_for_namespaced_mcp_tools() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const tool = ALL_TOOLS.find( | |
| ({ name }) => name === "mcp__rmcp__echo" | |
| ); | |
| text(JSON.stringify(tool)); | |
| "#; | |
| let (_test, second_mock) = | |
| run_code_mode_turn_with_rmcp(&server, "use exec to inspect ALL_TOOLS", code).await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec ALL_TOOLS MCP lookup failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str( | |
| &custom_tool_output_last_non_empty_text(&req, "call-1") | |
| .expect("exec ALL_TOOLS MCP lookup should emit JSON"), | |
| )?; | |
| assert_eq!( | |
| parsed, | |
| serde_json::json!({ | |
| "name": "mcp__rmcp__echo", | |
| "description": concat!( | |
| "Use these tools to exercise the rmcp test server.\n\n", | |
| "Echo back the provided message and include environment data.\n\n", | |
| "exec tool declaration:\n", | |
| "```ts\n", | |
| "declare const tools: { mcp__rmcp__echo(args: { env_var?: string; message: string; }): ", | |
| "Promise<CallToolResult<{ echo: string; env: string | null; }>>; };\n", | |
| "```", | |
| ), | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_uses_the_first_dynamic_tool_for_a_normalized_name() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| for use_responses_lite in [false, true] { | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.5", move |model_info| { | |
| model_info.use_responses_lite = use_responses_lite; | |
| model_info.tool_mode = Some(ToolMode::CodeMode); | |
| }) | |
| .with_config(|config| { | |
| config.tool_registry.turn_metadata_includes_tool_info = true; | |
| config | |
| .features | |
| .enable(Feature::CodeMode) | |
| .expect("code mode should be enabled"); | |
| }); | |
| let base_test = builder.build_with_auto_env(&server).await?; | |
| let new_thread = base_test | |
| .thread_manager | |
| .start_thread(StartThreadOptions { | |
| dynamic_tools: [ | |
| ("foo-bar", "First normalized dynamic tool."), | |
| ("foo_bar", "Shadowed normalized dynamic tool."), | |
| ] | |
| .into_iter() | |
| .map(|(name, description)| { | |
| DynamicToolSpec::Function(DynamicToolFunctionSpec { | |
| name: name.to_string(), | |
| description: description.to_string(), | |
| input_schema: serde_json::json!({ | |
| "type": "object", | |
| "properties": {}, | |
| "additionalProperties": false, | |
| }), | |
| defer_loading: false, | |
| }) | |
| }) | |
| .collect(), | |
| ..StartThreadOptions::new(base_test.config.clone()) | |
| }) | |
| .await?; | |
| let mut test = base_test; | |
| test.codex = new_thread.thread; | |
| test.session_configured = new_thread.session_configured; | |
| let first_response = if use_responses_lite { | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| r#" | |
| const matches = ALL_TOOLS.filter(({ name }) => name === "foo_bar"); | |
| const output = await tools.foo_bar({}); | |
| text(JSON.stringify({ | |
| count: matches.length, | |
| name: matches[0]?.name ?? null, | |
| description: matches[0]?.description ?? null, | |
| output, | |
| })); | |
| "#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]) | |
| } else { | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-1"), | |
| ]) | |
| }; | |
| let first_mock = responses::mount_sse_once(&server, first_response).await; | |
| let second_mock = if use_responses_lite { | |
| Some( | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await, | |
| ) | |
| } else { | |
| None | |
| }; | |
| let cwd = test.config.cwd.clone(); | |
| let (sandbox_policy, permission_profile) = | |
| turn_permission_fields(PermissionProfile::Disabled, cwd.as_path()); | |
| test.codex | |
| .start_or_steer_turn( | |
| TurnInputRequest::user_input(vec![UserInput::Text { | |
| text: "inspect and call normalized dynamic tools".to_string(), | |
| text_elements: Vec::new(), | |
| }]) | |
| .with_thread_settings(ThreadSettingsOverrides { | |
| environments: Some(codex_protocol::protocol::TurnEnvironmentSelections::new( | |
| cwd, | |
| Vec::new(), | |
| )), | |
| approval_policy: Some(AskForApproval::Never), | |
| sandbox_policy: Some(sandbox_policy), | |
| permission_profile, | |
| collaboration_mode: Some(CollaborationMode { | |
| mode: ModeKind::Default, | |
| settings: Settings { | |
| model: test.session_configured.model.clone(), | |
| reasoning_effort: None, | |
| developer_instructions: None, | |
| }, | |
| }), | |
| ..Default::default() | |
| }), | |
| ) | |
| .await?; | |
| let turn_id = wait_for_event_match(&test.codex, |event| match event { | |
| EventMsg::TurnStarted(event) => Some(event.turn_id.clone()), | |
| _ => None, | |
| }) | |
| .await; | |
| if use_responses_lite { | |
| let request = wait_for_event_match(&test.codex, |event| match event { | |
| EventMsg::DynamicToolCallRequest(request) => Some(request.clone()), | |
| _ => None, | |
| }) | |
| .await; | |
| assert_eq!(request.namespace, None); | |
| assert_eq!(request.tool, "foo-bar"); | |
| assert_eq!(request.arguments, serde_json::json!({})); | |
| test.codex | |
| .submit(Op::DynamicToolResponse { | |
| id: request.call_id, | |
| response: DynamicToolResponse { | |
| content_items: vec![DynamicToolCallOutputContentItem::InputText { | |
| text: "first-winner".to_string(), | |
| }], | |
| success: true, | |
| }, | |
| }) | |
| .await?; | |
| } | |
| wait_for_event(&test.codex, |event| match event { | |
| EventMsg::TurnComplete(event) => event.turn_id == turn_id, | |
| _ => false, | |
| }) | |
| .await; | |
| let first_body = first_mock.single_request().body_json(); | |
| let model_tools = if use_responses_lite { | |
| first_body["input"] | |
| .as_array() | |
| .and_then(|input| { | |
| input.iter().find(|item| { | |
| item.get("type").and_then(Value::as_str) == Some("additional_tools") | |
| }) | |
| }) | |
| .and_then(|item| item["tools"].as_array()) | |
| .and_then(|tools| { | |
| tools.iter().find(|tool| { | |
| tool.get("type").and_then(Value::as_str) == Some("namespace") | |
| && tool.get("name").and_then(Value::as_str) == Some("functions") | |
| }) | |
| }) | |
| .and_then(|namespace| namespace["tools"].as_array()) | |
| .expect("the Responses Lite request should contain its default-namespace tools") | |
| } else { | |
| first_body["tools"] | |
| .as_array() | |
| .expect("the Responses request should contain its visible tools") | |
| }; | |
| let visible_dynamic_tools = model_tools | |
| .iter() | |
| .filter(|tool| matches!(tool["name"].as_str(), Some("foo-bar" | "foo_bar"))) | |
| .map(|tool| { | |
| ( | |
| tool["name"] | |
| .as_str() | |
| .expect("dynamic tools should have a name"), | |
| tool["description"] | |
| .as_str() | |
| .expect("dynamic tools should have a description"), | |
| ) | |
| }) | |
| .collect::<Vec<_>>(); | |
| assert_eq!( | |
| visible_dynamic_tools, | |
| [ | |
| ( | |
| "foo-bar", | |
| concat!( | |
| "First normalized dynamic tool.\n\n", | |
| "exec tool declaration:\n", | |
| "```ts\n", | |
| "declare const tools: { foo_bar(args: {}): Promise<unknown>; };\n", | |
| "```", | |
| ), | |
| ), | |
| ("foo_bar", "Shadowed normalized dynamic tool."), | |
| ] | |
| ); | |
| if use_responses_lite { | |
| let metadata: Value = serde_json::from_str( | |
| first_body["client_metadata"]["x-codex-turn-metadata"] | |
| .as_str() | |
| .expect("Responses Lite should contain serialized turn metadata"), | |
| )?; | |
| let functions = &metadata["tool_namespaces_info"]["functions"]["functions"]; | |
| assert_eq!(functions["foo-bar"]["code_mode_name"], "foo_bar"); | |
| assert!(functions["foo_bar"]["code_mode_name"].is_null()); | |
| } | |
| let exec_description = model_tools | |
| .iter() | |
| .find(|tool| tool["name"] == "exec") | |
| .and_then(|tool| tool["description"].as_str()) | |
| .expect("the model request should contain the code-mode exec tool"); | |
| assert!(!exec_description.contains("First normalized dynamic tool.")); | |
| assert!(!exec_description.contains("Shadowed normalized dynamic tool.")); | |
| if let Some(second_mock) = second_mock { | |
| let request = second_mock.single_request(); | |
| let output = custom_tool_output_last_non_empty_text(&request, "call-1") | |
| .expect("code mode should return normalized tool metadata"); | |
| let result: Value = serde_json::from_str(&output)?; | |
| assert_eq!(result["count"], serde_json::json!(1)); | |
| assert_eq!(result["name"], serde_json::json!("foo_bar")); | |
| assert_eq!(result["output"], serde_json::json!("first-winner")); | |
| let description = result["description"] | |
| .as_str() | |
| .expect("the winning tool should have a description"); | |
| assert!(description.contains("First normalized dynamic tool.")); | |
| assert!(!description.contains("Shadowed normalized dynamic tool.")); | |
| } | |
| } | |
| Ok(()) | |
| } | |
| async fn code_mode_renders_local_refs_in_outbound_exec_description() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let response = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let mut builder = test_codex().with_config(|config| { | |
| config | |
| .features | |
| .enable(Feature::CodeModeOnly) | |
| .expect("code mode only should be enabled"); | |
| }); | |
| let base_test = builder.build_with_auto_env(&server).await?; | |
| let new_thread = base_test | |
| .thread_manager | |
| .start_thread(StartThreadOptions { | |
| dynamic_tools: vec![DynamicToolSpec::Function(DynamicToolFunctionSpec { | |
| name: "boolean_search".to_string(), | |
| description: "Search with nested Boolean clauses.".to_string(), | |
| input_schema: serde_json::json!({ | |
| "type": "object", | |
| "properties": { | |
| "clauses": { | |
| "type": "array", | |
| "items": { "$ref": "#/$defs/Clause" } | |
| } | |
| }, | |
| "$defs": { | |
| "Clause": { | |
| "type": "object", | |
| "properties": { | |
| "query": { | |
| "oneOf": [ | |
| { "type": "string" }, | |
| { | |
| "type": "object", | |
| "properties": { | |
| "clauses": { | |
| "type": "array", | |
| "items": { "$ref": "#/$defs/Clause" } | |
| } | |
| } | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| }), | |
| defer_loading: false, | |
| })], | |
| ..StartThreadOptions::new(base_test.config.clone()) | |
| }) | |
| .await?; | |
| let mut test = base_test; | |
| test.codex = new_thread.thread; | |
| test.session_configured = new_thread.session_configured; | |
| test.submit_turn("inspect the tool schema").await?; | |
| let body = response.single_request().body_json(); | |
| let exec_description = body["tools"] | |
| .as_array() | |
| .expect("request should contain tools") | |
| .iter() | |
| .find_map(|tool| { | |
| (tool["name"].as_str() == Some("exec")) | |
| .then(|| tool["description"].as_str()) | |
| .flatten() | |
| }) | |
| .expect("Code Mode exec should remain available"); | |
| assert!(exec_description.contains("query?: string | { clauses?: Array<")); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_call_hidden_dynamic_tools() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let base_test = builder.build(&server).await?; | |
| let new_thread = base_test | |
| .thread_manager | |
| .start_thread(StartThreadOptions { | |
| dynamic_tools: vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec { | |
| name: "codex_app".to_string(), | |
| description: "Codex app tools.".to_string(), | |
| tools: vec![DynamicToolNamespaceTool::Function( | |
| DynamicToolFunctionSpec { | |
| name: "hidden_dynamic_tool".to_string(), | |
| description: "A hidden dynamic tool.".to_string(), | |
| input_schema: serde_json::json!({ | |
| "type": "object", | |
| "properties": { | |
| "city": { "type": "string" } | |
| }, | |
| "required": ["city"], | |
| "additionalProperties": false, | |
| }), | |
| defer_loading: true, | |
| }, | |
| )], | |
| })], | |
| ..StartThreadOptions::new(base_test.config.clone()) | |
| }) | |
| .await?; | |
| let mut test = base_test; | |
| test.codex = new_thread.thread; | |
| test.session_configured = new_thread.session_configured; | |
| let code = r#" | |
| const tool = ALL_TOOLS.find(({ name }) => name === "codex_app__hidden_dynamic_tool"); | |
| const out = await tools.codex_app__hidden_dynamic_tool({ city: "Paris" }); | |
| text( | |
| JSON.stringify({ | |
| name: tool?.name ?? null, | |
| description: tool?.description ?? null, | |
| out, | |
| }) | |
| ); | |
| "#; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call("call-1", "exec", code), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| let cwd = test.config.cwd.clone(); | |
| let (sandbox_policy, permission_profile) = | |
| turn_permission_fields(PermissionProfile::Disabled, cwd.as_path()); | |
| test.codex | |
| .start_or_steer_turn( | |
| TurnInputRequest::user_input(vec![UserInput::Text { | |
| text: "use exec to inspect and call hidden tools".into(), | |
| text_elements: Vec::new(), | |
| }]) | |
| .with_thread_settings(ThreadSettingsOverrides { | |
| environments: Some(codex_protocol::protocol::TurnEnvironmentSelections::new( | |
| cwd, | |
| Vec::new(), | |
| )), | |
| approval_policy: Some(AskForApproval::Never), | |
| sandbox_policy: Some(sandbox_policy), | |
| permission_profile, | |
| collaboration_mode: Some(CollaborationMode { | |
| mode: ModeKind::Default, | |
| settings: Settings { | |
| model: test.session_configured.model.clone(), | |
| reasoning_effort: None, | |
| developer_instructions: None, | |
| }, | |
| }), | |
| ..Default::default() | |
| }), | |
| ) | |
| .await?; | |
| let turn_id = wait_for_event_match(&test.codex, |event| match event { | |
| EventMsg::TurnStarted(event) => Some(event.turn_id.clone()), | |
| _ => None, | |
| }) | |
| .await; | |
| let request = wait_for_event_match(&test.codex, |event| match event { | |
| EventMsg::DynamicToolCallRequest(request) => Some(request.clone()), | |
| _ => None, | |
| }) | |
| .await; | |
| assert_eq!(request.namespace.as_deref(), Some("codex_app")); | |
| assert_eq!(request.tool, "hidden_dynamic_tool"); | |
| assert_eq!(request.arguments, serde_json::json!({ "city": "Paris" })); | |
| test.codex | |
| .submit(Op::DynamicToolResponse { | |
| id: request.call_id, | |
| response: DynamicToolResponse { | |
| content_items: vec![DynamicToolCallOutputContentItem::InputText { | |
| text: "hidden-ok".to_string(), | |
| }], | |
| success: true, | |
| }, | |
| }) | |
| .await?; | |
| wait_for_event(&test.codex, |event| match event { | |
| EventMsg::TurnComplete(event) => event.turn_id == turn_id, | |
| _ => false, | |
| }) | |
| .await; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec hidden dynamic tool call failed unexpectedly: {output}" | |
| ); | |
| let parsed: Value = serde_json::from_str( | |
| &custom_tool_output_last_non_empty_text(&req, "call-1") | |
| .expect("exec hidden dynamic tool lookup should emit JSON"), | |
| )?; | |
| assert_eq!( | |
| parsed.get("name"), | |
| Some(&Value::String("codex_app__hidden_dynamic_tool".to_string())) | |
| ); | |
| assert_eq!( | |
| parsed.get("out"), | |
| Some(&Value::String("hidden-ok".to_string())) | |
| ); | |
| assert!( | |
| parsed | |
| .get("description") | |
| .and_then(Value::as_str) | |
| .is_some_and(|description| { | |
| description.contains("Codex app tools.") | |
| && description.contains("A hidden dynamic tool.") | |
| && description.contains("declare const tools:") | |
| && description.contains("codex_app__hidden_dynamic_tool(args:") | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_excludes_configured_nested_tool_namespaces() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(|config| { | |
| config.update_plan_enabled = true; | |
| let _ = config.features.enable(Feature::CodeMode); | |
| config.code_mode.excluded_tool_namespaces = vec!["excluded".to_string()]; | |
| }); | |
| let base_test = builder.build(&server).await?; | |
| let new_thread = base_test | |
| .thread_manager | |
| .start_thread(StartThreadOptions { | |
| dynamic_tools: vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec { | |
| name: "excluded".to_string(), | |
| description: "Excluded tools.".to_string(), | |
| tools: vec![DynamicToolNamespaceTool::Function( | |
| DynamicToolFunctionSpec { | |
| name: "lookup".to_string(), | |
| description: "An excluded dynamic tool.".to_string(), | |
| input_schema: serde_json::json!({ | |
| "type": "object", | |
| "properties": {}, | |
| "additionalProperties": false, | |
| }), | |
| defer_loading: false, | |
| }, | |
| )], | |
| })], | |
| ..StartThreadOptions::new(base_test.config.clone()) | |
| }) | |
| .await?; | |
| let mut test = base_test; | |
| test.codex = new_thread.thread; | |
| test.session_configured = new_thread.session_configured; | |
| let first_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| r#" | |
| text(JSON.stringify({ | |
| excludedType: typeof tools.excluded__lookup, | |
| excludedMetadata: ALL_TOOLS.some(({ name }) => name === "excluded__lookup"), | |
| allowedType: typeof tools.update_plan, | |
| allowedMetadata: ALL_TOOLS.some(({ name }) => name === "update_plan"), | |
| })); | |
| "#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("use exec to inspect nested tool namespaces") | |
| .await?; | |
| assert!( | |
| tool_names(&first_mock.single_request().body_json()).contains(&"excluded".to_string()), | |
| "excluded namespace should remain directly exposed in mixed code mode" | |
| ); | |
| let request = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec configured namespace exclusion failed unexpectedly: {output}" | |
| ); | |
| assert_eq!( | |
| serde_json::from_str::<Value>(&output)?, | |
| serde_json::json!({ | |
| "excludedType": "undefined", | |
| "excludedMetadata": false, | |
| "allowedType": "function", | |
| "allowedMetadata": true, | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_omits_configured_mcp_server_tools() -> Result<()> { | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "requires a Windows test_stdio_server in the Wine-exec environment" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let rmcp_test_server_bin = remote_aware_stdio_server_bin()?; | |
| let environment_id = remote_aware_environment_id(); | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.4", |model| { | |
| model.supports_search_tool = false; | |
| }) | |
| .with_config(move |config| { | |
| config.update_plan_enabled = true; | |
| let _ = config.features.enable(Feature::CodeMode); | |
| let mut servers = config.mcp_servers.get().clone(); | |
| servers.insert( | |
| "rmcp".to_string(), | |
| serde_json::from_value(serde_json::json!({ | |
| "command": rmcp_test_server_bin, | |
| "environment_id": environment_id, | |
| "cwd": config.cwd, | |
| "omit_tools_from": ["code_mode"], | |
| })) | |
| .expect("test MCP server config should be valid"), | |
| ); | |
| config | |
| .mcp_servers | |
| .set(servers) | |
| .expect("test config should allow MCP servers"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| wait_for_mcp_server(&test.codex, "rmcp").await?; | |
| let first_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| r#" | |
| text(JSON.stringify({ | |
| excludedType: typeof tools.mcp__rmcp__echo, | |
| excludedMetadata: ALL_TOOLS.some(({ name }) => name === "mcp__rmcp__echo"), | |
| allowedType: typeof tools.update_plan, | |
| allowedMetadata: ALL_TOOLS.some(({ name }) => name === "update_plan"), | |
| })); | |
| "#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("use exec to inspect nested tool namespaces") | |
| .await?; | |
| assert!( | |
| tool_names(&first_mock.single_request().body_json()).contains(&"mcp__rmcp".to_string()), | |
| "omitted MCP server should remain directly exposed in mixed code mode" | |
| ); | |
| let request = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec configured MCP omission failed unexpectedly: {output}" | |
| ); | |
| assert_eq!( | |
| serde_json::from_str::<Value>(&output)?, | |
| serde_json::json!({ | |
| "excludedType": "undefined", | |
| "excludedMetadata": false, | |
| "allowedType": "function", | |
| "allowedMetadata": true, | |
| }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_only_keeps_mcp_tools_direct_when_nested_exposure_is_omitted() -> Result<()> { | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "requires a Windows test_stdio_server in the Wine-exec environment" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let rmcp_test_server_bin = remote_aware_stdio_server_bin()?; | |
| let environment_id = remote_aware_environment_id(); | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.4", |model| { | |
| model.supports_search_tool = true; | |
| }) | |
| .with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeModeOnly); | |
| let mut servers = config.mcp_servers.get().clone(); | |
| servers.insert( | |
| "rmcp".to_string(), | |
| serde_json::from_value(serde_json::json!({ | |
| "command": rmcp_test_server_bin, | |
| "environment_id": environment_id, | |
| "cwd": config.cwd, | |
| "omit_tools_from": ["code_mode"], | |
| })) | |
| .expect("test MCP server config should be valid"), | |
| ); | |
| config | |
| .mcp_servers | |
| .set(servers) | |
| .expect("test config should allow MCP servers"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| wait_for_mcp_server(&test.codex, "rmcp").await?; | |
| let first_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| responses::ev_function_call_with_namespace( | |
| "call-1", | |
| "mcp__rmcp", | |
| "echo", | |
| r#"{"message":"ping"}"#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("call the directly exposed MCP echo tool") | |
| .await?; | |
| let first_request = first_mock.single_request().body_json(); | |
| assert!( | |
| namespace_child_tool(&first_request, "mcp__rmcp", "echo").is_some(), | |
| "an MCP tool omitted only from Code Mode must remain directly callable" | |
| ); | |
| let exec_description = first_request["tools"] | |
| .as_array() | |
| .expect("request should contain tools") | |
| .iter() | |
| .find_map(|tool| { | |
| (tool["name"].as_str() == Some("exec")) | |
| .then(|| tool["description"].as_str()) | |
| .flatten() | |
| }) | |
| .expect("Code Mode exec should remain available"); | |
| assert!( | |
| !exec_description.contains("mcp__rmcp__echo"), | |
| "an MCP tool omitted from Code Mode must not appear in the exec declaration" | |
| ); | |
| let output = second_mock.single_request().function_call_output("call-1"); | |
| assert!( | |
| output["output"] | |
| .as_str() | |
| .is_some_and(|output| output.contains("ECHOING: ping")), | |
| "the directly exposed MCP tool must execute successfully: {output:?}" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_only_can_call_mcp_tools_hidden_from_direct_and_deferred_exposure() -> Result<()> | |
| { | |
| skip_if_wine_exec!( | |
| Ok(()), | |
| "requires a Windows test_stdio_server in the Wine-exec environment" | |
| ); | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let rmcp_test_server_bin = remote_aware_stdio_server_bin()?; | |
| let environment_id = remote_aware_environment_id(); | |
| let mut builder = test_codex() | |
| .with_model_info_override("gpt-5.4", |model| { | |
| model.supports_search_tool = true; | |
| }) | |
| .with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeModeOnly); | |
| let mut servers = config.mcp_servers.get().clone(); | |
| servers.insert( | |
| "rmcp".to_string(), | |
| serde_json::from_value(serde_json::json!({ | |
| "command": rmcp_test_server_bin, | |
| "environment_id": environment_id, | |
| "cwd": config.cwd, | |
| "omit_tools_from": ["direct", "deferred"], | |
| "supports_parallel_tool_calls": true, | |
| })) | |
| .expect("test MCP server config should be valid"), | |
| ); | |
| config | |
| .mcp_servers | |
| .set(servers) | |
| .expect("test config should allow MCP servers"); | |
| }); | |
| let test = builder.build_with_auto_env(&server).await?; | |
| wait_for_mcp_server(&test.codex, "rmcp").await?; | |
| let first_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| r#" | |
| const results = await Promise.all([ | |
| tools.mcp__rmcp__echo({ message: "ping" }), | |
| tools.mcp__rmcp__echo({ message: "pong" }), | |
| ]); | |
| text(results.map(({ structuredContent }) => structuredContent?.echo ?? "missing").join(",")); | |
| "#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let second_mock = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "done"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("call the MCP echo tool from Code Mode") | |
| .await?; | |
| let first_request = first_mock.single_request().body_json(); | |
| assert!( | |
| namespace_child_tool(&first_request, "mcp__rmcp", "echo").is_none(), | |
| "MCP tool omitted from direct and deferred exposure must remain hidden at the top level" | |
| ); | |
| let exec_description = first_request["tools"] | |
| .as_array() | |
| .expect("request should contain tools") | |
| .iter() | |
| .find_map(|tool| { | |
| (tool["name"].as_str() == Some("exec")) | |
| .then(|| tool["description"].as_str()) | |
| .flatten() | |
| }) | |
| .expect("Code Mode exec should remain available"); | |
| assert!( | |
| exec_description.contains("mcp__rmcp__echo"), | |
| "hidden MCP tool must remain available inside Code Mode" | |
| ); | |
| let request = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "hidden Code Mode MCP tool call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!(output, "ECHOING: ping,ECHOING: pong"); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_print_content_only_mcp_tool_result_fields() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const { content, structuredContent, isError } = await tools.mcp__rmcp__image_scenario({ | |
| scenario: "text_only", | |
| caption: "caption from mcp", | |
| }); | |
| text( | |
| `firstType=${content[0]?.type ?? "missing"}\n` + | |
| `firstText=${content[0]?.text ?? "missing"}\n` + | |
| `structuredContent=${String(structuredContent ?? null)}\n` + | |
| `isError=${String(isError)}` | |
| ); | |
| "#; | |
| let (_test, second_mock) = run_code_mode_turn_with_rmcp( | |
| &server, | |
| "use exec to run the rmcp image scenario tool", | |
| code, | |
| ) | |
| .await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec rmcp image scenario call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!( | |
| output, | |
| "firstType=text | |
| firstText=caption from mcp | |
| structuredContent=null | |
| isError=false" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_print_error_mcp_tool_result_fields() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let code = r#" | |
| const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({}); | |
| const firstText = content[0]?.text ?? ""; | |
| const mentionsMissingMessage = | |
| firstText.includes("missing field") && firstText.includes("message"); | |
| text( | |
| `isError=${String(isError)}\n` + | |
| `contentLength=${content.length}\n` + | |
| `mentionsMissingMessage=${String(mentionsMissingMessage)}\n` + | |
| `structuredContent=${String(structuredContent ?? null)}` | |
| ); | |
| "#; | |
| let (_test, second_mock) = | |
| run_code_mode_turn_with_rmcp(&server, "use exec to call rmcp echo badly", code).await?; | |
| let req = second_mock.single_request(); | |
| let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); | |
| assert_ne!( | |
| success, | |
| Some(false), | |
| "exec rmcp error call failed unexpectedly: {output}" | |
| ); | |
| assert_eq!( | |
| output, | |
| "isError=true | |
| contentLength=1 | |
| mentionsMissingMessage=true | |
| structuredContent=null" | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_store_and_load_values_across_turns() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let mut builder = test_codex().with_config(move |config| { | |
| let _ = config.features.enable(Feature::CodeMode); | |
| }); | |
| let test = builder.build(&server).await?; | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-1"), | |
| ev_custom_tool_call( | |
| "call-1", | |
| "exec", | |
| r#" | |
| store("nb", { title: "Notebook", items: [1, true, null] }); | |
| text("stored"); | |
| "#, | |
| ), | |
| ev_completed("resp-1"), | |
| ]), | |
| ) | |
| .await; | |
| let first_follow_up = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-1", "stored"), | |
| ev_completed("resp-2"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("store value for later").await?; | |
| let first_request = first_follow_up.single_request(); | |
| let (first_output, first_success) = | |
| custom_tool_output_body_and_success(&first_request, "call-1"); | |
| assert_ne!( | |
| first_success, | |
| Some(false), | |
| "exec store call failed unexpectedly: {first_output}" | |
| ); | |
| assert_eq!(first_output, "stored"); | |
| responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_response_created("resp-3"), | |
| ev_custom_tool_call( | |
| "call-2", | |
| "exec", | |
| r#" | |
| text(JSON.stringify(load("nb"))); | |
| "#, | |
| ), | |
| ev_completed("resp-3"), | |
| ]), | |
| ) | |
| .await; | |
| let second_follow_up = responses::mount_sse_once( | |
| &server, | |
| sse(vec![ | |
| ev_assistant_message("msg-2", "loaded"), | |
| ev_completed("resp-4"), | |
| ]), | |
| ) | |
| .await; | |
| test.submit_turn("load the stored value").await?; | |
| let second_request = second_follow_up.single_request(); | |
| let (second_output, second_success) = | |
| custom_tool_output_body_and_success(&second_request, "call-2"); | |
| assert_ne!( | |
| second_success, | |
| Some(false), | |
| "exec load call failed unexpectedly: {second_output}" | |
| ); | |
| let loaded: Value = serde_json::from_str( | |
| &custom_tool_output_last_non_empty_text(&second_request, "call-2") | |
| .expect("exec load call should emit JSON"), | |
| )?; | |
| assert_eq!( | |
| loaded, | |
| serde_json::json!({ "title": "Notebook", "items": [1, true, null] }) | |
| ); | |
| Ok(()) | |
| } | |
| async fn code_mode_can_compare_elapsed_time_around_set_timeout() -> Result<()> { | |
| skip_if_no_network!(Ok(())); | |
| let server = responses::start_mock_server().await; | |
| let (_test, second_mock) = run_code_mode_turn( | |
| &server, | |
| "measure elapsed time around setTimeout", | |
| r#" | |
| const start_ms = Date.now(); | |
| await new Promise((resolve) => setTimeout(resolve, 100)); | |
| const end_ms = Date.now(); | |
| text(JSON.stringify({ | |
| start_ms, | |
| end_ms, | |
| elapsed_ms: end_ms - start_ms, | |
| waited_long_enough: end_ms - start_ms >= 100, | |
| })); | |
| "#, | |
| ) | |
| .await?; | |
| let second_request = second_mock.single_request(); | |
| let (second_output, second_success) = | |
| custom_tool_output_body_and_success(&second_request, "call-1"); | |
| assert_ne!( | |
| second_success, | |
| Some(false), | |
| "exec compare time call failed unexpectedly: {second_output}" | |
| ); | |
| let compared: Value = serde_json::from_str( | |
| &custom_tool_output_last_non_empty_text(&second_request, "call-1") | |
| .expect("exec compare time call should emit JSON"), | |
| )?; | |
| let elapsed_ms = compared | |
| .get("elapsed_ms") | |
| .and_then(Value::as_i64) | |
| .expect("elapsed_ms should be an integer"); | |
| assert!( | |
| elapsed_ms >= 100, | |
| "expected elapsed_ms >= 100, got {elapsed_ms}" | |
| ); | |
| assert_eq!(compared.get("waited_long_enough"), Some(&Value::Bool(true))); | |
| Ok(()) | |
| } | |