diff --git a/Cargo.lock b/Cargo.lock index 87445a35e02bcd9710a889c85c674d1d85dcfa35..5d7343a7129070f1be7ad091fd2f098566f65a1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4514,6 +4514,7 @@ name = "koharu-ai" version = "0.48.0" dependencies = [ "anyhow", + "async-trait", "base64 0.22.1", "clap", "eventsource-stream", @@ -4535,6 +4536,7 @@ dependencies = [ "arc-swap", "async-trait", "atomicwrites", + "base64 0.22.1", "blake3", "camino", "chrono", @@ -4546,6 +4548,7 @@ dependencies = [ "imageproc", "indexmap 2.14.0", "inventory", + "koharu-ai", "koharu-core", "koharu-llm", "koharu-ml", diff --git a/koharu-ai/Cargo.toml b/koharu-ai/Cargo.toml index fb7c5a03a260fffa3fb83b509b9b6ac967818b97..c8ef139c01033f68e247f298482a1b8508d72fcb 100644 --- a/koharu-ai/Cargo.toml +++ b/koharu-ai/Cargo.toml @@ -17,6 +17,7 @@ path = "bin/codex.rs" [dependencies] anyhow = { workspace = true } +async-trait = { workspace = true } base64 = { workspace = true } clap = { workspace = true } eventsource-stream = { workspace = true } diff --git a/koharu-ai/bin/codex.rs b/koharu-ai/bin/codex.rs index f1f904e20cd2bef95fef7f288dadd20834513a19..a9dd59ad00f97e5aafe6f2509aef48cf76bfe5cf 100644 --- a/koharu-ai/bin/codex.rs +++ b/koharu-ai/bin/codex.rs @@ -7,7 +7,7 @@ use eventsource_stream::Eventsource; use futures::StreamExt; use koharu_ai::codex::{ CodexClient, CodexConfig, CodexImageGenerationRequest, CodexInputImage, CodexTaskRequest, - DEFAULT_RESPONSES_URL, + DEFAULT_RESPONSES_URL, image_response_stream_result, }; use serde_json::Value; @@ -207,8 +207,10 @@ async fn image_cmd(client: &CodexClient, command: ImageCommand) -> anyhow::Resul } let response = client.create_response_raw(&request).await?; - let Some(url) = image_response_stream_url(response).await? else { - anyhow::bail!("No image URL or image result found in response stream"); + let result = image_response_stream_result(response).await?; + let Some(url) = result.image_url else { + let response_text = result.response_text.as_deref().unwrap_or("none"); + anyhow::bail!("No image URL or image result found in response stream: {response_text}"); }; println!("{url}"); Ok(()) @@ -252,77 +254,6 @@ async fn print_response_stream(response: reqwest::Response) -> anyhow::Result<() Ok(()) } -async fn image_response_stream_url(response: reqwest::Response) -> anyhow::Result> { - let mut stream = response.bytes_stream().eventsource(); - let mut result = None; - - while let Some(event) = stream.next().await { - let event = event?; - let Ok(data) = serde_json::from_str::(&event.data) else { - continue; - }; - if let Some(url) = extract_image_url(&data) { - result = Some(url); - } - } - - Ok(result) -} - -fn extract_image_url(value: &Value) -> Option { - match value { - Value::Object(map) => { - if matches!( - map.get("type").and_then(Value::as_str), - Some("image_generation_call") - ) && let Some(url) = extract_image_result(map.get("result")?) - { - return Some(url); - } - - if let Some(call) = map.get("image_generation_call") - && let Some(url) = extract_image_url(call) - { - return Some(url); - } - - if let Some(url) = map.get("url").and_then(Value::as_str) - && (url.starts_with("http://") - || url.starts_with("https://") - || url.starts_with("data:image/")) - { - return Some(url.to_string()); - } - - for child in map.values() { - if let Some(url) = extract_image_url(child) { - return Some(url); - } - } - None - } - Value::Array(items) => items.iter().find_map(extract_image_url), - _ => None, - } -} - -fn extract_image_result(value: &Value) -> Option { - match value { - Value::String(value) if value.starts_with("http://") || value.starts_with("https://") => { - Some(value.clone()) - } - Value::String(value) if value.starts_with("data:image/") => Some(value.clone()), - Value::String(value) if !value.is_empty() => Some(format!("data:image/png;base64,{value}")), - Value::Object(map) => map - .get("url") - .and_then(Value::as_str) - .map(ToOwned::to_owned) - .or_else(|| map.values().find_map(extract_image_url)), - Value::Array(items) => items.iter().find_map(extract_image_url), - _ => None, - } -} - fn image_data_url(path: &Path) -> anyhow::Result { let bytes = std::fs::read(path)?; let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); @@ -346,6 +277,7 @@ fn image_mime_type(path: &Path) -> &'static str { #[cfg(test)] mod tests { use super::*; + use koharu_ai::codex::extract_image_url; #[test] fn extracts_nested_image_generation_url() { diff --git a/koharu-ai/src/codex/client.rs b/koharu-ai/src/codex/client.rs index 49ece9e97b3618c35bb7f46325593365125f048d..5e6fd3f88b1df1fcd30b9dd8913e72b80cfd9c1e 100644 --- a/koharu-ai/src/codex/client.rs +++ b/koharu-ai/src/codex/client.rs @@ -9,12 +9,14 @@ use serde_json::Value; use super::config::CodexConfig; use super::device::{DeviceAuthorization, DeviceCode}; use super::error::{CodexError, Result}; +use super::image::{CodexImageGenerationRequest, CodexInputImage, image_response_stream_result}; use super::requests::{ TokenExchangeRequest, TokenExchangeResponse, TokenPollRequest, TokenPollSuccessResponse, TokenRefreshRequest, TokenRefreshResponse, UserCodeRequest, UserCodeResponse, }; use super::token_store::TokenStore; use super::tokens::CodexTokens; +use crate::provider::{AiImageProvider, AiImageRequest, AiImageResult}; const USER_AGENT: &str = concat!("koharu-ai/", env!("CARGO_PKG_VERSION")); @@ -306,3 +308,38 @@ async fn ensure_success(endpoint: &str, response: reqwest::Response) -> Result anyhow::Result { + let action = request.action.unwrap_or_else(|| { + if request.input_image.is_some() { + "edit".to_string() + } else { + "generate".to_string() + } + }); + + let mut codex_request = CodexImageGenerationRequest::new(request.model, request.prompt) + .with_instructions(request.instructions) + .with_quality(request.quality) + .with_action(action); + if let Some(size) = request.size { + codex_request = codex_request.with_size(size); + } + if let Some(image) = request.input_image { + codex_request = codex_request + .with_input_image(CodexInputImage::new(image.data_url).with_detail(image.detail)); + } + + let response = self.create_response_raw(&codex_request).await?; + let result = image_response_stream_result(response).await?; + let image_url = result.image_url.ok_or_else(|| { + let response_text = result.response_text.as_deref().unwrap_or("none"); + anyhow::anyhow!( + "Codex returned no image URL or image result. Response text: {response_text}" + ) + })?; + Ok(AiImageResult { image_url }) + } +} diff --git a/koharu-ai/src/codex/image.rs b/koharu-ai/src/codex/image.rs index b60c536caff1631e87a25826714a0336a014dc15..c5574f049d0bc04fce6e04c0d365e90c21abb0ec 100644 --- a/koharu-ai/src/codex/image.rs +++ b/koharu-ai/src/codex/image.rs @@ -1,4 +1,8 @@ use serde::Serialize; +use serde_json::Value; + +use eventsource_stream::Eventsource; +use futures::StreamExt; use super::responses::{CodexInputContent, CodexInputItem}; @@ -10,11 +14,18 @@ pub struct CodexImageGenerationRequest { pub model: String, pub instructions: String, pub tools: [CodexImageGenerationTool; 1], + pub tool_choice: CodexImageToolChoice, pub input: Vec, pub stream: bool, pub store: bool, } +#[derive(Debug, Clone, Serialize)] +pub struct CodexImageToolChoice { + #[serde(rename = "type")] + pub tool_type: &'static str, +} + #[derive(Debug, Clone, Serialize)] pub struct CodexImageGenerationTool { #[serde(rename = "type")] @@ -38,12 +49,19 @@ pub struct CodexInputImage { pub detail: String, } +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CodexImageStreamResult { + pub image_url: Option, + pub response_text: Option, +} + impl CodexImageGenerationRequest { pub fn new(model: impl Into, prompt: impl Into) -> Self { Self { model: model.into(), instructions: DEFAULT_IMAGE_INSTRUCTIONS.to_string(), tools: [CodexImageGenerationTool::default()], + tool_choice: CodexImageToolChoice::image_generation(), input: vec![CodexInputItem::user_text(prompt)], stream: true, store: false, @@ -85,6 +103,14 @@ impl CodexImageGenerationRequest { } } +impl CodexImageToolChoice { + pub fn image_generation() -> Self { + Self { + tool_type: "image_generation", + } + } +} + impl CodexImageGenerationTool { pub fn new(image_generation: CodexImageGenerationConfig) -> Self { Self { @@ -124,6 +150,236 @@ impl CodexInputImage { } } +pub async fn image_response_stream_url( + response: reqwest::Response, +) -> anyhow::Result> { + Ok(image_response_stream_result(response).await?.image_url) +} + +pub async fn image_response_stream_result( + response: reqwest::Response, +) -> anyhow::Result { + let mut stream = response.bytes_stream().eventsource(); + let mut collector = CodexImageStreamCollector::default(); + + while let Some(event) = stream.next().await { + let event = event?; + let Ok(data) = serde_json::from_str::(&event.data) else { + continue; + }; + collector.push(&data)?; + } + + Ok(collector.finish()) +} + +#[derive(Debug, Default)] +struct CodexImageStreamCollector { + final_image: Option, + partial_image: Option, + output_text: Vec, + final_text: Option, +} + +impl CodexImageStreamCollector { + fn push(&mut self, value: &Value) -> anyhow::Result<()> { + if let Some(error) = extract_response_error(value) { + anyhow::bail!("{error}"); + } + + if let Some(url) = extract_final_image_url(value) { + self.final_image = Some(url); + } + if let Some(url) = extract_partial_image_url(value) { + self.partial_image = Some(url); + } + if let Some(delta) = extract_output_text_delta(value) { + self.output_text.push(delta); + } + if let Some(text) = extract_response_text(value) { + self.final_text = Some(text); + } + + Ok(()) + } + + fn finish(self) -> CodexImageStreamResult { + CodexImageStreamResult { + image_url: self.final_image.or(self.partial_image), + response_text: self + .final_text + .or_else(|| join_text_fragments(self.output_text)), + } + } +} + +pub fn extract_image_url(value: &Value) -> Option { + extract_final_image_url(value).or_else(|| extract_partial_image_url(value)) +} + +fn extract_final_image_url(value: &Value) -> Option { + find_map_value(value, &mut |value| { + let Value::Object(map) = value else { + return None; + }; + + if matches!( + map.get("type").and_then(Value::as_str), + Some("image_generation_call") + ) { + return map.get("result").and_then(extract_image_result); + } + + map.get("image_generation_call") + .and_then(extract_final_image_url) + .or_else(|| { + map.get("url") + .or_else(|| map.get("image_url")) + .and_then(Value::as_str) + .filter(|url| is_image_url(url)) + .map(ToOwned::to_owned) + }) + }) +} + +fn extract_partial_image_url(value: &Value) -> Option { + let Value::Object(map) = value else { + return None; + }; + + let b64 = map + .get("partial_image_b64") + .or_else(|| map.get("b64_json")) + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty())?; + Some(format!("data:image/png;base64,{b64}")) +} + +fn extract_image_result(value: &Value) -> Option { + match value { + Value::String(value) if value.starts_with("http://") || value.starts_with("https://") => { + Some(value.clone()) + } + Value::String(value) if value.starts_with("data:image/") => Some(value.clone()), + Value::String(value) if !value.is_empty() => Some(format!("data:image/png;base64,{value}")), + Value::Object(map) => map + .get("url") + .and_then(Value::as_str) + .filter(|url| is_image_url(url)) + .map(ToOwned::to_owned) + .or_else(|| map.values().find_map(extract_final_image_url)), + Value::Array(items) => items.iter().find_map(extract_final_image_url), + _ => None, + } +} + +fn is_image_url(value: &str) -> bool { + value.starts_with("http://") + || value.starts_with("https://") + || value.starts_with("data:image/") +} + +fn extract_output_text_delta(value: &Value) -> Option { + let Value::Object(map) = value else { + return None; + }; + if !matches!( + map.get("type").and_then(Value::as_str), + Some("response.output_text.delta") + ) { + return None; + } + map.get("delta") + .and_then(Value::as_str) + .filter(|text| !text.is_empty()) + .map(ToOwned::to_owned) +} + +fn extract_response_text(value: &Value) -> Option { + let mut fragments = Vec::new(); + collect_response_text(value, &mut fragments); + join_text_fragments(fragments) +} + +fn collect_response_text(value: &Value, fragments: &mut Vec) { + match value { + Value::Object(map) => { + if matches!( + map.get("type").and_then(Value::as_str), + Some("response.output_text.done" | "output_text") + ) && let Some(text) = map.get("text").and_then(Value::as_str) + && !text.is_empty() + { + fragments.push(text.to_string()); + return; + } + + if matches!(map.get("type").and_then(Value::as_str), Some("message")) + && !matches!(map.get("role").and_then(Value::as_str), Some("assistant")) + { + return; + } + + for key in ["response", "output", "item", "content"] { + if let Some(child) = map.get(key) { + collect_response_text(child, fragments); + } + } + } + Value::Array(items) => { + for item in items { + collect_response_text(item, fragments); + } + } + _ => {} + } +} + +fn join_text_fragments(fragments: Vec) -> Option { + let text = fragments.concat(); + let text = text.trim(); + if text.is_empty() { + None + } else { + Some(text.to_string()) + } +} + +fn find_map_value(value: &Value, f: &mut impl FnMut(&Value) -> Option) -> Option { + if let Some(found) = f(value) { + return Some(found); + } + + match value { + Value::Object(map) => map.values().find_map(|child| find_map_value(child, f)), + Value::Array(items) => items.iter().find_map(|child| find_map_value(child, f)), + _ => None, + } +} + +fn extract_response_error(value: &Value) -> Option { + let Value::Object(map) = value else { + return None; + }; + let event_type = map.get("type").and_then(Value::as_str); + if !matches!( + event_type, + Some("response.failed" | "response.incomplete" | "error") + ) { + return None; + } + + map.get("error") + .and_then(|error| { + error + .get("message") + .and_then(Value::as_str) + .or_else(|| error.as_str()) + }) + .or_else(|| map.get("message").and_then(Value::as_str)) + .map(ToOwned::to_owned) +} + #[cfg(test)] mod tests { use super::*; @@ -145,6 +401,7 @@ mod tests { "draw a koharu logo" ); assert_eq!(value["tools"][0]["type"], "image_generation"); + assert_eq!(value["tool_choice"]["type"], "image_generation"); assert_eq!(value["tools"][0]["quality"], "high"); assert_eq!(value["tools"][0]["action"], "generate"); assert_eq!(value["stream"], true); @@ -168,6 +425,134 @@ mod tests { ); assert_eq!(value["input"][0]["content"][1]["detail"], "high"); assert_eq!(value["tools"][0]["action"], "edit"); + assert_eq!(value["tool_choice"]["type"], "image_generation"); assert!(value.get("input_image").is_none()); } + + #[test] + fn extracts_nested_image_generation_url() { + let value = serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "image_generation_call", + "result": { + "url": "https://example.test/image.png" + } + } + }); + + assert_eq!( + extract_image_url(&value), + Some("https://example.test/image.png".to_string()) + ); + } + + #[test] + fn converts_base64_image_generation_result_to_data_url() { + let value = serde_json::json!({ + "type": "image_generation_call", + "result": "abc123" + }); + + assert_eq!( + extract_image_url(&value), + Some("data:image/png;base64,abc123".to_string()) + ); + } + + #[test] + fn extracts_responses_partial_image_event() { + let value = serde_json::json!({ + "type": "response.image_generation_call.partial_image", + "partial_image_b64": "abc123" + }); + + assert_eq!( + extract_image_url(&value), + Some("data:image/png;base64,abc123".to_string()) + ); + } + + #[test] + fn extracts_images_stream_completed_event() { + let value = serde_json::json!({ + "type": "image_generation.completed", + "b64_json": "def456" + }); + + assert_eq!( + extract_image_url(&value), + Some("data:image/png;base64,def456".to_string()) + ); + } + + #[test] + fn extracts_stream_error_message() { + let value = serde_json::json!({ + "type": "response.failed", + "error": { "message": "image generation failed" } + }); + + assert_eq!( + extract_response_error(&value), + Some("image generation failed".to_string()) + ); + } + + #[test] + fn extracts_response_text_from_completed_message() { + let value = serde_json::json!({ + "type": "response.completed", + "response": { + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I could not generate the image." + } + ] + } + ] + } + }); + + assert_eq!( + extract_response_text(&value), + Some("I could not generate the image.".to_string()) + ); + } + + #[test] + fn collector_prefers_final_image_and_keeps_text() { + let mut collector = CodexImageStreamCollector::default(); + collector + .push(&serde_json::json!({ + "type": "response.output_text.delta", + "delta": "Working" + })) + .unwrap(); + collector + .push(&serde_json::json!({ + "type": "response.image_generation_call.partial_image", + "partial_image_b64": "partial" + })) + .unwrap(); + collector + .push(&serde_json::json!({ + "type": "image_generation_call", + "result": "final" + })) + .unwrap(); + + assert_eq!( + collector.finish(), + CodexImageStreamResult { + image_url: Some("data:image/png;base64,final".to_string()), + response_text: Some("Working".to_string()), + } + ); + } } diff --git a/koharu-ai/src/codex/mod.rs b/koharu-ai/src/codex/mod.rs index ce8a717d6a7f5f9fe50b35b622c441c79a12d945..19eff01a08179fa6870a49451a41e25f84c85ad8 100644 --- a/koharu-ai/src/codex/mod.rs +++ b/koharu-ai/src/codex/mod.rs @@ -16,7 +16,8 @@ pub use device::{DeviceAuthorization, DeviceCode}; pub use error::{CodexError, Result}; pub use image::{ CodexImageGenerationConfig, CodexImageGenerationRequest, CodexImageGenerationTool, - CodexInputImage, + CodexImageStreamResult, CodexInputImage, extract_image_url, image_response_stream_result, + image_response_stream_url, }; pub use responses::{CodexInputContent, CodexInputItem}; pub use task::CodexTaskRequest; diff --git a/koharu-ai/src/lib.rs b/koharu-ai/src/lib.rs index 955f645eabf3fe2461028a6ad1f6a42335d3b692..ffb50e4a7759372fe9c9ef1856c88bbbe9b11613 100644 --- a/koharu-ai/src/lib.rs +++ b/koharu-ai/src/lib.rs @@ -1 +1,4 @@ pub mod codex; +pub mod provider; + +pub use provider::{AiImageProvider, AiImageRequest, AiImageResult, AiInputImage}; diff --git a/koharu-ai/src/provider.rs b/koharu-ai/src/provider.rs new file mode 100644 index 0000000000000000000000000000000000000000..a99534833d322d32ee9980dd82f9b331e6aa57fb --- /dev/null +++ b/koharu-ai/src/provider.rs @@ -0,0 +1,50 @@ +use async_trait::async_trait; + +#[derive(Debug, Clone)] +pub struct AiInputImage { + pub data_url: String, + pub detail: String, +} + +#[derive(Debug, Clone)] +pub struct AiImageRequest { + pub model: String, + pub instructions: String, + pub prompt: String, + pub input_image: Option, + pub quality: String, + pub size: Option, + pub action: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AiImageResult { + pub image_url: String, +} + +#[async_trait] +pub trait AiImageProvider: Send + Sync { + async fn generate_image(&self, request: AiImageRequest) -> anyhow::Result; +} + +impl AiImageRequest { + pub fn new(model: impl Into, prompt: impl Into) -> Self { + Self { + model: model.into(), + instructions: "Generate or edit the requested image.".to_string(), + prompt: prompt.into(), + input_image: None, + quality: "high".to_string(), + size: None, + action: None, + } + } + + pub fn with_input_image(mut self, data_url: impl Into) -> Self { + self.input_image = Some(AiInputImage { + data_url: data_url.into(), + detail: "high".to_string(), + }); + self + } +} diff --git a/koharu-app/Cargo.toml b/koharu-app/Cargo.toml index 2ab59b3b2ef915026638243da7dc8e3b359e3853..499a8daabebc824c1095caa18cc215f0fe938262 100644 --- a/koharu-app/Cargo.toml +++ b/koharu-app/Cargo.toml @@ -19,6 +19,7 @@ path = "bin/pipeline.rs" clap = { workspace = true } tracing-subscriber = { workspace = true } koharu-core = { workspace = true } +koharu-ai = { workspace = true } koharu-ml = { workspace = true } koharu-renderer = { workspace = true } koharu-llm = { workspace = true } @@ -27,6 +28,7 @@ anyhow = { workspace = true } arc-swap = { workspace = true } async-trait = { workspace = true } atomicwrites = { workspace = true } +base64 = { workspace = true } blake3 = { workspace = true } camino = { workspace = true } chrono = { workspace = true } diff --git a/koharu-app/src/ai.rs b/koharu-app/src/ai.rs new file mode 100644 index 0000000000000000000000000000000000000000..96842ceadda29029fe3f97bf76a776e33f75eb00 --- /dev/null +++ b/koharu-app/src/ai.rs @@ -0,0 +1,431 @@ +use std::io::Cursor; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use base64::Engine as _; +use dashmap::DashMap; +use image::DynamicImage; +use koharu_ai::codex::{CodexClient, CodexConfig}; +use koharu_ai::{AiImageProvider, AiImageRequest}; +use koharu_core::{ + BlobRef, ImageData, ImageDataPatch, ImageRole, Node, NodeDataPatch, NodeId, NodeKind, + NodePatch, Op, PageId, Scene, Transform, +}; +use koharu_runtime::{RuntimeHttpClient, RuntimeManager}; +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; +use tracing::Instrument as _; +use utoipa::ToSchema; +use uuid::Uuid; + +use crate::session::ProjectSession; + +const DEFAULT_CODEX_IMAGE_MODEL: &str = "gpt-5.5"; +const DEFAULT_CODEX_IMAGE_INSTRUCTIONS: &str = "Generate or edit the requested image."; +const DEFAULT_CODEX_IMAGE_QUALITY: &str = "high"; + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum CodexAuthAttemptStatus { + Pending, + Succeeded, + Failed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CodexDeviceLogin { + pub login_id: String, + pub verification_url: String, + pub user_code: String, + pub interval_seconds: u64, + pub timeout_seconds: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CodexDeviceLoginStatus { + pub login_id: String, + pub status: CodexAuthAttemptStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub account_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CodexAuthStatus { + pub signed_in: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub account_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub login: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CodexImageGenerationOptions { + pub page_id: PageId, + pub prompt: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub instructions: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quality: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, +} + +#[derive(Debug, Clone)] +struct LoginAttempt { + status: CodexAuthAttemptStatus, + account_id: Option, + error: Option, +} + +pub struct AiManager { + codex: CodexClient, + http_client: RuntimeHttpClient, + codex_device_timeout: Duration, + codex_logins: Arc>, + latest_codex_login: RwLock>, +} + +impl AiManager { + pub fn new(runtime: &RuntimeManager) -> Self { + let config = CodexConfig::default(); + let codex_device_timeout = config.device_auth_timeout; + Self { + codex: CodexClient::with_http_client(config, runtime.http_client()), + http_client: runtime.http_client(), + codex_device_timeout, + codex_logins: Arc::new(DashMap::new()), + latest_codex_login: RwLock::new(None), + } + } + + pub fn codex_auth_status(&self) -> Result { + let tokens = self.codex.token_store().load()?; + let account_id = tokens + .as_ref() + .and_then(|tokens| tokens.chatgpt_account_id()); + let login = self + .latest_codex_login + .read() + .as_ref() + .and_then(|id| { + self.codex_logins + .get(id) + .map(|entry| (id.clone(), entry.clone())) + }) + .map(|(login_id, attempt)| CodexDeviceLoginStatus { + login_id, + status: attempt.status, + account_id: attempt.account_id, + error: attempt.error, + }); + + Ok(CodexAuthStatus { + signed_in: tokens.is_some(), + account_id, + login, + }) + } + + pub async fn start_codex_device_login(self: &Arc) -> Result { + let device_code = self.codex.request_device_code().await?; + let login_id = Uuid::new_v4().to_string(); + self.codex_logins.insert( + login_id.clone(), + LoginAttempt { + status: CodexAuthAttemptStatus::Pending, + account_id: None, + error: None, + }, + ); + *self.latest_codex_login.write() = Some(login_id.clone()); + + let manager = Arc::clone(self); + let device_code_for_task = device_code.clone(); + let login_id_for_task = login_id.clone(); + tokio::spawn(async move { + let result = manager + .codex + .complete_device_code_login(&device_code_for_task) + .await; + let attempt = match result { + Ok(tokens) => LoginAttempt { + status: CodexAuthAttemptStatus::Succeeded, + account_id: tokens.chatgpt_account_id(), + error: None, + }, + Err(err) => LoginAttempt { + status: CodexAuthAttemptStatus::Failed, + account_id: None, + error: Some(format!("{err:#}")), + }, + }; + manager.codex_logins.insert(login_id_for_task, attempt); + }); + + let interval_seconds = device_code.interval().as_secs().max(1); + Ok(CodexDeviceLogin { + login_id, + verification_url: device_code.verification_url, + user_code: device_code.user_code, + interval_seconds, + timeout_seconds: self.codex_device_timeout.as_secs(), + }) + } + + pub fn logout_codex(&self) -> Result<()> { + self.codex.token_store().delete()?; + Ok(()) + } + + pub async fn generate_codex_page_image( + &self, + session: Arc, + options: CodexImageGenerationOptions, + cancel: Arc, + ) -> Result<()> { + let workflow_span = tracing::info_span!( + "codex_image_generation_workflow", + page_id = %options.page_id + ); + async move { + let prompt = options.prompt.trim().to_string(); + if prompt.is_empty() { + bail!("prompt is required"); + } + + let source = tracing::info_span!("codex_source_image_load").in_scope(|| { + let scene = session.scene_snapshot(); + let (_, image_data) = source_image(&scene, options.page_id)?; + session.blobs.load_image(&image_data.blob) + })?; + + let source_data_url = tracing::info_span!("codex_source_image_encode") + .in_scope(|| image_data_url(&source))?; + tracing::info!(bytes = source_data_url.len(), "encoded Codex source image"); + + check_cancelled(&cancel)?; + let mut request = AiImageRequest::new( + options + .model + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_CODEX_IMAGE_MODEL.to_string()), + prompt, + ) + .with_input_image(source_data_url); + request.instructions = options + .instructions + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_CODEX_IMAGE_INSTRUCTIONS.to_string()); + request.quality = options + .quality + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_CODEX_IMAGE_QUALITY.to_string()); + request.size = options + .size + .filter(|value| !value.trim().is_empty()) + .or_else(|| Some("auto".to_string())); + request.action = Some("edit".to_string()); + + let result = self + .codex + .generate_image(request) + .instrument(tracing::info_span!("codex_image_request")) + .await?; + tracing::info!("Codex image request completed"); + + check_cancelled(&cancel)?; + let generated_bytes = self + .load_generated_image_bytes(&result.image_url) + .instrument(tracing::info_span!("codex_generated_image_load")) + .await?; + tracing::info!( + bytes = generated_bytes.len(), + "loaded Codex generated image bytes" + ); + + let (width, height, blob) = tracing::info_span!("codex_generated_image_store") + .in_scope(|| { + let generated = image::load_from_memory(&generated_bytes) + .with_context(|| "failed to decode Codex image result")?; + let (width, height) = image_dimensions(&generated); + let blob = session.blobs.put_webp(&generated)?; + Ok::<_, anyhow::Error>((width, height, blob)) + })?; + tracing::info!(width, height, "decoded and stored Codex generated image"); + + check_cancelled(&cancel)?; + let scene = session.scene_snapshot(); + let op = upsert_image_blob( + &scene, + options.page_id, + ImageRole::Rendered, + blob, + width, + height, + )?; + session.apply(Op::Batch { + ops: vec![op], + label: format!("codex-image: page {}", options.page_id), + })?; + tracing::info!("finished Codex image generation workflow"); + Ok(()) + } + .instrument(workflow_span) + .await + } + + async fn load_generated_image_bytes(&self, url: &str) -> Result> { + if let Some(bytes) = decode_data_image_url(url)? { + return Ok(bytes); + } + + if !(url.starts_with("http://") || url.starts_with("https://")) { + bail!("unsupported Codex image result URL: {url}"); + } + + let response = self.http_client.get(url).send().await?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + bail!("failed to fetch Codex image result ({status}): {body}"); + } + Ok(response.bytes().await?.to_vec()) + } +} + +fn check_cancelled(cancel: &std::sync::atomic::AtomicBool) -> Result<()> { + if cancel.load(std::sync::atomic::Ordering::Relaxed) { + bail!("cancelled"); + } + Ok(()) +} + +fn source_image(scene: &Scene, page_id: PageId) -> Result<(NodeId, &ImageData)> { + let page = scene + .page(page_id) + .with_context(|| format!("page {} not found", page_id))?; + page.nodes + .iter() + .find_map(|(id, node)| match &node.kind { + NodeKind::Image(image) if image.role == ImageRole::Source => Some((*id, image)), + _ => None, + }) + .ok_or_else(|| anyhow!("page has no Source image node")) +} + +fn image_data_url(image: &DynamicImage) -> Result { + let mut buf = Cursor::new(Vec::new()); + image.write_to(&mut buf, image::ImageFormat::Png)?; + let encoded = base64::engine::general_purpose::STANDARD.encode(buf.into_inner()); + Ok(format!("data:image/png;base64,{encoded}")) +} + +fn decode_data_image_url(url: &str) -> Result>> { + let Some(rest) = url.strip_prefix("data:image/") else { + return Ok(None); + }; + let Some((_, data)) = rest.split_once(',') else { + bail!("invalid data image URL"); + }; + let decoded = base64::engine::general_purpose::STANDARD + .decode(data) + .context("failed to decode data image URL")?; + Ok(Some(decoded)) +} + +fn image_dimensions(image: &DynamicImage) -> (u32, u32) { + use image::GenericImageView as _; + image.dimensions() +} + +fn upsert_image_blob( + scene: &Scene, + page: PageId, + role: ImageRole, + blob: BlobRef, + natural_width: u32, + natural_height: u32, +) -> Result { + let page_ref = scene + .page(page) + .with_context(|| format!("page {} not found", page))?; + + if let Some((node_id, _)) = page_ref + .nodes + .iter() + .find_map(|(id, node)| match &node.kind { + NodeKind::Image(image) if image.role == role => Some((*id, image)), + _ => None, + }) + { + return Ok(Op::UpdateNode { + page, + id: node_id, + patch: NodePatch { + data: Some(NodeDataPatch::Image(ImageDataPatch { + blob: Some(blob), + opacity: None, + name: None, + natural_width: Some(natural_width), + natural_height: Some(natural_height), + })), + transform: None, + visible: None, + }, + prev: NodePatch::default(), + }); + } + + let at = if role == ImageRole::Inpainted { + 1.min(page_ref.nodes.len()) + } else { + page_ref.nodes.len() + }; + Ok(Op::AddNode { + page, + node: Node { + id: NodeId::new(), + transform: Transform::default(), + visible: role != ImageRole::Rendered, + kind: NodeKind::Image(ImageData { + role, + blob, + opacity: 1.0, + natural_width, + natural_height, + name: None, + }), + }, + at, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_data_image_url() { + let encoded = base64::engine::general_purpose::STANDARD.encode(b"png"); + let decoded = decode_data_image_url(&format!("data:image/png;base64,{encoded}")).unwrap(); + assert_eq!(decoded, Some(b"png".to_vec())); + } + + #[test] + fn ignores_non_data_url() { + assert!( + decode_data_image_url("https://example.test/image.png") + .unwrap() + .is_none() + ); + } +} diff --git a/koharu-app/src/app.rs b/koharu-app/src/app.rs index a9056d8624d88ee9e27a3106ea1a8d961a5d90a5..280c50f2e5497cdf6cd5248bafc07d781f2fa99d 100644 --- a/koharu-app/src/app.rs +++ b/koharu-app/src/app.rs @@ -23,6 +23,7 @@ use koharu_core::{AppEvent, DownloadProgress, JobSummary, LlmStateStatus}; use koharu_runtime::{ComputePolicy, RuntimeManager}; use tokio::sync::Mutex; +use crate::ai::AiManager; use crate::autosave::{self, AutosaveSignal}; use crate::bus::EventBus; use crate::config::AppConfig; @@ -62,6 +63,7 @@ pub struct App { pub jobs: Arc>, pub downloads: Arc>, pub bus: Arc, + pub ai: Arc, pub llm: Arc, pub renderer: Arc, /// Autosave handle (tx + join) for the currently-open session. `None` = no project open. @@ -92,6 +94,7 @@ impl App { ) -> Result { let backend = shared_llama_backend(&runtime)?; let llm = Arc::new(llm::Model::new((*runtime).clone(), cpu, backend)); + let ai = Arc::new(AiManager::new(&runtime)); let renderer = Arc::new(renderer::Renderer::new()?); Ok(Self { config: Arc::new(ArcSwap::from_pointee(config)), @@ -101,6 +104,7 @@ impl App { jobs: shared.jobs, downloads: shared.downloads, bus: shared.bus, + ai, llm, renderer, autosave: Mutex::new(None), diff --git a/koharu-app/src/lib.rs b/koharu-app/src/lib.rs index e766070a8b61b071f0db260e7a7a172027786243..0f8cda081f8cfb37dc07053cf3b14cb9c5261268 100644 --- a/koharu-app/src/lib.rs +++ b/koharu-app/src/lib.rs @@ -3,6 +3,7 @@ //! //! See [`crate::app::App`] for the entry point. +pub mod ai; pub mod app; pub mod archive; pub mod autosave; @@ -18,6 +19,7 @@ pub mod renderer; pub mod session; pub mod utils; +pub use ai::AiManager; pub use app::{App, AppSharedState}; pub use blobs::BlobStore; pub use config::AppConfig; diff --git a/koharu-rpc/src/api.rs b/koharu-rpc/src/api.rs index 7d4d385b20751d789c4050c238a12315cf6779e4..07d274fb854cbeb5b4f84641330300fe06908216 100644 --- a/koharu-rpc/src/api.rs +++ b/koharu-rpc/src/api.rs @@ -37,6 +37,7 @@ fn app_api() -> OpenApiRouter { .merge(routes::meta::router()) .merge(routes::fonts::router()) .merge(routes::llm::router()) + .merge(routes::ai::router()) .merge(routes::pipelines::router()) .merge(binary::router()) } diff --git a/koharu-rpc/src/routes/ai.rs b/koharu-rpc/src/routes/ai.rs new file mode 100644 index 0000000000000000000000000000000000000000..8c15559e87beaaac60521cc2949d4653dbdd705d --- /dev/null +++ b/koharu-rpc/src/routes/ai.rs @@ -0,0 +1,133 @@ +//! AI workflow routes. These are separate from `/llm/*` because Codex image +//! generation is not a translation model lifecycle concern. + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use axum::Json; +use axum::extract::State; +use axum::http::StatusCode; +use koharu_app::ai::{CodexAuthStatus, CodexDeviceLogin, CodexImageGenerationOptions}; +use koharu_core::{AppEvent, JobFinishedEvent, JobStatus, JobSummary}; +use serde::{Deserialize, Serialize}; +use utoipa_axum::{router::OpenApiRouter, routes}; +use uuid::Uuid; + +use crate::AppState; +use crate::error::{ApiError, ApiResult}; +use crate::routes::operations::{register_cancel, unregister_cancel}; + +pub fn router() -> OpenApiRouter { + OpenApiRouter::default() + .routes(routes!(get_codex_auth_status)) + .routes(routes!(start_codex_device_login)) + .routes(routes!(delete_codex_session)) + .routes(routes!(start_codex_image_generation)) +} + +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CodexImageGenerationResponse { + pub operation_id: String, +} + +#[utoipa::path( + get, + path = "/ai/codex/auth/status", + responses((status = 200, body = CodexAuthStatus)) +)] +async fn get_codex_auth_status(State(app): State) -> ApiResult> { + app.ai + .codex_auth_status() + .map(Json) + .map_err(ApiError::internal) +} + +#[utoipa::path( + post, + path = "/ai/codex/auth/device-code", + responses((status = 200, body = CodexDeviceLogin)) +)] +async fn start_codex_device_login( + State(app): State, +) -> ApiResult> { + app.ai + .start_codex_device_login() + .await + .map(Json) + .map_err(ApiError::internal) +} + +#[utoipa::path(delete, path = "/ai/codex/auth/session", responses((status = 204)))] +async fn delete_codex_session(State(app): State) -> ApiResult { + app.ai.logout_codex().map_err(ApiError::internal)?; + Ok(StatusCode::NO_CONTENT) +} + +#[utoipa::path( + post, + path = "/ai/codex/images", + request_body = CodexImageGenerationOptions, + responses((status = 200, body = CodexImageGenerationResponse)) +)] +async fn start_codex_image_generation( + State(app): State, + Json(req): Json, +) -> ApiResult> { + let session = app + .current_session() + .ok_or_else(|| ApiError::bad_request("no project open"))?; + + let operation_id = Uuid::new_v4().to_string(); + let cancel = Arc::new(AtomicBool::new(false)); + register_cancel(operation_id.clone(), cancel.clone()); + + app.jobs.insert( + operation_id.clone(), + JobSummary { + id: operation_id.clone(), + kind: "ai".to_string(), + status: JobStatus::Running, + error: None, + }, + ); + app.bus.publish(AppEvent::JobStarted { + id: operation_id.clone(), + kind: "ai".to_string(), + }); + + let app_c = app.clone(); + let session_c = session.clone(); + let op_id_c = operation_id.clone(); + tokio::spawn(async move { + let result = app_c + .ai + .generate_codex_page_image(session_c, req, cancel) + .await; + let (status, error) = match result { + Ok(()) => (JobStatus::Completed, None), + Err(e) if e.to_string().contains("cancelled") => (JobStatus::Cancelled, None), + Err(e) => { + tracing::warn!(operation_id = %op_id_c, "Codex image generation failed: {e:#}"); + (JobStatus::Failed, Some(format!("{e:#}"))) + } + }; + app_c.jobs.insert( + op_id_c.clone(), + JobSummary { + id: op_id_c.clone(), + kind: "ai".to_string(), + status, + error: error.clone(), + }, + ); + app_c.bus.publish(AppEvent::JobFinished(JobFinishedEvent { + id: op_id_c.clone(), + status, + error, + })); + unregister_cancel(&op_id_c); + }); + + Ok(Json(CodexImageGenerationResponse { operation_id })) +} diff --git a/koharu-rpc/src/routes/mod.rs b/koharu-rpc/src/routes/mod.rs index a4a20e91f59c15ce0e8eafa2d962d49e57c696f9..7865c8585de1230c663518767826737d3c7187d3 100644 --- a/koharu-rpc/src/routes/mod.rs +++ b/koharu-rpc/src/routes/mod.rs @@ -2,6 +2,7 @@ //! `OpenApiRouter` that can be merged into the top-level router in //! `api.rs`. +pub mod ai; pub mod config; pub mod downloads; pub mod fonts; diff --git a/koharu-rpc/tests/snapshots/openapi__openapi_paths_snapshot.snap b/koharu-rpc/tests/snapshots/openapi__openapi_paths_snapshot.snap index 79e2857ef0a19c368a6e5f01dbfe84c8a599a561..39c94328fb2954d0dc0a41c3adaa109a0540406e 100644 --- a/koharu-rpc/tests/snapshots/openapi__openapi_paths_snapshot.snap +++ b/koharu-rpc/tests/snapshots/openapi__openapi_paths_snapshot.snap @@ -4,6 +4,30 @@ assertion_line: 34 expression: paths --- [ + ( + "/ai/codex/auth/device-code", + [ + "post", + ], + ), + ( + "/ai/codex/auth/session", + [ + "delete", + ], + ), + ( + "/ai/codex/auth/status", + [ + "get", + ], + ), + ( + "/ai/codex/images", + [ + "post", + ], + ), ( "/blobs/{hash}", [ diff --git a/ui/components/Panels.tsx b/ui/components/Panels.tsx index e569586a4254cf13cb90a285740b6d4f3960c02d..bafc0039e2ed3c7688eb4d7b47b4b9e30c8d18dd 100644 --- a/ui/components/Panels.tsx +++ b/ui/components/Panels.tsx @@ -1,16 +1,26 @@ 'use client' -import { LayersIcon, SlidersHorizontalIcon } from 'lucide-react' +import { LayersIcon, SlidersHorizontalIcon, SparklesIcon, TypeIcon } from 'lucide-react' +import { useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' +import { AiPanel } from '@/components/panels/AiPanel' import { LayersPanel } from '@/components/panels/LayersPanel' import { RenderControlsPanel } from '@/components/panels/RenderControlsPanel' import { TextBlocksPanel } from '@/components/panels/TextBlocksPanel' import { ScrollArea } from '@/components/ui/scroll-area' import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' +import { useGetCodexAuthStatus } from '@/lib/api/default/default' export function Panels() { const { t } = useTranslation() + const { data: codexAuth } = useGetCodexAuthStatus() + const codexSignedIn = codexAuth?.signedIn === true + const [workTab, setWorkTab] = useState('text') + + useEffect(() => { + if (!codexSignedIn && workTab === 'ai') setWorkTab('text') + }, [codexSignedIn, workTab]) return (
@@ -57,8 +67,51 @@ export function Panels() { - {/* Text Blocks Section - takes remaining space */} - + + {codexSignedIn && ( + + + + + {t('layers.textBlocks')} + + + + + + {t('panels.ai')} + + + + )} + + + + + + {codexSignedIn && ( + + +
+ +
+ + + )} +
) } diff --git a/ui/components/SettingsDialog.tsx b/ui/components/SettingsDialog.tsx index 10ae428dffa44f8628a7de0e6288750b9b8130c7..978e8805894563dd715744abc2f95fc6509859a3 100644 --- a/ui/components/SettingsDialog.tsx +++ b/ui/components/SettingsDialog.tsx @@ -17,6 +17,11 @@ import { SaveIcon, RotateCcwIcon, AlertTriangleIcon, + CopyIcon, + ExternalLinkIcon, + LogInIcon, + LogOutIcon, + SparklesIcon, } from 'lucide-react' import { useTheme } from 'next-themes' import { Fragment, useEffect, useMemo, useRef, useState, type ReactNode } from 'react' @@ -57,10 +62,15 @@ import { getGetCatalogQueryKey as getGetLlmCatalogQueryKey, getMeta, patchConfig, + deleteCodexSession, + getGetCodexAuthStatusQueryKey, + startCodexDeviceLogin, + useGetCodexAuthStatus, } from '@/lib/api/default/default' import type { AppConfig, ConfigPatch, + CodexDeviceLogin, EngineCatalog as GetEngineCatalog200, LlmProviderCatalog, ProviderConfig, @@ -127,6 +137,7 @@ const TABS = [ { id: 'appearance', icon: PaletteIcon, labelKey: 'settings.appearance' }, { id: 'engines', icon: CpuIcon, labelKey: 'settings.engines' }, { id: 'providers', icon: KeyIcon, labelKey: 'settings.apiKeys' }, + { id: 'ai', icon: SparklesIcon, labelKey: 'settings.ai' }, { id: 'keybinds', icon: KeyboardIcon, labelKey: 'settings.keybinds' }, { id: 'runtime', icon: HardDriveIcon, labelKey: 'settings.runtime' }, { id: 'about', icon: InfoIcon, labelKey: 'settings.about' }, @@ -386,6 +397,7 @@ export function SettingsDialog({ }} /> )} + {tab === 'ai' && } {tab === 'runtime' && ( (null) + const [loginOpen, setLoginOpen] = useState(false) + const [busy, setBusy] = useState(false) + const [copied, setCopied] = useState(false) + const [actionError, setActionError] = useState(null) + const { data: auth, refetch } = useGetCodexAuthStatus() + + const loginStatus = auth?.login?.status + const signedIn = auth?.signedIn === true + + useEffect(() => { + if (!loginOpen && loginStatus !== 'pending') return + const id = window.setInterval(() => void refetch(), 2000) + return () => window.clearInterval(id) + }, [loginOpen, loginStatus, refetch]) + + useEffect(() => { + if (loginOpen && (signedIn || loginStatus === 'succeeded')) { + const id = window.setTimeout(() => setLoginOpen(false), 700) + return () => window.clearTimeout(id) + } + }, [loginOpen, loginStatus, signedIn]) + + const statusLabel = useMemo(() => { + if (signedIn) return auth?.accountId ? auth.accountId : t('ai.signedIn') + if (loginStatus === 'failed') return t('ai.signInFailed') + if (loginStatus === 'pending') return t('ai.signInPending') + return t('ai.signedOut') + }, [auth?.accountId, loginStatus, signedIn, t]) + + const invalidateAuth = () => + queryClient.invalidateQueries({ queryKey: getGetCodexAuthStatusQueryKey() }) + + const handleSignIn = async () => { + setBusy(true) + setActionError(null) + try { + const next = await startCodexDeviceLogin() + setLogin(next) + setCopied(false) + setLoginOpen(true) + void invalidateAuth() + void openExternalUrl(next.verificationUrl) + } catch (err) { + setActionError(String(err)) + } finally { + setBusy(false) + } + } + + const handleLogout = async () => { + setBusy(true) + setActionError(null) + try { + await deleteCodexSession() + await invalidateAuth() + } catch (err) { + setActionError(String(err)) + } finally { + setBusy(false) + } + } + + const handleCopyCode = async () => { + if (!login?.userCode || typeof navigator === 'undefined') return + await navigator.clipboard?.writeText(login.userCode) + setCopied(true) + window.setTimeout(() => setCopied(false), 1200) + } + + return ( +
+
+ {t('settings.codexTwoFactorDescription')} +
+
+
+
+
+ +
+
+
Codex
+
{statusLabel}
+
+
+ {signedIn ? ( + + ) : ( + + )} +
+ {(actionError || (auth?.login?.status === 'failed' && auth.login.error)) && ( +

+ {actionError || auth?.login?.error} +

+ )} +
+ + + + {t('ai.signInTitle')} + {t('ai.signIn')} +
+
+
+ {t('ai.userCode')} +
+
+ {login?.userCode ?? '...'} +
+
+ +
+ +
+ {signedIn || loginStatus === 'succeeded' ? ( + <> + + {t('ai.signInComplete')} + + ) : loginStatus === 'failed' ? ( + <> + + {auth?.login?.error ?? t('ai.signInFailed')} + + ) : ( + <> + + {t('ai.signInPending')} + + )} +
+ + +
+ ) +} + const SHORTCUT_ITEMS = [ { key: 'select', labelKey: 'toolRail.select' }, { key: 'block', labelKey: 'toolRail.block' }, diff --git a/ui/components/canvas/SubToolRail.tsx b/ui/components/canvas/SubToolRail.tsx index 08d564671f361268e67b1a58e65cee4cfb53bb22..d546ec9571c5df74401246ea13bc10f50b76e29b 100644 --- a/ui/components/canvas/SubToolRail.tsx +++ b/ui/components/canvas/SubToolRail.tsx @@ -30,84 +30,84 @@ export function SubToolRail() { {isBrushTool && ( -
- {/* Brush Size */} -
-

- {t('toolbar.brushSize')} -

-
- setLocalSize(vals[0] ?? localSize)} - onValueCommit={(vals) => setBrushConfig({ size: vals[0] ?? localSize })} - className='flex-1' - aria-labelledby='brush-size-label' - /> -
- +
+ {/* Brush Size */} +
+

+ {t('toolbar.brushSize')} +

+
+ setLocalSize(vals[0] ?? localSize)} + onValueCommit={(vals) => setBrushConfig({ size: vals[0] ?? localSize })} + className='flex-1' + aria-labelledby='brush-size-label' /> - +
+ + +
-
- {/* Color Picker Section */} - - {mode === 'brush' && ( - -
-

- {t('toolbar.brushColor')} -

-
-
- - )} - -
- + + )} + +
+ )} ) diff --git a/ui/components/panels/AiPanel.tsx b/ui/components/panels/AiPanel.tsx new file mode 100644 index 0000000000000000000000000000000000000000..d07ec562b9f07c9a084ff39448dbc66e247c8bda --- /dev/null +++ b/ui/components/panels/AiPanel.tsx @@ -0,0 +1,99 @@ +'use client' + +import { LoaderCircleIcon, SparklesIcon } from 'lucide-react' +import { useState } from 'react' +import { useTranslation } from 'react-i18next' + +import { Button } from '@/components/ui/button' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Textarea } from '@/components/ui/textarea' +import { useCurrentPage } from '@/hooks/useCurrentPage' +import { startCodexImageGeneration, useGetCodexAuthStatus } from '@/lib/api/default/default' +import { useEditorUiStore } from '@/lib/stores/editorUiStore' +import { useJobsStore } from '@/lib/stores/jobsStore' +import { usePreferencesStore } from '@/lib/stores/preferencesStore' + +export function AiPanel() { + const { t } = useTranslation() + const page = useCurrentPage() + const prompt = usePreferencesStore((s) => s.codexImagePrompt) + const setPrompt = usePreferencesStore((s) => s.setCodexImagePrompt) + const model = usePreferencesStore((s) => s.codexImageModel) + const setModel = usePreferencesStore((s) => s.setCodexImageModel) + const setShowRenderedImage = useEditorUiStore((s) => s.setShowRenderedImage) + const showError = useEditorUiStore((s) => s.showError) + const [busy, setBusy] = useState(false) + const { data: auth } = useGetCodexAuthStatus() + const isProcessing = useJobsStore((s) => + Object.values(s.jobs).some((job) => job.status === 'running'), + ) + + const signedIn = auth?.signedIn === true + const promptReady = !!prompt?.trim() + const modelValue = model?.trim() || 'gpt-5.5' + const canGenerate = signedIn && !!page && promptReady && !isProcessing && !busy + + const handleGenerate = async () => { + if (!signedIn || !page || !promptReady) return + setBusy(true) + try { + setShowRenderedImage(true) + await startCodexImageGeneration({ + pageId: page.id, + prompt: prompt!.trim(), + model: modelValue, + quality: 'high', + }) + } catch (err) { + showError(String(err)) + } finally { + setBusy(false) + } + } + + if (!signedIn) return null + + return ( +
+
+ + setModel(event.target.value || undefined)} + className='h-7 px-2 text-xs' + /> +
+ +
+ +