feat(gemini): implement default safety settings and image data URL support

Default safety settings are now enforced across all Gemini calls to prevent unintentional content filtering by the API's aggressive defaults.

- The RotatingClient applies a standard set of safety thresholds (HARM_CATEGORY_* set to OFF or BLOCK_NONE) if the user has not provided explicit settings or if their settings are incomplete.
- The `GeminiProvider` conversion logic is enhanced to merge missing categories when converting generic safety settings (dict) or when receiving direct Gemini-style lists.
- The `GeminiCliProvider` now supports multimodal inputs by parsing image content provided as data URLs (`data:image/...`) and converting them to `inlineData` parts.
- Usage metadata reporting in the CLI provider is updated to include `thoughtsTokenCount` within the prompt tokens and optionally detail reasoning tokens.
- Function tool schema translation for the CLI provider is corrected to use `parametersJsonSchema` and enforce default schemas.
- Function call IDs during streaming are now generated with nanosecond precision for guaranteed uniqueness.

src/rotator_library/client.py

@@ -332,6 +332,53 @@ class RotatingClient:
 
         return kwargs
 
+    def _apply_default_safety_settings(self, litellm_kwargs: Dict[str, Any], provider: str):
+        """
+        Ensure default Gemini safety settings are present when calling the Gemini provider.
+        This will not override any explicit settings provided by the request. It accepts
+        either OpenAI-compatible generic `safety_settings` (dict) or direct Gemini-style
+        `safetySettings` (list of dicts). Missing categories will be added with safe defaults.
+        """
+        if provider != "gemini":
+            return
+
+        # Generic defaults (openai-compatible style)
+        default_generic = {
+            "harassment": "OFF",
+            "hate_speech": "OFF",
+            "sexually_explicit": "OFF",
+            "dangerous_content": "OFF",
+            "civic_integrity": "BLOCK_NONE",
+        }
+
+        # Gemini defaults (direct Gemini format)
+        default_gemini = [
+            {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "OFF"},
+            {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "OFF"},
+            {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "OFF"},
+            {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "OFF"},
+            {"category": "HARM_CATEGORY_CIVIC_INTEGRITY", "threshold": "BLOCK_NONE"},
+        ]
+
+        # If generic form is present, ensure missing generic keys are filled in
+        if "safety_settings" in litellm_kwargs and isinstance(litellm_kwargs["safety_settings"], dict):
+            for k, v in default_generic.items():
+                if k not in litellm_kwargs["safety_settings"]:
+                    litellm_kwargs["safety_settings"][k] = v
+            return
+
+        # If Gemini form is present, ensure missing gemini categories are appended
+        if "safetySettings" in litellm_kwargs and isinstance(litellm_kwargs["safetySettings"], list):
+            present = {item.get("category") for item in litellm_kwargs["safetySettings"] if isinstance(item, dict)}
+            for d in default_gemini:
+                if d["category"] not in present:
+                    litellm_kwargs["safetySettings"].append(d)
+            return
+
+        # Neither present: set generic defaults so provider conversion will translate them
+        if "safety_settings" not in litellm_kwargs and "safetySettings" not in litellm_kwargs:
+            litellm_kwargs["safety_settings"] = default_generic.copy()
+
     def get_oauth_credentials(self) -> Dict[str, List[str]]:
         return self.oauth_credentials
 
@@ -666,6 +713,13 @@ class RotatingClient:
 
                     provider_instance = self._get_provider_instance(provider)
                     if provider_instance:
+                        # Ensure default Gemini safety settings are present (without overriding request)
+                        try:
+                            self._apply_default_safety_settings(litellm_kwargs, provider)
+                        except Exception:
+                            # If anything goes wrong here, avoid breaking the request flow.
+                            lib_logger.debug("Could not apply default safety settings; continuing.")
+
                         if "safety_settings" in litellm_kwargs:
                             converted_settings = (
                                 provider_instance.convert_safety_settings(
@@ -1138,6 +1192,12 @@ class RotatingClient:
 
                     provider_instance = self._get_provider_instance(provider)
                     if provider_instance:
+                        # Ensure default Gemini safety settings are present (without overriding request)
+                        try:
+                            self._apply_default_safety_settings(litellm_kwargs, provider)
+                        except Exception:
+                            lib_logger.debug("Could not apply default safety settings for streaming path; continuing.")
+
                         if "safety_settings" in litellm_kwargs:
                             converted_settings = (
                                 provider_instance.convert_safety_settings(

src/rotator_library/providers/gemini_cli_provider.py

@@ -201,13 +201,35 @@ class GeminiCliProvider(GeminiAuthBase, ProviderInterface):
             gemini_role = "model" if role == "assistant" else "tool" if role == "tool" else "user"
 
             if role == "user":
-                text_content = ""
                 if isinstance(content, str):
-                    text_content = content
+                    # Simple text content
+                    if content:
+                        parts.append({"text": content})
                 elif isinstance(content, list):
-                    text_content = "\n".join(p.get("text", "") for p in content if p.get("type") == "text")
-                if text_content:
-                    parts.append({"text": text_content})
+                    # Multi-part content (text, images, etc.)
+                    for item in content:
+                        if item.get("type") == "text":
+                            text = item.get("text", "")
+                            if text:
+                                parts.append({"text": text})
+                        elif item.get("type") == "image_url":
+                            # Handle image data URLs
+                            image_url = item.get("image_url", {}).get("url", "")
+                            if image_url.startswith("data:"):
+                                try:
+                                    # Parse: data:image/png;base64,iVBORw0KG...
+                                    header, data = image_url.split(",", 1)
+                                    mime_type = header.split(":")[1].split(";")[0]
+                                    parts.append({
+                                        "inlineData": {
+                                            "mimeType": mime_type,
+                                            "data": data
+                                        }
+                                    })
+                                except Exception as e:
+                                    lib_logger.warning(f"Failed to parse image data URL: {e}")
+                            else:
+                                lib_logger.warning(f"Non-data-URL images not supported: {image_url[:50]}...")
 
             elif role == "assistant":
                 if isinstance(content, str):
@@ -292,12 +314,15 @@ class GeminiCliProvider(GeminiAuthBase, ProviderInterface):
 
             if 'functionCall' in part:
                 function_call = part['functionCall']
+                function_name = function_call.get('name', 'unknown')
+                # Generate unique ID with nanosecond precision (matching Go implementation)
+                tool_call_id = f"call_{function_name}_{int(time.time() * 1_000_000_000)}"
                 delta['tool_calls'] = [{
                     "index": 0,
-                    "id": f"tool-call-{time.time()}",
+                    "id": tool_call_id,
                     "type": "function",
                     "function": {
-                        "name": function_call.get('name'),
+                        "name": function_name,
                         "arguments": json.dumps(function_call.get('args', {}))
                     }
                 }]
@@ -326,11 +351,21 @@ class GeminiCliProvider(GeminiAuthBase, ProviderInterface):
 
             if 'usageMetadata' in response_data:
                 usage = response_data['usageMetadata']
+                prompt_tokens = usage.get("promptTokenCount", 0)
+                thoughts_tokens = usage.get("thoughtsTokenCount", 0)
+                candidate_tokens = usage.get("candidatesTokenCount", 0)
+
                 openai_chunk["usage"] = {
-                    "prompt_tokens": usage.get("promptTokenCount", 0),
-                    "completion_tokens": usage.get("candidatesTokenCount", 0),
+                    "prompt_tokens": prompt_tokens + thoughts_tokens,  # Include thoughts in prompt tokens
+                    "completion_tokens": candidate_tokens,
                     "total_tokens": usage.get("totalTokenCount", 0),
                 }
+
+                # Add reasoning tokens details if present (OpenAI o1 format)
+                if thoughts_tokens > 0:
+                    if "completion_tokens_details" not in openai_chunk["usage"]:
+                        openai_chunk["usage"]["completion_tokens_details"] = {}
+                    openai_chunk["usage"]["completion_tokens_details"]["reasoning_tokens"] = thoughts_tokens
             
             yield openai_chunk
 
@@ -482,9 +517,15 @@ class GeminiCliProvider(GeminiAuthBase, ProviderInterface):
                 # The Gemini CLI API does not support the 'strict' property.
                 new_function.pop("strict", None)
 
-                if "parameters" in new_function and isinstance(new_function["parameters"], dict):
-                    new_function["parameters"] = self._gemini_cli_transform_schema(new_function["parameters"])
-                
+                # Gemini CLI expects 'parametersJsonSchema' instead of 'parameters'
+                if "parameters" in new_function:
+                    schema = self._gemini_cli_transform_schema(new_function["parameters"])
+                    new_function["parametersJsonSchema"] = schema
+                    del new_function["parameters"]
+                elif "parametersJsonSchema" not in new_function:
+                    # Set default empty schema if neither exists
+                    new_function["parametersJsonSchema"] = {"type": "object", "properties": {}}
+
                 transformed_declarations.append(new_function)
         
         return transformed_declarations
@@ -548,8 +589,7 @@ class GeminiCliProvider(GeminiAuthBase, ProviderInterface):
             }
             if "temperature" in kwargs:
                 gen_config["temperature"] = kwargs["temperature"]
-            else:
-                gen_config["temperature"] = 0.7
+            # No else - let Gemini use its default temperature (matches OpenAI behavior)
             if "top_k" in kwargs:
                 gen_config["topK"] = kwargs["top_k"]
             if "top_p" in kwargs:
@@ -583,7 +623,17 @@ class GeminiCliProvider(GeminiAuthBase, ProviderInterface):
                 tool_config = self._translate_tool_choice(kwargs["tool_choice"])
                 if tool_config:
                     request_payload["request"]["toolConfig"] = tool_config
-            
+
+            # Add default safety settings to prevent content filtering
+            if "safetySettings" not in request_payload["request"]:
+                request_payload["request"]["safetySettings"] = [
+                    {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "OFF"},
+                    {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "OFF"},
+                    {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "OFF"},
+                    {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "OFF"},
+                    {"category": "HARM_CATEGORY_CIVIC_INTEGRITY", "threshold": "BLOCK_NONE"},
+                ]
+
             # Log the final payload for debugging and to the dedicated file
             #lib_logger.debug(f"Gemini CLI Request Payload: {json.dumps(request_payload, indent=2)}")
             file_logger.log_request(request_payload)

src/rotator_library/providers/gemini_provider.py

@@ -32,23 +32,57 @@ class GeminiProvider(ProviderInterface):
         Converts generic safety settings to the Gemini-specific format.
         """
         if not settings:
-            return []
+            # Return full defaults if nothing provided
+            return [
+                {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "OFF"},
+                {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "OFF"},
+                {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "OFF"},
+                {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "OFF"},
+                {"category": "HARM_CATEGORY_CIVIC_INTEGRITY", "threshold": "BLOCK_NONE"},
+            ]
+
+        # Default gemini-format settings for merging
+        default_gemini = {
+            "HARM_CATEGORY_HARASSMENT": "OFF",
+            "HARM_CATEGORY_HATE_SPEECH": "OFF",
+            "HARM_CATEGORY_SEXUALLY_EXPLICIT": "OFF",
+            "HARM_CATEGORY_DANGEROUS_CONTENT": "OFF",
+            "HARM_CATEGORY_CIVIC_INTEGRITY": "BLOCK_NONE",
+        }
 
+        # If the caller already provided Gemini-style list, merge defaults without overwriting
+        if isinstance(settings, list):
+            existing = {item.get("category"): item for item in settings if isinstance(item, dict) and item.get("category")}
+            merged = list(settings)
+            for cat, thr in default_gemini.items():
+                if cat not in existing:
+                    merged.append({"category": cat, "threshold": thr})
+            return merged
+
+        # Otherwise assume a generic mapping (dict) and convert
         gemini_settings = []
         category_map = {
             "harassment": "HARM_CATEGORY_HARASSMENT",
             "hate_speech": "HARM_CATEGORY_HATE_SPEECH",
             "sexually_explicit": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
             "dangerous_content": "HARM_CATEGORY_DANGEROUS_CONTENT",
+            "civic_integrity": "HARM_CATEGORY_CIVIC_INTEGRITY",
         }
 
         for generic_category, threshold in settings.items():
             if generic_category in category_map:
+                thr = (threshold or "").upper()
                 gemini_settings.append({
                     "category": category_map[generic_category],
-                    "threshold": threshold.upper()
+                    "threshold": thr if thr else default_gemini[category_map[generic_category]]
                 })
-        
+
+        # Add any missing defaults
+        present = {s["category"] for s in gemini_settings}
+        for cat, thr in default_gemini.items():
+            if cat not in present:
+                gemini_settings.append({"category": cat, "threshold": thr})
+
         return gemini_settings
 
     def handle_thinking_parameter(self, payload: Dict[str, Any], model: str):