Upload oculus_unified_model/configuration_oculus.py with huggingface_hub

oculus_unified_model/configuration_oculus.py

@@ -2,6 +2,7 @@
 Oculus Configuration
 
 HuggingFace-compatible configuration for the unified Oculus model.
+Supports Isaac 0.2 features: Thinking Traces, Focus/Zoom, Structured Output, Complex OCR, Desktop UI.
 """
 
 from typing import Optional, Dict, Any, List
@@ -11,104 +12,142 @@ from transformers import PretrainedConfig
 class OculusConfig(PretrainedConfig):
     """
     Configuration class for Oculus vision-language model.
-    
-    Args:
-        vision_config: Configuration for vision encoders
-        projector_config: Configuration for vision-to-language projector
-        text_config: Configuration for language model
-        reasoning_enabled: Whether to enable thinking traces
-        output_mode: Default output mode ("text", "point", "box", "polygon")
+
+    Architecture: DINOv3 + SigLIP2 + LFM2.5-1.2B
+
+    Isaac 0.2 Features:
+    - Reasoning via Thinking Traces
+    - Perceptive Tool Calling + Focus (Zoom & Crop)
+    - Structured Outputs (JSON)
+    - Complex OCR
+    - Desktop UI Understanding
     """
-    
+
     model_type = "oculus"
-    
+
     def __init__(
         self,
-        # Vision encoder settings
-        dinov3_model_id: str = "facebook/dinov2-large",
-        siglip_model_id: str = "google/siglip-base-patch16-224",
-        dinov3_hidden_size: int = 1280,  # DINOv3 ViT-H/16+ output dim
-        siglip_hidden_size: int = 768,   # SigLIP2 base output dim
-        
+        # Vision encoder settings (DINOv3 + SigLIP2)
+        dinov3_model_id: str = "facebook/dinov3-vitl16-pretrain-lvd1689m",
+        siglip_model_id: str = "google/siglip2-so400m-patch16-naflex",
+        dinov3_hidden_size: int = 1024,   # DINOv3 ViT-L/16 output dim
+        siglip_hidden_size: int = 1152,   # SigLIP2 SO400M output dim
+
         # Projector settings
-        projector_hidden_dim: int = 2048,
+        projector_hidden_dim: int = 4352,
         num_vision_tokens: int = 64,
-        
-        # Language model settings
-        text_model_id: str = "Salesforce/blip-image-captioning-base",
+
+        # Language model settings (LFM2.5-1.2B)
+        lm_model_id: str = "LiquidAI/LFM2.5-1.2B-Base",
         lm_hidden_size: int = 1536,
         vocab_size: int = 131072,
         max_position_embeddings: int = 32768,
-        
-        # Reasoning settings
+
+        # Reasoning / Thinking Traces
         reasoning_enabled: bool = True,
         thinking_token: str = "<think>",
         thinking_end_token: str = "</think>",
         max_thinking_tokens: int = 256,
-        
+        thinking_style: str = "structured",  # "structured", "verbose", "minimal"
+
+        # Focus System (Perceptive Tool Calling)
+        enable_focus: bool = True,
+        focus_token: str = "<focus>",
+        focus_end_token: str = "</focus>",
+        max_focus_regions: int = 4,
+        focus_min_size: int = 64,  # Minimum crop size in pixels
+        auto_focus_threshold: float = 0.7,  # Confidence threshold to trigger focus
+
+        # Structured Output
+        structured_output_enabled: bool = True,
+        json_token: str = "<json>",
+        json_end_token: str = "</json>",
+
+        # OCR Settings
+        ocr_enabled: bool = True,
+        ocr_languages: List[str] = None,  # None = auto-detect
+        ocr_confidence_threshold: float = 0.5,
+
+        # Desktop UI Understanding
+        ui_understanding_enabled: bool = True,
+        ui_element_classes: int = 50,  # button, text_field, checkbox, etc.
+
         # Output mode settings
-        output_mode: str = "text",  # "text", "point", "box", "polygon"
+        output_mode: str = "text",  # "text", "point", "box", "polygon", "json"
         num_detection_classes: int = 80,
         num_segmentation_classes: int = 150,
-        
+
         # Generation settings
         max_new_tokens: int = 512,
         temperature: float = 0.7,
         top_p: float = 0.95,
-        
-        # Tool calling / Focus system
-        enable_focus: bool = True,
-        focus_token: str = "<focus>",
-        focus_end_token: str = "</focus>",
-        
+
         **kwargs
     ):
         super().__init__(**kwargs)
-        
-        # Vision
+
+        # Vision encoders
         self.dinov3_model_id = dinov3_model_id
         self.siglip_model_id = siglip_model_id
         self.dinov3_hidden_size = dinov3_hidden_size
         self.siglip_hidden_size = siglip_hidden_size
         self.fused_vision_dim = dinov3_hidden_size + siglip_hidden_size
-        
+
         # Projector
         self.projector_hidden_dim = projector_hidden_dim
         self.num_vision_tokens = num_vision_tokens
-        
-        # Language model
-        self.text_model_id = text_model_id
+
+        # Language model (LFM2.5)
+        self.lm_model_id = lm_model_id
         self.lm_hidden_size = lm_hidden_size
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings
-        
-        # Reasoning
+
+        # Reasoning / Thinking Traces
         self.reasoning_enabled = reasoning_enabled
         self.thinking_token = thinking_token
         self.thinking_end_token = thinking_end_token
         self.max_thinking_tokens = max_thinking_tokens
-        
+        self.thinking_style = thinking_style
+
+        # Focus System
+        self.enable_focus = enable_focus
+        self.focus_token = focus_token
+        self.focus_end_token = focus_end_token
+        self.max_focus_regions = max_focus_regions
+        self.focus_min_size = focus_min_size
+        self.auto_focus_threshold = auto_focus_threshold
+
+        # Structured Output
+        self.structured_output_enabled = structured_output_enabled
+        self.json_token = json_token
+        self.json_end_token = json_end_token
+
+        # OCR
+        self.ocr_enabled = ocr_enabled
+        self.ocr_languages = ocr_languages or ["en"]
+        self.ocr_confidence_threshold = ocr_confidence_threshold
+
+        # Desktop UI
+        self.ui_understanding_enabled = ui_understanding_enabled
+        self.ui_element_classes = ui_element_classes
+
         # Output modes
         self.output_mode = output_mode
         self.num_detection_classes = num_detection_classes
         self.num_segmentation_classes = num_segmentation_classes
-        
+
         # Generation
         self.max_new_tokens = max_new_tokens
         self.temperature = temperature
         self.top_p = top_p
-        
-        # Focus system
-        self.enable_focus = enable_focus
-        self.focus_token = focus_token
-        self.focus_end_token = focus_end_token
-    
+
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
         """Load config from pretrained path."""
         config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
         return cls.from_dict(config_dict, **kwargs)
-    
+
     def to_dict(self) -> Dict[str, Any]:
         """Serialize config to dictionary."""
         output = super().to_dict()