Upload cleaned source files

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +0,0 @@
----
-title: Elite 17
-emoji: 🐢
-colorFrom: purple
-colorTo: green
-sdk: gradio
-sdk_version: 6.1.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,297 +1,285 @@
-import cv2
-import numpy as np
-import json
 import os
-from pathlib import Path
-from fastapi import FastAPI, File, UploadFile, Form, HTTPException
+import sys
+import time
+import subprocess
+import numpy as np
+from PIL import Image
+from io import BytesIO
+import requests
+import threading
+
+# FastAPI imports
+from fastapi import FastAPI, UploadFile, File, HTTPException, Form
 from fastapi.responses import JSONResponse
-from typing import Dict, Any, Tuple, Optional, Union
-import io
-import aiohttp
 import uvicorn
-from urllib.parse import urlparse
-
-# --- Original Cursor Detection Functions (Adapted for Server) ---
-
-def to_rgb(img: np.ndarray) -> Optional[np.ndarray]:
-    """Converts image to BGR format (3 channels). Handles None input."""
-    if img is None:
-        return None
-    if len(img.shape) == 2:
-        # Grayscale to BGR
-        return cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
-    if img.shape[2] == 4:
-        # BGRA to BGR (removes alpha channel)
-        return cv2.cvtColor(img, cv2.COLOR_BGRA2BGR)
-    # Already BGR or RGB (assuming OpenCV reads as BGR)
-    return img
 
-def get_mask_from_alpha(template_img: np.ndarray) -> Optional[np.ndarray]:
-    """Extracts a mask from the alpha channel of a 4-channel image."""
-    if template_img is not None and len(template_img.shape) == 3 and template_img.shape[2] == 4:
-        # Create a mask where alpha is greater than 0
-        return (template_img[:, :, 3] > 0).astype(np.uint8) * 255
-    return None
-
-def detect_cursor_in_frame_multi(
-    frame: np.ndarray, 
-    cursor_templates: Dict[str, np.ndarray], 
-    threshold: float = 0.8
-) -> Tuple[Optional[Tuple[int, int]], float, Optional[str]]:
-    """
-    Detects the best matching cursor template in a single frame.
-    Returns (position, confidence, template_name).
-    """
-    best_pos = None
-    best_conf = -1.0
-    best_template_name = None
-    frame_rgb = to_rgb(frame)
-    
-    if frame_rgb is None:
-        return None, -1.0, None
+# 1. Environment Setup & Dependency Installation
+def setup_environment():
+    print("--- Setting up environment ---")
+    dependencies = ["huggingface_hub", "onnxruntime", "transformers", "pillow", "numpy"]
+    try:
+        import huggingface_hub
+        import onnxruntime
+        import transformers
+        print("Dependencies already satisfied.")
+    except ImportError:
+        print("Installing dependencies...")
+        subprocess.check_call([sys.executable, "-m", "pip", "install"] + dependencies)
+
+# 2. Model Download
+def download_model(repo_id="Heliosoph/florence-2-base-ft-quantized-onnx", local_dir="florence2_quantized"):
+    from huggingface_hub import snapshot_download
+    if not os.path.exists(local_dir):
+        print(f"--- Downloading model from {repo_id} ---")
+        snapshot_download(repo_id=repo_id, local_dir=local_dir)
+        print("Download complete.")
+    else:
+        print(f"Model directory '{local_dir}' already exists.")
 
-    for template_name, cursor_template in cursor_templates.items():
-        template_rgb = to_rgb(cursor_template)
-        mask = get_mask_from_alpha(cursor_template)
+# 3. Inference Engine
+class Florence2ONNXEngine:
+    def __init__(self, model_dir="florence2_quantized"):
+        import onnxruntime as ort
+        from transformers import CLIPImageProcessor, AutoTokenizer
+        
+        self.model_dir = model_dir
+        print("--- Initializing ONNX Engine ---")
         
-        if template_rgb is None or template_rgb.shape[2] != frame_rgb.shape[2]:
-            # print(f"[WARN] Skipping template {template_name} due to channel mismatch or load error.")
-            continue
+        # Load processors
+        self.image_processor = CLIPImageProcessor.from_pretrained("microsoft/Florence-2-base-ft")
+        self.tokenizer = AutoTokenizer.from_pretrained("facebook/bart-base")
+        
+        # Load ONNX sessions
+        providers = ['CPUExecutionProvider']
+        self.vision_session = ort.InferenceSession(os.path.join(model_dir, 'vision_encoder_quantized.onnx'), providers=providers)
+        self.embed_session = ort.InferenceSession(os.path.join(model_dir, 'embed_tokens_quantized.onnx'), providers=providers)
+        self.encoder_session = ort.InferenceSession(os.path.join(model_dir, 'encoder_model_quantized.onnx'), providers=providers)
+        self.decoder_session = ort.InferenceSession(os.path.join(model_dir, 'decoder_model_quantized.onnx'), providers=providers)
+        print("✓ Florence-2 ONNX Engine initialized successfully")
+
+    def generate_caption(self, image_path=None, image_array=None, task_prompt="<MORE_DETAILED_CAPTION>", max_new_tokens=1024):
+        """Generate caption from image path or PIL Image object"""
+        if image_path:
+            image = Image.open(image_path).convert("RGB")
+        elif image_array is not None and isinstance(image_array, Image.Image):
+            image = image_array.convert("RGB")
+        else:
+            raise ValueError("Either image_path or image_array must be provided")
         
-        # Ensure template is smaller than or equal to the frame
-        if template_rgb.shape[0] > frame_rgb.shape[0] or template_rgb.shape[1] > frame_rgb.shape[1]:
-            # print(f"[WARN] Skipping template {template_name}: template larger than frame.")
-            continue
+        print(f"--- Running Inference (Max Tokens: {max_new_tokens}) ---")
+        pixel_values = self.image_processor(images=image, return_tensors="np")['pixel_values']
+        
+        # Map specific prompts to descriptive strings if needed
+        prompt_map = {
+            "<CAPTION>": "What does the image describe?",
+            "<DETAILED_CAPTION>": "Describe this image in detail.",
+            "<MORE_DETAILED_CAPTION>": "Describe this image in great detail with every object and background."
+        }
+        text_prompt = prompt_map.get(task_prompt, task_prompt)
+        input_ids = self.tokenizer(text_prompt, return_tensors="np")['input_ids']
 
-        try:
-            # Match template. Use mask for non-rectangular templates.
-            result = cv2.matchTemplate(frame_rgb, template_rgb, cv2.TM_CCOEFF_NORMED, mask=mask)
-        except Exception as e:
-            # print(f"[WARN] matchTemplate failed for {template_name}: {e}")
-            continue
+        # 1. Vision Features
+        start_time = time.time()
+        image_features = self.vision_session.run(None, {'pixel_values': pixel_values})[0]
         
-        _, max_val, _, max_loc = cv2.minMaxLoc(result)
+        # 2. Text Embeddings
+        text_embeds = self.embed_session.run(None, {'input_ids': input_ids})[0]
         
-        if max_val > best_conf:
-            best_conf = max_val
-            if max_val >= threshold:
-                cursor_w, cursor_h = template_rgb.shape[1], template_rgb.shape[0]
-                # Calculate center position of the detected area
-                cursor_x = max_loc[0] + cursor_w // 2
-                cursor_y = max_loc[1] + cursor_h // 2
-                best_pos = (cursor_x, cursor_y)
-                best_template_name = template_name
+        # 3. Encoder Fusion
+        combined_embeds = np.concatenate([image_features, text_embeds], axis=1)
+        attention_mask = np.ones((1, combined_embeds.shape[1]), dtype=np.int64)
+        encoder_outputs = self.encoder_session.run(None, {
+            'inputs_embeds': combined_embeds,
+            'attention_mask': attention_mask
+        })
+        last_hidden_state = encoder_outputs[0]
+
+        # 4. Autoregressive Decoding with Repetition Penalty
+        generated_ids = [2] # BART Start Token
+        min_new_tokens = 250  # Enforce minimum generation
+        repetition_penalty = 1.5  # Penalize repeated tokens
+        
+        for i in range(max_new_tokens):
+            decoder_input_ids = np.array([generated_ids], dtype=np.int64)
+            decoder_embeds = self.embed_session.run(None, {'input_ids': decoder_input_ids})[0]
+            
+            logits = self.decoder_session.run(None, {
+                'inputs_embeds': decoder_embeds,
+                'encoder_hidden_states': last_hidden_state,
+                'encoder_attention_mask': attention_mask
+            })[0]
+            
+            # Apply repetition penalty to recently generated tokens
+            current_logits = logits[0, -1, :].copy()
+            for prev_token in set(generated_ids[-50:]):  # Check last 50 tokens
+                if current_logits[prev_token] > 0:
+                    current_logits[prev_token] /= repetition_penalty
+                else:
+                    current_logits[prev_token] *= repetition_penalty
+            
+            next_token = np.argmax(current_logits)
+            # Only allow EOS token after minimum generation
+            if next_token == 2 and i < min_new_tokens:
+                # Force a different token by reducing EOS probability
+                current_logits[2] = -1e9
+                next_token = np.argmax(current_logits)
+            if next_token == 2: break # EOS Token
+            generated_ids.append(next_token)
+            
+            if (i + 1) % 50 == 0:
+                print(f"Generated {i+1} tokens...")
+
+        end_time = time.time()
+        caption = self.tokenizer.decode(generated_ids, skip_special_tokens=True)
+        
+        print(f"Inference complete in {end_time - start_time:.2f}s")
+        return caption
 
-    if best_conf >= threshold:
-        return best_pos, best_conf, best_template_name
-    return None, best_conf, None
 
-async def download_image_from_url(url: str) -> bytes:
-    """Download image from URL and return as bytes."""
-    async with aiohttp.ClientSession() as session:
-        async with session.get(url) as response:
-            if response.status != 200:
-                raise HTTPException(
-                    status_code=400,
-                    detail=f"Failed to fetch image from URL. Status code: {response.status}"
-                )
-            return await response.read()
+# Global engine instance
+engine = None
 
-# --- Server Setup ---
+def initialize_engine():
+    """Initialize the Florence2 ONNX engine"""
+    global engine
+    setup_environment()
+    download_model()
+    engine = Florence2ONNXEngine()
 
+# FastAPI app setup
 app = FastAPI(
-    title="Cursor Tracker API",
-    description="API to detect and track mouse cursors in uploaded images using template matching."
+    title="Florence-2 ONNX Image Captioning Server",
+    description="Auto-captions images using Florence-2 ONNX models"
 )
 
-# Global variable to store loaded templates
-CURSOR_TEMPLATES: Dict[str, np.ndarray] = {}
-CURSOR_TEMPLATES_DIR = Path("cursors")
-
-def load_cursor_templates():
-    """Loads all cursor templates from the specified directory."""
-    global CURSOR_TEMPLATES
-    if CURSOR_TEMPLATES:
-        print("Templates already loaded.")
-        return
-
-    print(f"Loading cursor templates from: {CURSOR_TEMPLATES_DIR}")
-    
-    if not CURSOR_TEMPLATES_DIR.is_dir():
-        print(f"Error: Template directory not found at {CURSOR_TEMPLATES_DIR}")
-        return
-
-    for template_file in CURSOR_TEMPLATES_DIR.glob('*.png'):
-        # Load image with alpha channel (IMREAD_UNCHANGED)
-        template_img = cv2.imread(str(template_file), cv2.IMREAD_UNCHANGED)
-        if template_img is not None:
-            CURSOR_TEMPLATES[template_file.name] = template_img
-        else:
-            print(f"[WARN] Could not load template: {template_file.name}")
+def load_image_from_url(image_url: str) -> Image.Image:
+    """Load an image from a URL."""
+    try:
+        response = requests.get(image_url, timeout=30)
+        response.raise_for_status()
+        image = Image.open(BytesIO(response.content))
+        return image.convert('RGB')
+    except Exception as e:
+        raise ValueError(f"Error loading image from URL: {e}")
 
-    if not CURSOR_TEMPLATES:
-        print(f"FATAL: No cursor templates found in: {CURSOR_TEMPLATES_DIR}")
-    else:
-        print(f"Successfully loaded {len(CURSOR_TEMPLATES)} templates.")
+def load_image_from_bytes(image_bytes: bytes) -> Image.Image:
+    """Load an image from bytes."""
+    try:
+        image = Image.open(BytesIO(image_bytes))
+        return image.convert('RGB')
+    except Exception as e:
+        raise ValueError(f"Error loading image from bytes: {e}")
 
-@app.on_event("startup")
-async def startup_event():
-    """Load templates when the application starts."""
-    load_cursor_templates()
+# API Endpoints
 
 @app.get("/")
 async def root():
-    """Simple root endpoint for health check."""
-    return {"message": "Cursor Tracker API is running. Use /track_cursor to upload an image."}
-
-@app.post("/track_cursor")
-async def track_cursor_endpoint(
-    file: UploadFile = File(...),
-    threshold: float = Form(0.8)
-):
-    """
-    Accepts an image file and returns the detected cursor position and details.
-    """
-    if not CURSOR_TEMPLATES:
-        raise HTTPException(
-            status_code=503, 
-            detail="Cursor templates are not loaded. Server initialization failed."
-        )
-
-    # 1. Read image file content
-    content = await file.read()
+    """Root endpoint - shows server status"""
+    return {
+        "name": "Florence-2 ONNX Image Captioning Server",
+        "status": "running",
+        "model": "Florence-2-base-ft-quantized-onnx",
+        "model_loaded": engine is not None,
+        "endpoints": {
+            "GET /health": "Health check",
+            "GET /analyze": "Analyze image from URL",
+            "POST /analyze": "Analyze uploaded image",
+        }
+    }
+
+@app.get("/health")
+async def health():
+    """Health check endpoint"""
+    return {
+        "status": "healthy" if engine is not None else "initializing",
+        "model": "Florence-2-base-ft-quantized-onnx",
+        "model_loaded": engine is not None,
+    }
+
+@app.get("/analyze")
+async def analyze_get(image_url: str = None):
+    """Analyze an image by URL.
     
-    # 2. Convert file content to OpenCV image format
-    np_array = np.frombuffer(content, np.uint8)
-    frame = cv2.imdecode(np_array, cv2.IMREAD_UNCHANGED)
+    Usage: /analyze?image_url=https://example.com/image.jpg
+    """
+    try:
+        if engine is None:
+            raise HTTPException(status_code=503, detail="Model not initialized")
+        
+        if not image_url:
+            raise HTTPException(status_code=400, detail="image_url query parameter is required")
+        
+        # Load image from URL
+        image = load_image_from_url(image_url)
+        
+        # Generate caption
+        caption = engine.generate_caption(image_array=image)
+        
+        return JSONResponse(content={
+            "success": True,
+            "caption": caption,
+            "image_size": {"width": image.width, "height": image.height},
+            "model": "Florence-2-base-ft-quantized-onnx"
+        })
     
-    if frame is None:
-        raise HTTPException(
-            status_code=400, 
-            detail="Could not decode image file. Ensure it is a valid image format (e.g., PNG, JPEG)."
+    except HTTPException:
+        raise
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, 
+            content={"success": False, "error": str(e)}
         )
 
-    # 3. Detect cursor
-    pos, conf, template_name = detect_cursor_in_frame_multi(frame, CURSOR_TEMPLATES, threshold)
-
-    # 4. Log values for debugging
-    print(f"pos: {pos}, type: {type(pos)}")
-    print(f"conf: {conf}, type: {type(conf)}")
-    print(f"template_name: {template_name}, type: {type(template_name)}")
-    print(f"frame.shape: {frame.shape}, type: {type(frame.shape)}")
-
-    # 5. Prepare response
-    # Handle infinite confidence values
-    confidence = float(conf)
-    if not (confidence == float('inf') or confidence == float('-inf')):
-        confidence_val = confidence
-    else:
-        confidence_val = 1.0 if confidence > 0 else 0.0
-
-    if pos is not None:
-        response_data = {
-            'cursor_active': True,
-            'x': pos[0],
-            'y': pos[1],
-            'confidence': confidence_val,
-            'template': template_name,
-            'image_shape': list(frame.shape)
-        }
-    else:
-        response_data = {
-            'cursor_active': False,
-            'x': None,
-            'y': None,
-            'confidence': confidence_val,
-            'template': None,
-            'image_shape': list(frame.shape)
-        }
-
-    return JSONResponse(content=response_data)
-
-# Optional: Endpoint to get a list of loaded templates
-@app.post("/track_cursor_url")
-async def track_cursor_url_endpoint(
-    image_url: str = Form(...),
-    threshold: float = Form(0.8)
-):
-    """
-    Accepts an image URL and returns the detected cursor position and details.
+@app.post("/analyze")
+async def analyze_post(file: UploadFile = File(None)):
+    """Analyze an uploaded image (multipart/form-data).
+    
+    Returns: JSON with caption and metadata
     """
-    if not CURSOR_TEMPLATES:
-        raise HTTPException(
-            status_code=503,
-            detail="Cursor templates are not loaded. Server initialization failed."
-        )
-
     try:
-        # Validate URL
-        parsed_url = urlparse(image_url)
-        if not all([parsed_url.scheme, parsed_url.netloc]):
-            raise HTTPException(
-                status_code=400,
-                detail="Invalid URL provided"
-            )
-
-        # Download image
-        content = await download_image_from_url(image_url)
+        if engine is None:
+            raise HTTPException(status_code=503, detail="Model not initialized")
         
-        # Convert to OpenCV format
-        np_array = np.frombuffer(content, np.uint8)
-        frame = cv2.imdecode(np_array, cv2.IMREAD_UNCHANGED)
+        if file is None:
+            raise HTTPException(status_code=400, detail="file is required")
         
-        if frame is None:
-            raise HTTPException(
-                status_code=400,
-                detail="Could not decode image from URL. Ensure it is a valid image format (e.g., PNG, JPEG)."
-            )
-
-        # Detect cursor
-        pos, conf, template_name = detect_cursor_in_frame_multi(frame, CURSOR_TEMPLATES, threshold)
-
-        # Prepare response
-        if pos is not None:
-            response_data = {
-                'cursor_active': True,
-                'x': pos[0],
-                'y': pos[1],
-                'confidence': float(conf),
-                'template': template_name,
-                'image_shape': list(frame.shape),
-                'source_url': image_url
-            }
-        else:
-            response_data = {
-                'cursor_active': False,
-                'x': None,
-                'y': None,
-                'confidence': float(conf),
-                'template': None,
-                'image_shape': list(frame.shape),
-                'source_url': image_url
-            }
-
-        return JSONResponse(content=response_data)
-
-    except aiohttp.ClientError as e:
-        raise HTTPException(
-            status_code=400,
-            detail=f"Failed to fetch image from URL: {str(e)}"
-        )
+        # Read uploaded file
+        content = await file.read()
+        
+        # Load image from bytes
+        try:
+            image = load_image_from_bytes(content)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Failed to read uploaded image: {e}")
+        
+        # Generate caption
+        caption = engine.generate_caption(image_array=image)
+        
+        return JSONResponse(content={
+            "success": True,
+            "caption": caption,
+            "filename": file.filename,
+            "image_size": {"width": image.width, "height": image.height},
+            "model": "Florence-2-base-ft-quantized-onnx"
+        })
+    
+    except HTTPException:
+        raise
     except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"An error occurred while processing the image: {str(e)}"
+        return JSONResponse(
+            status_code=500, 
+            content={"success": False, "error": str(e)}
         )
 
-@app.get("/templates")
-async def list_templates():
-    """Returns a list of all loaded cursor template names."""
-    return {"templates": list(CURSOR_TEMPLATES.keys()), "count": len(CURSOR_TEMPLATES)}
-
+# Get the port from environment variable
 port = int(os.environ.get("PORT", 7860))
 
-# Launch FastAPI with uvicorn when run directly
+# Launch server
 if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=port, timeout_keep_alive=75)
+    print("Initializing Florence-2 ONNX Engine...")
+    initialize_engine()
+    
+    print(f"\n✓ Server ready! Starting on 0.0.0.0:{port}")
+    print(f"API Documentation: http://localhost:{port}/docs")
+    
+    uvicorn.run(app, host="0.0.0.0", port=port)

requirements.txt

@@ -1,11 +1,9 @@
-fastapi==0.104.1
-uvicorn==0.24.0
-aiofiles==23.2.1
-python-multipart==0.0.6
-huggingface-hub==0.18.0
-aiohttp
-jinja2
-pydantic
-datasets
-opencv-python
-numpy
+numpy
+pillow
+huggingface_hub
+onnxruntime
+transformers
+fastapi
+uvicorn[standard]
+requests
+python-multipart