ket usage

core/models.py

@@ -129,3 +129,22 @@ class GeminiJob(Base):
     def __repr__(self):
         return f"<GeminiJob(job_id={self.job_id}, type={self.job_type}, status={self.status}, priority={self.priority})>"
 
+
+class ApiKeyUsage(Base):
+    """
+    Track API key usage for round-robin load balancing.
+    Only stores the key index, not the actual key for security.
+    """
+    __tablename__ = "api_key_usage"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    key_index = Column(Integer, unique=True, index=True, nullable=False)  # Index of key in GEMINI_API_KEYS
+    total_requests = Column(Integer, default=0)
+    success_count = Column(Integer, default=0)
+    failure_count = Column(Integer, default=0)
+    last_error = Column(Text, nullable=True)  # Stores the last error message for review
+    last_used_at = Column(DateTime(timezone=True), nullable=True)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+
+    def __repr__(self):
+        return f"<ApiKeyUsage(index={self.key_index}, total={self.total_requests}, success={self.success_count}, failed={self.failure_count})>"

routers/general.py

@@ -1,7 +1,10 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends
 from fastapi.responses import HTMLResponse
+from sqlalchemy.ext.asyncio import AsyncSession
 import os
 
+from core.database import get_db
+
 router = APIRouter()
 
 @router.get("/health")
@@ -27,3 +30,19 @@ async def root():
     template_path = os.path.join(base_dir, "templates", "index.html")
     with open(template_path, "r") as f:
         return HTMLResponse(content=f.read())
+
+
+@router.get("/api/key-stats")
+async def get_api_key_stats(db: AsyncSession = Depends(get_db)):
+    """
+    Get API key usage statistics.
+    Returns usage stats by key index (not actual keys for security).
+    """
+    from services.api_key_manager import get_all_usage_stats, get_key_count
+    
+    stats = await get_all_usage_stats(db)
+    return {
+        "total_keys": get_key_count(),
+        "keys": stats
+    }
+

services/api_key_manager.py

@@ -0,0 +1,159 @@
+"""
+API Key Manager - Round-robin management for multiple Gemini API keys.
+
+Selects the least-used key to avoid rate limiting.
+Tracks usage per key index (not the actual key for security).
+"""
+import os
+import logging
+from datetime import datetime
+from typing import Optional, List, Tuple
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+# Cache for API keys (loaded once from env)
+_api_keys: Optional[List[str]] = None
+
+
+def get_api_keys() -> List[str]:
+    """Load API keys from GEMINI_API_KEYS environment variable."""
+    global _api_keys
+    if _api_keys is None:
+        keys_str = os.getenv("GEMINI_API_KEYS", "")
+        if not keys_str:
+            # Fallback to single key
+            single_key = os.getenv("GEMINI_API_KEY", "")
+            if single_key:
+                _api_keys = [single_key]
+            else:
+                _api_keys = []  # Return empty list if no keys configured
+        else:
+            _api_keys = [k.strip() for k in keys_str.split(",") if k.strip()]
+        
+        if _api_keys:
+            logger.info(f"Loaded {len(_api_keys)} API key(s)")
+        else:
+            logger.warning("Unable to authenticate GEMINI.")
+    
+    return _api_keys
+
+
+def get_key_count() -> int:
+    """Get the number of available API keys."""
+    return len(get_api_keys())
+
+
+async def get_least_used_key(db: AsyncSession) -> Tuple[int, str]:
+    """
+    Get the API key with least requests (round-robin style).
+    
+    Returns:
+        Tuple of (key_index, api_key)
+    
+    Raises:
+        ValueError: If no API keys are configured
+    """
+    from core.models import ApiKeyUsage
+    
+    keys = get_api_keys()
+    if not keys:
+        raise ValueError("No API keys configured. Set GEMINI_API_KEYS or GEMINI_API_KEY in environment.")
+    
+    # Get all usage stats
+    query = select(ApiKeyUsage).order_by(ApiKeyUsage.total_requests)
+    result = await db.execute(query)
+    usages = {u.key_index: u for u in result.scalars().all()}
+    
+    # Find the key with least usage
+    min_requests = float('inf')
+    selected_index = 0
+    
+    for i in range(len(keys)):
+        if i in usages:
+            if usages[i].total_requests < min_requests:
+                min_requests = usages[i].total_requests
+                selected_index = i
+        else:
+            # Key not in DB yet - create it and use it (0 requests)
+            new_usage = ApiKeyUsage(key_index=i, total_requests=0, success_count=0, failure_count=0)
+            db.add(new_usage)
+            await db.commit()
+            selected_index = i
+            break
+    
+    logger.debug(f"Selected API key index {selected_index} (least used)")
+    return selected_index, keys[selected_index]
+
+
+async def record_usage(db: AsyncSession, key_index: int, success: bool, error_message: Optional[str] = None):
+    """
+    Record API key usage after a request.
+    
+    Args:
+        db: Database session
+        key_index: Index of the key used
+        success: Whether the request succeeded
+        error_message: Error message if request failed
+    """
+    from core.models import ApiKeyUsage
+    
+    query = select(ApiKeyUsage).where(ApiKeyUsage.key_index == key_index)
+    result = await db.execute(query)
+    usage = result.scalar_one_or_none()
+    
+    if not usage:
+        usage = ApiKeyUsage(key_index=key_index, total_requests=0, success_count=0, failure_count=0)
+        db.add(usage)
+    
+    usage.total_requests += 1
+    if success:
+        usage.success_count += 1
+    else:
+        usage.failure_count += 1
+        if error_message:
+            usage.last_error = error_message[:1000]  # Truncate to 1000 chars
+    usage.last_used_at = datetime.utcnow()
+    
+    await db.commit()
+    logger.debug(f"Recorded {'success' if success else 'failure'} for key index {key_index}")
+
+
+async def get_all_usage_stats(db: AsyncSession) -> List[dict]:
+    """
+    Get usage stats for all API keys.
+    
+    Returns:
+        List of dicts with key_index, total_requests, success_count, failure_count
+    """
+    from core.models import ApiKeyUsage
+    
+    keys = get_api_keys()
+    query = select(ApiKeyUsage).order_by(ApiKeyUsage.key_index)
+    result = await db.execute(query)
+    usages = {u.key_index: u for u in result.scalars().all()}
+    
+    stats = []
+    for i in range(len(keys)):
+        if i in usages:
+            u = usages[i]
+            stats.append({
+                "key_index": i,
+                "total_requests": u.total_requests,
+                "success_count": u.success_count,
+                "failure_count": u.failure_count,
+                "last_error": u.last_error,
+                "last_used_at": u.last_used_at.isoformat() if u.last_used_at else None
+            })
+        else:
+            stats.append({
+                "key_index": i,
+                "total_requests": 0,
+                "success_count": 0,
+                "failure_count": 0,
+                "last_error": None,
+                "last_used_at": None
+            })
+    
+    return stats

services/gemini_job_worker.py

@@ -38,54 +38,72 @@ def get_priority_for_job_type(job_type: str) -> str:
 
 
 class GeminiJobProcessor(JobProcessor[GeminiJob]):
-    """Processes Gemini AI jobs (text, image, video generation)."""
+    """Processes Gemini AI jobs (text, image, video generation) with round-robin API keys."""
     
-    def __init__(self, api_key: Optional[str] = None):
-        """Initialize with optional API key (uses env var if not provided)."""
-        self.api_key = api_key
+    async def _get_service_with_key(self, session: AsyncSession) -> tuple:
+        """Get a GeminiService with the least-used API key."""
+        from services.api_key_manager import get_least_used_key
+        key_index, api_key = await get_least_used_key(session)
+        return key_index, GeminiService(api_key=api_key)
     
-    def _get_service(self) -> GeminiService:
-        """Get a GeminiService instance."""
-        return GeminiService(api_key=self.api_key)
+    async def _record_usage(self, session: AsyncSession, key_index: int, success: bool, error_message: Optional[str] = None):
+        """Record API key usage after request."""
+        from services.api_key_manager import record_usage
+        await record_usage(session, key_index, success, error_message)
     
     async def process(self, job: GeminiJob, session: AsyncSession) -> GeminiJob:
-        """Start processing a new job."""
-        service = self._get_service()
+        """Start processing a new job with round-robin API key."""
+        key_index, service = await self._get_service_with_key(session)
         input_data = job.input_data or {}
+        success = False
+        error_msg = None
         
         try:
             if job.job_type == "video":
                 job = await self._start_video(job, session, service, input_data)
+                success = True
             elif job.job_type == "image":
                 job = await self._process_image(job, service, input_data)
+                success = True
             elif job.job_type == "text":
                 job = await self._process_text(job, service, input_data)
+                success = True
             elif job.job_type == "analyze":
                 job = await self._process_analyze(job, service, input_data)
+                success = True
             elif job.job_type == "animation_prompt":
                 job = await self._process_animation_prompt(job, service, input_data)
+                success = True
             else:
                 job.status = "failed"
                 job.error_message = f"Unknown job type: {job.job_type}"
                 job.completed_at = datetime.utcnow()
+                error_msg = job.error_message
         except Exception as e:
             logger.error(f"Error processing job {job.job_id}: {e}")
             job.status = "failed"
             job.error_message = str(e)
             job.completed_at = datetime.utcnow()
+            success = False
+            error_msg = str(e)
+        
+        # Record usage
+        await self._record_usage(session, key_index, success, error_msg)
         
         return job
     
     async def check_status(self, job: GeminiJob, session: AsyncSession) -> GeminiJob:
         """Check status of an in-progress job (video generation)."""
         if job.job_type != "video" or not job.third_party_id:
-            # Non-video jobs or missing third_party_id - shouldn't happen
             job.status = "failed"
             job.error_message = "Invalid job state for status check"
             job.completed_at = datetime.utcnow()
             return job
         
-        service = self._get_service()
+        # Use round-robin key for status check
+        key_index, service = await self._get_service_with_key(session)
+        success = False
+        error_msg = None
         
         try:
             status_result = await service.check_video_status(job.third_party_id)
@@ -97,12 +115,15 @@ class GeminiJobProcessor(JobProcessor[GeminiJob]):
                         filename = await service.download_video(video_url, job.job_id)
                         job.status = "completed"
                         job.output_data = {"filename": filename}
+                        success = True
                     else:
                         job.status = "failed"
                         job.error_message = "No video URL returned"
+                        error_msg = job.error_message
                 else:
                     job.status = "failed"
                     job.error_message = status_result.get("error", "Unknown error")
+                    error_msg = job.error_message
                 
                 job.completed_at = datetime.utcnow()
             else:
@@ -111,7 +132,7 @@ class GeminiJobProcessor(JobProcessor[GeminiJob]):
                 config = WorkerConfig.from_env()
                 interval = get_interval_for_priority(job.priority, config)
                 job.next_process_at = datetime.utcnow() + timedelta(seconds=interval)
-                logger.debug(f"Job {job.job_id}: retry #{job.retry_count}, next check at {job.next_process_at}")
+                success = True  # Status check succeeded even if video not ready
         
         except Exception as e:
             logger.error(f"Error checking video status for {job.job_id}: {e}")
@@ -119,6 +140,11 @@ class GeminiJobProcessor(JobProcessor[GeminiJob]):
             config = WorkerConfig.from_env()
             interval = get_interval_for_priority(job.priority, config)
             job.next_process_at = datetime.utcnow() + timedelta(seconds=interval)
+            success = False
+            error_msg = str(e)
+        
+        # Record usage
+        await self._record_usage(session, key_index, success, error_msg)
         
         return job
     

services/gemini_service.py

@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 MODELS = {
     "text_generation": "gemini-2.5-flash",
     "image_edit": "gemini-2.5-flash-image",
-    "video_generation": "veo-3.1-fast-generate-preview"
+    "video_generation": "veo-3.1-generate-preview"
 }
 
 # Type aliases
@@ -73,7 +73,7 @@ def get_gemini_api_key() -> str:
     """Get Gemini API key from environment."""
     api_key = os.getenv("GEMINI_API_KEY")
     if not api_key:
-        raise ValueError("GEMINI_API_KEY environment variable not set")
+        raise ValueError("Server Authentication Error with GEMINI")
     return api_key
 
 

templates/index.html

@@ -180,11 +180,30 @@
             font-weight: 600;
         }
 
-        .status-success { background: rgba(34, 197, 94, 0.2); color: #22c55e; }
-        .status-failed { background: rgba(239, 68, 68, 0.2); color: #ef4444; }
-        .status-queued { background: rgba(234, 179, 8, 0.2); color: #eab308; }
-        .status-processing { background: rgba(59, 130, 246, 0.2); color: #3b82f6; }
-        .status-completed { background: rgba(34, 197, 94, 0.2); color: #22c55e; }
+        .status-success {
+            background: rgba(34, 197, 94, 0.2);
+            color: #22c55e;
+        }
+
+        .status-failed {
+            background: rgba(239, 68, 68, 0.2);
+            color: #ef4444;
+        }
+
+        .status-queued {
+            background: rgba(234, 179, 8, 0.2);
+            color: #eab308;
+        }
+
+        .status-processing {
+            background: rgba(59, 130, 246, 0.2);
+            color: #3b82f6;
+        }
+
+        .status-completed {
+            background: rgba(34, 197, 94, 0.2);
+            color: #22c55e;
+        }
 
         .credits-badge {
             background: linear-gradient(135deg, #00d4ff, #7b2cbf);
@@ -280,6 +299,7 @@
             <button class="tab-btn" onclick="switchTab('users')">👥 Users</button>
             <button class="tab-btn" onclick="switchTab('audit')">📝 Audit Logs</button>
             <button class="tab-btn" onclick="switchTab('jobs')">⚡ Gemini Jobs</button>
+            <button class="tab-btn" onclick="switchTab('keys')">🔑 API Keys</button>
         </div>
 
         <div class="stats" id="statsContainer">
@@ -310,13 +330,20 @@
                         </tr>
                     </thead>
                     <tbody id="blinkBody">
-                        <tr><td colspan="8"><div class="loading"><div class="spinner"></div>Loading...</div></td></tr>
+                        <tr>
+                            <td colspan="8">
+                                <div class="loading">
+                                    <div class="spinner"></div>Loading...
+                                </div>
+                            </td>
+                        </tr>
                     </tbody>
                 </table>
             </div>
             <div class="pagination">
                 <button onclick="prevPage('blink')" id="blinkPrev" disabled>← Previous</button>
-                <span class="page-info">Page <span id="blinkCurrentPage">1</span> of <span id="blinkTotalPages">1</span></span>
+                <span class="page-info">Page <span id="blinkCurrentPage">1</span> of <span
+                        id="blinkTotalPages">1</span></span>
                 <button onclick="nextPage('blink')" id="blinkNext">Next →</button>
             </div>
         </div>
@@ -338,13 +365,20 @@
                         </tr>
                     </thead>
                     <tbody id="usersBody">
-                        <tr><td colspan="8"><div class="loading"><div class="spinner"></div>Loading...</div></td></tr>
+                        <tr>
+                            <td colspan="8">
+                                <div class="loading">
+                                    <div class="spinner"></div>Loading...
+                                </div>
+                            </td>
+                        </tr>
                     </tbody>
                 </table>
             </div>
             <div class="pagination">
                 <button onclick="prevPage('users')" id="usersPrev" disabled>← Previous</button>
-                <span class="page-info">Page <span id="usersCurrentPage">1</span> of <span id="usersTotalPages">1</span></span>
+                <span class="page-info">Page <span id="usersCurrentPage">1</span> of <span
+                        id="usersTotalPages">1</span></span>
                 <button onclick="nextPage('users')" id="usersNext">Next →</button>
             </div>
         </div>
@@ -365,13 +399,20 @@
                         </tr>
                     </thead>
                     <tbody id="auditBody">
-                        <tr><td colspan="7"><div class="loading"><div class="spinner"></div>Loading...</div></td></tr>
+                        <tr>
+                            <td colspan="7">
+                                <div class="loading">
+                                    <div class="spinner"></div>Loading...
+                                </div>
+                            </td>
+                        </tr>
                     </tbody>
                 </table>
             </div>
             <div class="pagination">
                 <button onclick="prevPage('audit')" id="auditPrev" disabled>← Previous</button>
-                <span class="page-info">Page <span id="auditCurrentPage">1</span> of <span id="auditTotalPages">1</span></span>
+                <span class="page-info">Page <span id="auditCurrentPage">1</span> of <span
+                        id="auditTotalPages">1</span></span>
                 <button onclick="nextPage('audit')" id="auditNext">Next →</button>
             </div>
         </div>
@@ -393,16 +434,54 @@
                         </tr>
                     </thead>
                     <tbody id="jobsBody">
-                        <tr><td colspan="8"><div class="loading"><div class="spinner"></div>Loading...</div></td></tr>
+                        <tr>
+                            <td colspan="8">
+                                <div class="loading">
+                                    <div class="spinner"></div>Loading...
+                                </div>
+                            </td>
+                        </tr>
                     </tbody>
                 </table>
             </div>
             <div class="pagination">
                 <button onclick="prevPage('jobs')" id="jobsPrev" disabled>← Previous</button>
-                <span class="page-info">Page <span id="jobsCurrentPage">1</span> of <span id="jobsTotalPages">1</span></span>
+                <span class="page-info">Page <span id="jobsCurrentPage">1</span> of <span
+                        id="jobsTotalPages">1</span></span>
                 <button onclick="nextPage('jobs')" id="jobsNext">Next →</button>
             </div>
         </div>
+
+        <!-- API Keys Usage Table -->
+        <div class="table-container hidden" id="keysTable">
+            <div class="table-wrapper">
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Key Index</th>
+                            <th>Total Requests</th>
+                            <th>Success</th>
+                            <th>Failed</th>
+                            <th>Success Rate</th>
+                            <th>Last Error</th>
+                            <th>Last Used</th>
+                        </tr>
+                    </thead>
+                    <tbody id="keysBody">
+                        <tr>
+                            <td colspan="7">
+                                <div class="loading">
+                                    <div class="spinner"></div>Loading...
+                                </div>
+                            </td>
+                        </tr>
+                    </tbody>
+                </table>
+            </div>
+            <div class="pagination">
+                <span class="page-info">Auto-refreshes every 10 seconds</span>
+            </div>
+        </div>
     </div>
 
     <script>
@@ -424,22 +503,27 @@
 
         function switchTab(tab) {
             currentTab = tab;
-            
+
             // Update tab buttons
             document.querySelectorAll('.tab-btn').forEach(btn => btn.classList.remove('active'));
             event.target.classList.add('active');
-            
+
             // Show/hide tables
             document.getElementById('blinkTable').classList.toggle('hidden', tab !== 'blink');
             document.getElementById('usersTable').classList.toggle('hidden', tab !== 'users');
             document.getElementById('auditTable').classList.toggle('hidden', tab !== 'audit');
             document.getElementById('jobsTable').classList.toggle('hidden', tab !== 'jobs');
-            
+            document.getElementById('keysTable').classList.toggle('hidden', tab !== 'keys');
+
             // Show/hide unique users stat (only for blink)
             document.getElementById('uniqueUsersCard').classList.toggle('hidden', tab !== 'blink');
-            
-            // Load data if not loaded
-            loadPage(tab, state[tab].page);
+
+            // Load data
+            if (tab === 'keys') {
+                loadKeyStats();
+            } else {
+                loadPage(tab, state[tab].page);
+            }
         }
 
         async function fetchData(tab, page) {
@@ -573,6 +657,49 @@
             }
         }
 
+        // Load and render API key stats
+        async function loadKeyStats() {
+            try {
+                const response = await fetch('/api/key-stats');
+                const data = await response.json();
+                renderKeysTable(data.keys);
+                document.getElementById('totalRecords').textContent = data.total_keys;
+            } catch (error) {
+                console.error('Error fetching key stats:', error);
+            }
+        }
+
+        function renderKeysTable(items) {
+            const tbody = document.getElementById('keysBody');
+            if (!items || items.length === 0) {
+                tbody.innerHTML = '<tr><td colspan="7"><div class="empty-state">No API keys configured</div></td></tr>';
+                return;
+            }
+            tbody.innerHTML = items.map(item => {
+                const successRate = item.total_requests > 0
+                    ? ((item.success_count / item.total_requests) * 100).toFixed(1)
+                    : '0.0';
+                return `
+                    <tr>
+                        <td><span class="credits-badge">Key ${item.key_index}</span></td>
+                        <td>${item.total_requests.toLocaleString()}</td>
+                        <td><span class="status-badge status-success">${item.success_count}</span></td>
+                        <td><span class="status-badge status-failed">${item.failure_count}</span></td>
+                        <td>${successRate}%</td>
+                        <td class="truncate" title="${item.last_error || ''}">${item.last_error || '-'}</td>
+                        <td class="timestamp">${item.last_used_at ? new Date(item.last_used_at).toLocaleString() : 'Never'}</td>
+                    </tr>
+                `;
+            }).join('');
+        }
+
+        // Auto-refresh key stats every 10 seconds when on keys tab
+        setInterval(() => {
+            if (currentTab === 'keys') {
+                loadKeyStats();
+            }
+        }, 10000);
+
         // Initial load
         loadPage('blink', 1);
     </script>