fix: add Groq Vision (chess Q04) + Groq Whisper (audio Q10/Q14) — target 95-100%

app.py

@@ -34,6 +34,69 @@ QUESTIONS_URL = f"{API_URL}/questions"
 FILES_URL = f"{API_URL}/files"
 SUBMIT_URL = f"{API_URL}/submit"
 
+# ─────────────────────────────────────────────────────────────────────────────
+# GROQ HELPERS — Vision (llama-3.2-11b-vision) & Audio (whisper-large-v3)
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _groq_client():
+    """Return a raw Groq HTTP client (uses requests, no extra SDK needed)."""
+    api_key = os.environ.get("GROQ_API_KEY")
+    if not api_key:
+        raise RuntimeError("GROQ_API_KEY not set")
+    return api_key
+
+
+def _transcribe_with_groq_whisper(audio_path: str) -> str:
+    """Send an audio file to Groq Whisper API and return the transcript."""
+    api_key = _groq_client()
+    with open(audio_path, "rb") as f:
+        audio_bytes = f.read()
+
+    filename = os.path.basename(audio_path)
+    resp = requests.post(
+        "https://api.groq.com/openai/v1/audio/transcriptions",
+        headers={"Authorization": f"Bearer {api_key}"},
+        files={"file": (filename, audio_bytes, "audio/mpeg")},
+        data={"model": "whisper-large-v3", "response_format": "text"},
+        timeout=60,
+    )
+    resp.raise_for_status()
+    return resp.text.strip()
+
+
+def _analyze_with_groq_vision(image_b64: str, mime_type: str = "image/png", prompt: str = "Describe this image in detail.") -> str:
+    """Send an image to Groq vision model and return the analysis."""
+    api_key = _groq_client()
+    payload = {
+        "model": "meta-llama/llama-4-scout-17b-16e-instruct",
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:{mime_type};base64,{image_b64}"},
+                    },
+                    {"type": "text", "text": prompt},
+                ],
+            }
+        ],
+        "max_tokens": 2048,
+        "temperature": 0,
+    }
+    resp = requests.post(
+        "https://api.groq.com/openai/v1/chat/completions",
+        headers={
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        },
+        json=payload,
+        timeout=60,
+    )
+    resp.raise_for_status()
+    return resp.json()["choices"][0]["message"]["content"]
+
+
 # ─────────────────────────────────────────────────────────────────────────────
 # TOOLS
 # ─────────────────────────────────────────────────────────────────────────────
@@ -282,67 +345,38 @@ def download_and_read_file(task_id: str) -> str:
 
         # ── Audio (MP3 / WAV) ─────────────────────────────────────────────
         if ext in ("mp3", "wav", "m4a", "ogg", "flac") or "audio" in content_type:
-            # Try speech-to-text with SpeechRecognition + pydub
+            # Save to temp file then transcribe with Groq Whisper
+            with tempfile.NamedTemporaryFile(suffix=f".{ext}", delete=False) as tmp:
+                tmp.write(raw)
+                tmp_path = tmp.name
             try:
-                import speech_recognition as sr
-                from pydub import AudioSegment
-
-                with tempfile.NamedTemporaryFile(suffix=f".{ext}", delete=False) as tmp:
-                    tmp.write(raw)
-                    tmp_path = tmp.name
-
-                # Convert to WAV if needed
-                if ext != "wav":
-                    audio = AudioSegment.from_file(tmp_path)
-                    wav_path = tmp_path.replace(f".{ext}", ".wav")
-                    audio.export(wav_path, format="wav")
-                else:
-                    wav_path = tmp_path
-
-                recognizer = sr.Recognizer()
-                with sr.AudioFile(wav_path) as source:
-                    audio_data = recognizer.record(source)
-                transcript = recognizer.recognize_google(audio_data)
-
-                # Cleanup
+                transcript = _transcribe_with_groq_whisper(tmp_path)
+                os.unlink(tmp_path)
+                return f"[Audio transcript — {len(raw)} bytes]\n{transcript}"
+            except Exception as e:
                 try:
                     os.unlink(tmp_path)
-                    if wav_path != tmp_path:
-                        os.unlink(wav_path)
                 except Exception:
                     pass
-
-                return f"[Audio transcript]\n{transcript}"
-            except ImportError:
-                return (
-                    f"[Audio file — {len(raw)} bytes — {ext.upper()}]\n"
-                    "Speech recognition libraries not available. "
-                    "The audio content cannot be transcribed automatically. "
-                    "Please use other context clues from the question to answer."
-                )
-            except Exception as e:
-                return f"[Audio file — {len(raw)} bytes] Transcription failed: {e}"
+                return f"[Audio file — {len(raw)} bytes — {ext.upper()}] Transcription failed: {e}"
 
         # ── Image ─────────────────────────────────────────────────────────
         if ext in ("png", "jpg", "jpeg", "gif", "bmp", "webp") or "image" in content_type:
-            result = f"[Image file — {filename} — {len(raw)} bytes]\n"
-            # Try OCR
-            try:
-                from PIL import Image
-                import pytesseract
-                img = Image.open(io.BytesIO(raw))
-                result += f"Size: {img.size}, Mode: {img.mode}\n"
-                ocr_text = pytesseract.image_to_string(img).strip()
-                if ocr_text:
-                    result += f"\n[OCR Text]\n{ocr_text}"
-                else:
-                    result += "\n[No text detected by OCR]"
-            except Exception as ocr_err:
-                result += f"\n[OCR not available: {ocr_err}]"
-            # Return base64 for visual analysis by multimodal models
+            # Use Groq Vision to analyse the image
             b64 = base64.b64encode(raw).decode()
-            result += f"\n\n[Image base64 (first 500 chars)]\n{b64[:500]}..."
-            return result
+            try:
+                vision_result = _analyze_with_groq_vision(
+                    b64,
+                    mime_type=f"image/{ext if ext != 'jpg' else 'jpeg'}",
+                    prompt=(
+                        "Describe this image in full detail. "
+                        "If it is a chess board, list ALL pieces and their exact positions in FEN notation, "
+                        "then state whose turn it is and identify the best/winning move."
+                    )
+                )
+                return f"[Image analysis — {filename} — {len(raw)} bytes]\n\n{vision_result}"
+            except Exception as e:
+                return f"[Image file — {filename} — {len(raw)} bytes]\nVision analysis failed: {e}\n[base64 prefix]\n{b64[:300]}..."
 
         # ── Plain text / fallback ─────────────────────────────────────────
         try: