Fix Greek OCR and update Latin OCR model

models/groq_client.py

@@ -23,7 +23,7 @@ class GroqClient:
         """Check if Groq API client is available and configured"""
         return self.client is not None
 
-    def generate_response(self, system_prompt: str, user_prompt: str, max_tokens: int = 1024) -> str:
+    def generate_response(self, system_prompt: str, user_prompt: str, max_tokens: int = 1024, response_format = None) -> str:
         """Generate response from Groq LLM"""
         if not self.is_available():
             print("[WARN] GroqClient not available for generating response.")
@@ -36,15 +36,19 @@ class GroqClient:
             if model == "openai/gpt-oss-120b":
                 model = "llama-3.1-8b-instant"  # standard Groq model
                 
-            completion = self.client.chat.completions.create(
-                model=model,
-                messages=[
+            params = {
+                "model": model,
+                "messages": [
                     {"role": "system", "content": system_prompt},
                     {"role": "user", "content": user_prompt}
                 ],
-                temperature=getattr(self.config, 'GROQ_TEMPERATURE', 0.7),
-                max_completion_tokens=max_tokens,
-            )
+                "temperature": getattr(self.config, 'GROQ_TEMPERATURE', 0.7),
+                "max_completion_tokens": max_tokens,
+            }
+            if response_format is not None:
+                params["response_format"] = response_format
+                
+            completion = self.client.chat.completions.create(**params)
             return completion.choices[0].message.content
         except Exception as e:
             print(f"[ERROR] Groq API call failed: {e}")

processors/greek_processor.py

@@ -25,6 +25,11 @@ class GreekProcessor(BaseScriptProcessor):
         # Register for dynamic VRAM management
         from utils.gpu_diagnostics import register_processor
         register_processor("greek", self)
+        
+        # Metrics for Greek Glossary
+        self.glossary_success_count = 0
+        self.glossary_json_failure_count = 0
+        self.regex_recovery_count = 0
 
     def setup_greek_trocr(self):
         """Setup TrOCR model — BEST for ancient Greek manuscripts"""
@@ -554,50 +559,77 @@ class GreekProcessor(BaseScriptProcessor):
         
         system_prompt = (
             "You are an expert classicist and lexicographer of Ancient Greek. "
-            "Respond ONLY with a JSON object. Do NOT wrap values in double quotes inside the strings. "
-            "Use single quotes '...' for any internal quotes, definitions, or translations."
+            "Return ONLY valid JSON matching the requested schema. "
+            "No markdown, no code fences (like ```json), no explanations, no prose."
         )
         user_prompt = (
-            f"For each of the following Ancient Greek words, provide a brief, scholarly one-sentence definition, "
-            f"etymological note, or grammatical gloss:\n\n"
+            f"For each of the following Ancient Greek words, provide a scholarly definition, "
+            f"etymological note, and grammatical gloss:\n\n"
             f"Words: {terms_list}\n\n"
-            f"Respond ONLY with a JSON object where the keys are the exact words and the values are the definitions.\n"
-            f"Do NOT use double quotes inside the definitions/values; use single quotes instead.\n"
-            f"Example: {{\"word1\": \"definition1\", \"word2\": \"definition2\"}}"
+            f"You MUST format the output as a single JSON object where the keys are the exact words "
+            f"and the values are objects containing 'definition', 'gloss', and 'etymology' keys.\n\n"
+            f"Output schema:\n"
+            f"{{\n"
+            f"  \"TERM\": {{\n"
+            f"    \"definition\": \"...\",\n"
+            f"    \"gloss\": \"...\",\n"
+            f"    \"etymology\": \"...\"\n"
+            f"  }}\n"
+            f"}}\n"
         )
         
         try:
             raw_response = self.groq_client.generate_response(
                 system_prompt=system_prompt,
                 user_prompt=user_prompt,
-                max_tokens=2048
+                max_tokens=2048,
+                response_format={"type": "json_object"}
             )
             # Safe print to avoid UnicodeEncodeError in Windows command prompt
             print(f"[INFO] Groq glossary raw response: {raw_response.encode('ascii', 'backslashreplace').decode()}")
             
             # Find JSON block in response
-            if "{" in raw_response and "}" in raw_response:
-                start = raw_response.find("{")
-                end = raw_response.rfind("}") + 1
-                json_str = raw_response[start:end]
-                import json
-                try:
-                    definitions = json.loads(json_str)
-                except Exception as je:
-                    print(f"[WARN] Standard JSON load failed: {je}. Attempting regex recovery...")
-                    import re
-                    definitions = {}
-                    # Matches "key": "value"
-                    pattern = re.compile(r'"([^"]+)":\s*"((?:[^"\\]|\\.)*)"')
-                    matches = pattern.findall(json_str)
-                    for k, v in matches:
-                        definitions[k] = v
-                return {k: str(v) for k, v in definitions.items()}
+            json_str = raw_response.strip()
+            if "{" in json_str and "}" in json_str:
+                start = json_str.find("{")
+                end = json_str.rfind("}") + 1
+                json_str = json_str[start:end]
+            
+            import json
+            definitions = {}
+            try:
+                definitions = json.loads(json_str)
+                self.glossary_success_count += 1
+            except Exception as je:
+                self.glossary_json_failure_count += 1
+                import logging
+                logger = logging.getLogger(__name__)
+                logger.warning(
+                    "Malformed Greek glossary JSON",
+                    extra={"response": raw_response[:2000]}
+                )
+                print(f"[WARN] Standard JSON load failed: {je}. Attempting regex recovery...")
+                
+                # Regex recovery fallback
+                import re
+                self.regex_recovery_count += 1
+                term_blocks = re.findall(r'"([^"]+)"\s*:\s*\{([^}]+)\}', json_str)
+                for term, block in term_blocks:
+                    def_match = re.search(r'"definition"\s*:\s*["\']([^"\']+)["\']', block)
+                    gloss_match = re.search(r'"gloss"\s*:\s*["\']([^"\']+)["\']', block)
+                    ety_match = re.search(r'"etymology"\s*:\s*["\']([^"\']+)["\']', block)
+                    definitions[term] = {
+                        "definition": def_match.group(1) if def_match else "",
+                        "gloss": gloss_match.group(1) if gloss_match else "",
+                        "etymology": ety_match.group(1) if ety_match else ""
+                    }
+                    
+            return definitions
         except Exception as e:
             print(f"[WARN] Failed to generate batch Greek explanations: {e}")
             
         return {}
-    
+
     def _build_uses_list(self, terms, greek_text):
         """Build list of symbol/word uses using RAG and batch Groq explanations"""
         import unicodedata
@@ -652,6 +684,18 @@ class GreekProcessor(BaseScriptProcessor):
                 definition = definitions.get(term)
                 if not definition:
                     definition = f"Ancient Greek lexical term. Characterized by specific diacritics and phonological values."
+                elif isinstance(definition, dict):
+                    parts = []
+                    d_val = definition.get("definition", "").strip()
+                    g_val = definition.get("gloss", "").strip()
+                    e_val = definition.get("etymology", "").strip()
+                    if d_val:
+                        parts.append(d_val)
+                    if g_val:
+                        parts.append(f"Gloss: {g_val}")
+                    if e_val:
+                        parts.append(f"Etymology: {e_val}")
+                    definition = " | ".join(parts) if parts else "Ancient Greek lexical term."
                 items.append(f"{term}: {definition}")
         
         # 2. Add significant paleographical/character markers found in the text if they are in the references