Upd SFT trans saver

app.py

@@ -390,12 +390,20 @@ def _run_job(dataset_key: str, params: ProcessParams):
         if params.vietnamese_translation:
             set_state(message="Loading Vietnamese translator", progress=0.05)
             try:
+                # Ensure cache directories are set up properly
+                cache_dir = os.path.abspath("cache/huggingface")
+                os.makedirs(cache_dir, exist_ok=True)
+                os.environ["HF_HOME"] = cache_dir
+                
                 vietnamese_translator.load_model()
                 translator = vietnamese_translator
                 logger.info("✅ Vietnamese translator loaded successfully")
             except Exception as e:
                 logger.error(f"❌ Failed to load Vietnamese translator: {e}")
-                set_state(message=f"Warning: Vietnamese translation failed - {e}", progress=0.1)
+                logger.warning("Continuing without Vietnamese translation...")
+                set_state(message=f"Warning: Vietnamese translation disabled - {e}", progress=0.1)
+                # Don't fail the entire job, just disable translation
+                translator = None
         
         if params.rag_processing:
             # RAG processing mode
@@ -429,6 +437,12 @@ def _run_job(dataset_key: str, params: ProcessParams):
                 progress_cb=lambda p, msg=None: set_state(progress=p, message=msg or STATE["message"]),
                 translator=translator
             )
+        # Log translation statistics if translator was used
+        if translator and hasattr(translator, 'get_stats'):
+            translation_stats = translator.get_stats()
+            logger.info(f"[JOB] Translation stats: {translation_stats}")
+            stats["translation_stats"] = translation_stats
+        
         logger.info(f"[JOB] Processed dataset={dataset_key} rows={count} stats={stats}")
         writer.close()
 

utils/rag.py

@@ -309,9 +309,13 @@ class RAGProcessor:
         if should_translate(opts.get("vietnamese_translation", False) if opts else False, translator):
             try:
                 row = translate_rag_row(row, translator, ["question", "answer", "context"])
-                row["vi_translated"] = True
+                # Add translation metadata
+                if "meta" not in row:
+                    row["meta"] = {}
+                row["meta"]["vietnamese_translated"] = True
             except Exception as e:
                 logger.error(f"Failed to translate RAG row: {e}")
+                # Continue with original row if translation fails
 
         writer.write(row)
         stats["written"] = stats.get("written", 0) + 1

vi/processing.py

@@ -114,11 +114,17 @@ def translate_sft_row(row: Dict[str, Any], translator, text_fields: List[str] =
                     if _validate_vi_translation(original, translated):
                         translated_sft[field] = _vi_sanitize_text(translated)
                         logger.debug(f"✅ Successfully translated field '{field}'")
+                        # Add success statistics if stats available
+                        if hasattr(translator, '_stats'):
+                            add_translation_stats(translator._stats, f"sft_{field}", True)
                     else:
                         logger.warning(f"❌ Invalid Vietnamese translation for field {field}, keeping original")
                         logger.warning(f"  Original: '{original[:50]}...'")
                         logger.warning(f"  Translated: '{translated[:50]}...'")
                         translated_sft[field] = original
+                        # Add failure statistics if stats available
+                        if hasattr(translator, '_stats'):
+                            add_translation_stats(translator._stats, f"sft_{field}", False)
                 except Exception as e:
                     logger.error(f"Failed to translate field '{field}': {e}")
                     translated_sft[field] = sft_data[field]
@@ -156,17 +162,44 @@ def translate_rag_row(row: Dict[str, Any], translator, text_fields: List[str] =
         text_fields = ["question", "answer", "context"]
     
     try:
-        translated_row = translator.translate_dict(row, text_fields)
-        # Validate and sanitize translated fields
-        for f in text_fields:
-            if f in translated_row:
-                original = row.get(f, "")
-                translated = translated_row[f]
-                if _validate_vi_translation(original, translated):
-                    translated_row[f] = _vi_sanitize_text(translated)
-                else:
-                    logger.warning(f"Invalid Vietnamese translation for field {f}, keeping original")
-                    translated_row[f] = original
+        # Create a copy of the row to avoid modifying the original
+        translated_row = row.copy()
+        
+        # Translate each field individually with proper validation
+        for field in text_fields:
+            if field in row and isinstance(row[field], str) and row[field].strip():
+                try:
+                    original = row[field]
+                    translated = translator.translate_text(original)
+                    
+                    # Debug logging
+                    logger.debug(f"RAG Translation attempt for field '{field}':")
+                    logger.debug(f"  Original: '{original[:50]}...'")
+                    logger.debug(f"  Translated: '{translated[:50]}...'")
+                    logger.debug(f"  Are they the same? {original == translated}")
+                    
+                    # Validate and sanitize translated field
+                    if _validate_vi_translation(original, translated):
+                        translated_row[field] = _vi_sanitize_text(translated)
+                        logger.debug(f"✅ Successfully translated RAG field '{field}'")
+                        # Add success statistics if stats available
+                        if hasattr(translator, '_stats'):
+                            add_translation_stats(translator._stats, f"rag_{field}", True)
+                    else:
+                        logger.warning(f"❌ Invalid Vietnamese translation for RAG field {field}, keeping original")
+                        logger.warning(f"  Original: '{original[:50]}...'")
+                        logger.warning(f"  Translated: '{translated[:50]}...'")
+                        translated_row[field] = original
+                        # Add failure statistics if stats available
+                        if hasattr(translator, '_stats'):
+                            add_translation_stats(translator._stats, f"rag_{field}", False)
+                except Exception as e:
+                    logger.error(f"Failed to translate RAG field '{field}': {e}")
+                    translated_row[field] = row[field]
+            else:
+                # Keep original if field doesn't exist or is empty
+                translated_row[field] = row.get(field, "")
+        
         logger.debug(f"Translated RAG row with fields: {text_fields}")
         return translated_row
     except Exception as e:
@@ -187,8 +220,12 @@ def should_translate(vietnamese_translation: bool, translator) -> bool:
     if not vietnamese_translation:
         return False
     
-    if not translator or not translator.is_loaded():
-        logger.warning("Vietnamese translation requested but translator not available")
+    if not translator:
+        logger.warning("Vietnamese translation requested but translator is None")
+        return False
+    
+    if not hasattr(translator, 'is_loaded') or not translator.is_loaded():
+        logger.warning("Vietnamese translation requested but translator not loaded")
         return False
     
     return True
@@ -202,4 +239,24 @@ def log_translation_stats(stats: Dict[str, Any], translated_count: int) -> None:
         translated_count: Number of items translated
     """
     stats["vietnamese_translated"] = translated_count
-    logger.info(f"Vietnamese translation completed: {translated_count} items translated")
+    logger.info(f"Vietnamese translation completed: {translated_count} items translated")
+
+def add_translation_stats(stats: Dict[str, Any], field: str, success: bool) -> None:
+    """
+    Add translation statistics for individual fields.
+    
+    Args:
+        stats: Statistics dictionary to update
+        field: Field name that was translated
+        success: Whether translation was successful
+    """
+    if "translation_stats" not in stats:
+        stats["translation_stats"] = {}
+    
+    if field not in stats["translation_stats"]:
+        stats["translation_stats"][field] = {"success": 0, "failed": 0}
+    
+    if success:
+        stats["translation_stats"][field]["success"] += 1
+    else:
+        stats["translation_stats"][field]["failed"] += 1

vi/translator.py

@@ -31,6 +31,7 @@ class VietnameseTranslator:
         self.model = None
         self.tokenizer = None
         self._is_loaded = False
+        self._stats = {"total_translations": 0, "successful_translations": 0, "failed_translations": 0}
         
         logger.info(f"VietnameseTranslator initialized with model: {self.model_name}")
         logger.info(f"Using device: {self.device}")
@@ -101,6 +102,8 @@ class VietnameseTranslator:
             return text
         
         try:
+            self._stats["total_translations"] += 1
+            
             # Prepare input with target language token
             # The model requires a target language token in the format >>id<<
             input_text = f">>vie<< {text.strip()}"
@@ -130,10 +133,13 @@ class VietnameseTranslator:
             logger.debug(f"Translation result: '{text[:50]}...' -> '{translated[:50]}...'")
             logger.debug(f"Are original and translated the same? {text.strip() == translated.strip()}")
             
+            # Track success
+            self._stats["successful_translations"] += 1
             return translated.strip()
             
         except Exception as e:
             logger.error(f"Translation failed for text: '{text[:100]}...' - Error: {e}")
+            self._stats["failed_translations"] += 1
             # Return original text if translation fails
             return text
     
@@ -270,3 +276,11 @@ class VietnameseTranslator:
             "device": self.device,
             "is_loaded": self._is_loaded
         }
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get translation statistics."""
+        return self._stats.copy()
+    
+    def reset_stats(self) -> None:
+        """Reset translation statistics."""
+        self._stats = {"total_translations": 0, "successful_translations": 0, "failed_translations": 0}