Update src/validation.py

src/validation.py

@@ -101,11 +101,17 @@ def validate_predictions_content(predictions: pd.DataFrame) -> Dict:
     if duplicate_predictions > len(predictions) * 0.5:  # More than 50%
         warnings.append(f"{duplicate_predictions} duplicate prediction texts")
     
-    # Check for non-text content
-    non_text_pattern = r'^[A-Za-z\s\'".,!?;:()\-]+$'
-    non_text_predictions = ~predictions['prediction'].str.match(non_text_pattern, na=False)
-    if non_text_predictions.sum() > 0:
-        warnings.append(f"{non_text_predictions.sum()} predictions contain unusual characters")
+    # Check for non-text content (more permissive regex for multiple languages)
+    # Allow Unicode characters for non-English languages
+    non_text_pattern = r'^[\w\s\'".,!?;:()\-àáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ]+$'
+    try:
+        non_text_predictions = ~predictions['prediction'].str.match(non_text_pattern, na=False)
+        unusual_char_count = non_text_predictions.sum()
+        if unusual_char_count > len(predictions) * 0.2:  # More than 20%
+            warnings.append(f"{unusual_char_count} predictions may contain special characters")
+    except:
+        # Skip this check if regex fails
+        pass
     
     return {
         'has_issues': len(issues) > 0,
@@ -166,8 +172,7 @@ def generate_validation_report(
     report = []
     
     # Header
-    report.append(f"# Validation Report: {model_name or 'Submission'}")
-    report.append(f"Generated: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    report.append(f"## Validation Report: {model_name or 'Submission'}")
     report.append("")
     
     # File format validation
@@ -244,7 +249,9 @@ def validate_submission_complete(file_content: bytes, filename: str, test_set: p
         return {
             'valid': False,
             'report': generate_validation_report(format_result, {}, {}, model_name),
-            'predictions': None
+            'predictions': None,
+            'coverage': 0.0,
+            'pair_coverage': {}
         }
     
     predictions = format_result['dataframe']
@@ -270,5 +277,9 @@ def validate_submission_complete(file_content: bytes, filename: str, test_set: p
         'coverage': test_set_result['overall_coverage'],
         'report': report,
         'predictions': predictions,
-        'pair_coverage': test_set_result['pair_coverage']
+        'pair_coverage': test_set_result['pair_coverage'],
+        'quality_score': content_result.get('quality_score', 0.8),
+        'warnings': content_result.get('warnings', []),
+        'matching_count': test_set_result['matching_count'],
+        'missing_count': test_set_result['missing_count']
     }