.csv -> sw)
+ language = file_path.stem.split('_')[1] if '_' in file_path.stem else 'unknown'
+
+ report = ValidationReport(
+ file_path=str(file_path),
+ language=language,
+ total_entries=0,
+ valid_entries=0,
+ issues=[]
+ )
+
+ try:
+ with open(file_path, 'r', encoding='utf-8') as f:
+ reader = csv.DictReader(f)
+
+ # Validate header
+ header_issues = self._validate_header(reader.fieldnames or [])
+ report.issues.extend(header_issues)
+
+ # Validate each row
+ for row_num, row in enumerate(reader, start=2):
+ report.total_entries += 1
+ row_issues = self._validate_row(row, row_num)
+
+ if not any(i.severity == ValidationSeverity.ERROR for i in row_issues):
+ report.valid_entries += 1
+
+ report.issues.extend(row_issues)
+
+ except FileNotFoundError:
+ report.issues.append(ValidationIssue(
+ row_number=0,
+ column="file",
+ issue_type="FILE_NOT_FOUND",
+ severity=ValidationSeverity.ERROR,
+ message=f"Lexicon file not found: {file_path}"
+ ))
+ except Exception as e:
+ report.issues.append(ValidationIssue(
+ row_number=0,
+ column="file",
+ issue_type="FILE_READ_ERROR",
+ severity=ValidationSeverity.ERROR,
+ message=f"Error reading file: {str(e)}"
+ ))
+
+ return report
+
+ def _validate_header(self, fieldnames: List[str]) -> List[ValidationIssue]:
+ """Validate CSV header has required columns."""
+ issues = []
+
+ for col in self.REQUIRED_COLUMNS:
+ if col not in fieldnames:
+ issues.append(ValidationIssue(
+ row_number=1,
+ column=col,
+ issue_type="MISSING_REQUIRED_COLUMN",
+ severity=ValidationSeverity.ERROR,
+ message=f"Required column '{col}' is missing from header"
+ ))
+
+ for col in self.AIBRIDGE_COLUMNS:
+ if col not in fieldnames:
+ issues.append(ValidationIssue(
+ row_number=1,
+ column=col,
+ issue_type="MISSING_AIBRIDGE_COLUMN",
+ severity=ValidationSeverity.WARNING,
+ message=f"AI BRIDGE column '{col}' is missing - recommended for compliance"
+ ))
+
+ return issues
+
+ def _validate_row(self, row: Dict[str, str], row_num: int) -> List[ValidationIssue]:
+ """Validate a single lexicon row."""
+ issues = []
+ # Handle None values from CSV (when trailing columns are empty)
+ biased = (row.get('biased') or '').strip()
+ neutral = (row.get('neutral_primary') or '').strip()
+
+ # Skip empty rows
+ if not biased:
+ return issues
+
+ # Check 1: Identical biased and neutral terms (CRITICAL)
+ if biased and neutral and biased == neutral:
+ severity = ValidationSeverity.ERROR
+ issues.append(ValidationIssue(
+ row_number=row_num,
+ column="biased/neutral_primary",
+ issue_type="IDENTICAL_TERMS",
+ severity=severity,
+ message="Biased term is identical to neutral_primary - this entry is non-functional",
+ biased_term=biased,
+ suggestion="Either provide a different neutral term, or remove this entry if the term is inherently neutral"
+ ))
+
+ # Check 2: Empty neutral_primary (except for morphology/suffix entries)
+ tags = row.get('tags') or ''
+ if not neutral and 'morphology' not in tags and 'suffix' not in tags:
+ issues.append(ValidationIssue(
+ row_number=row_num,
+ column="neutral_primary",
+ issue_type="MISSING_NEUTRAL",
+ severity=ValidationSeverity.WARNING,
+ message="No neutral_primary provided",
+ biased_term=biased,
+ suggestion="Add a neutral alternative term"
+ ))
+
+ # Check 3: Identical example sentences
+ example_biased = (row.get('example_biased') or '').strip()
+ example_neutral = (row.get('example_neutral') or '').strip()
+
+ if example_biased and example_neutral:
+ if example_biased == example_neutral:
+ issues.append(ValidationIssue(
+ row_number=row_num,
+ column="example_biased/example_neutral",
+ issue_type="IDENTICAL_EXAMPLES",
+ severity=ValidationSeverity.ERROR,
+ message="Example sentences are identical - no pedagogical value",
+ biased_term=biased,
+ suggestion="Provide distinct examples that show the difference between biased and neutral usage"
+ ))
+ elif self._examples_too_similar(example_biased, example_neutral, biased, neutral):
+ issues.append(ValidationIssue(
+ row_number=row_num,
+ column="example_biased/example_neutral",
+ issue_type="SIMILAR_EXAMPLES",
+ severity=ValidationSeverity.WARNING,
+ message="Example sentences are nearly identical (only differ by the target term)",
+ biased_term=biased,
+ suggestion="Consider if the examples adequately demonstrate the bias"
+ ))
+
+ # Check 4: Missing examples
+ if not example_biased and example_neutral:
+ issues.append(ValidationIssue(
+ row_number=row_num,
+ column="example_biased",
+ issue_type="MISSING_EXAMPLE_BIASED",
+ severity=ValidationSeverity.WARNING,
+ message="Missing biased example sentence",
+ biased_term=biased
+ ))
+
+ if example_biased and not example_neutral:
+ issues.append(ValidationIssue(
+ row_number=row_num,
+ column="example_neutral",
+ issue_type="MISSING_EXAMPLE_NEUTRAL",
+ severity=ValidationSeverity.WARNING,
+ message="Missing neutral example sentence",
+ biased_term=biased
+ ))
+
+ # Check 5: AI BRIDGE metadata
+ bias_label = (row.get('bias_label') or '').strip()
+ stereotype_category = (row.get('stereotype_category') or '').strip()
+
+ if not bias_label:
+ issues.append(ValidationIssue(
+ row_number=row_num,
+ column="bias_label",
+ issue_type="MISSING_BIAS_LABEL",
+ severity=ValidationSeverity.INFO,
+ message="Missing bias_label (AI BRIDGE field)",
+ biased_term=biased,
+ suggestion="Add one of: stereotype, counter-stereotype, derogation, neutral"
+ ))
+
+ if not stereotype_category:
+ issues.append(ValidationIssue(
+ row_number=row_num,
+ column="stereotype_category",
+ issue_type="MISSING_STEREOTYPE_CATEGORY",
+ severity=ValidationSeverity.INFO,
+ message="Missing stereotype_category (AI BRIDGE field)",
+ biased_term=biased,
+ suggestion="Add one of: profession, family_role, leadership, capability, appearance, emotion, sexuality, violence, daily_life, intersectional"
+ ))
+
+ return issues
+
+ def _examples_too_similar(self, ex_biased: str, ex_neutral: str,
+ biased: str, neutral: str) -> bool:
+ """
+ Check if examples only differ by the biased/neutral term swap.
+
+ Returns True if the examples are essentially identical except for
+ the term being demonstrated.
+ """
+ # Normalize for comparison
+ ex_biased_norm = ex_biased.lower().replace(biased.lower(), '___TERM___')
+ ex_neutral_norm = ex_neutral.lower().replace(neutral.lower(), '___TERM___')
+
+ return ex_biased_norm == ex_neutral_norm
+
+ def validate_all_lexicons(self, rules_dir: str | Path = "rules") -> Dict[str, ValidationReport]:
+ """
+ Validate all lexicon files in a directory.
+
+ Args:
+ rules_dir: Directory containing lexicon files
+
+ Returns:
+ Dictionary mapping language codes to validation reports
+ """
+ rules_dir = Path(rules_dir)
+ reports = {}
+
+ for lexicon_file in rules_dir.glob(lexicon_glob_pattern()):
+ report = self.validate_file(lexicon_file)
+ reports[report.language] = report
+
+ return reports
+
+
+class LexiconValidationError(Exception):
+ """Raised when lexicon validation fails with errors."""
+
+ def __init__(self, report: ValidationReport):
+ self.report = report
+ super().__init__(f"Lexicon validation failed for {report.language}: {report.error_count} errors found")
+
+
+def validate_lexicon_on_load(file_path: str | Path,
+ strict: bool = False,
+ raise_on_error: bool = True) -> Tuple[bool, ValidationReport]:
+ """
+ Convenience function to validate a lexicon before loading.
+
+ Args:
+ file_path: Path to lexicon file
+ strict: If True, warnings become errors
+ raise_on_error: If True, raises LexiconValidationError on failure
+
+ Returns:
+ Tuple of (is_valid, report)
+
+ Raises:
+ LexiconValidationError: If validation fails and raise_on_error is True
+ """
+ validator = LexiconValidator(strict_mode=strict)
+ report = validator.validate_file(file_path)
+
+ if not report.is_valid and raise_on_error:
+ raise LexiconValidationError(report)
+
+ return report.is_valid, report
+
+
+# CLI interface for running validation standalone
+if __name__ == "__main__":
+ import sys
+
+ print("=" * 60)
+ print("LEXICON VALIDATION TOOL")
+ print("AI BRIDGE Compliance Checker")
+ print("=" * 60)
+
+ validator = LexiconValidator()
+
+ if len(sys.argv) > 1:
+ # Validate specific file
+ file_path = sys.argv[1]
+ report = validator.validate_file(file_path)
+ print(report.summary())
+ sys.exit(0 if report.is_valid else 1)
+ else:
+ # Validate all lexicons
+ reports = validator.validate_all_lexicons()
+
+ all_valid = True
+ total_errors = 0
+ total_warnings = 0
+
+ for lang, report in reports.items():
+ print(report.summary())
+ if not report.is_valid:
+ all_valid = False
+ total_errors += report.error_count
+ total_warnings += report.warning_count
+
+ print("\n" + "=" * 60)
+ print("OVERALL SUMMARY")
+ print("=" * 60)
+ print(f"Languages validated: {len(reports)}")
+ print(f"Total errors: {total_errors}")
+ print(f"Total warnings: {total_warnings}")
+ print(f"Overall status: {'PASS' if all_valid else 'FAIL'}")
+ print("=" * 60)
+
+ sys.exit(0 if all_valid else 1)
diff --git a/eval/metrics_calculator.py b/eval/metrics_calculator.py
new file mode 100644
index 0000000000000000000000000000000000000000..b58d1c953e78c1af99d868aec8177d8ba2da3ca4
--- /dev/null
+++ b/eval/metrics_calculator.py
@@ -0,0 +1,213 @@
+"""
+Metrics calculation service for bias detection evaluation.
+
+This module provides clean interfaces for calculating evaluation metrics.
+"""
+from typing import List, Dict
+from collections import defaultdict
+
+from .models import (
+ EvaluationMetrics,
+ LanguageEvaluationResult,
+ GroundTruthSample,
+ BiasDetectionResult,
+ Language,
+ BiasCategory
+)
+
+
+class MetricsCalculator:
+ """
+ Service for calculating evaluation metrics from predictions and ground truth.
+
+ Provides methods for calculating precision, recall, F1 scores both overall
+ and per-category.
+ """
+
+ def calculate_language_metrics(
+ self,
+ ground_truth: List[GroundTruthSample],
+ predictions: List[BiasDetectionResult],
+ language: Language
+ ) -> LanguageEvaluationResult:
+ """
+ Calculate comprehensive evaluation metrics for a language.
+
+ Args:
+ ground_truth: List of ground truth samples
+ predictions: List of prediction results
+ language: Language being evaluated
+
+ Returns:
+ LanguageEvaluationResult with overall and per-category metrics
+
+ Raises:
+ ValueError: If ground truth and predictions don't match in length
+ """
+ if len(ground_truth) != len(predictions):
+ raise ValueError(
+ f"Ground truth ({len(ground_truth)}) and predictions ({len(predictions)}) "
+ f"must have the same length"
+ )
+
+ # Calculate overall metrics
+ overall_metrics = self._calculate_overall_metrics(ground_truth, predictions)
+
+ # Calculate per-category metrics
+ category_metrics = self._calculate_category_metrics(ground_truth, predictions)
+
+ return LanguageEvaluationResult(
+ language=language,
+ overall_metrics=overall_metrics,
+ category_metrics=category_metrics,
+ total_samples=len(ground_truth)
+ )
+
+ def _calculate_overall_metrics(
+ self,
+ ground_truth: List[GroundTruthSample],
+ predictions: List[BiasDetectionResult]
+ ) -> EvaluationMetrics:
+ """Calculate overall evaluation metrics."""
+ tp = fp = fn = tn = 0
+
+ for gt, pred in zip(ground_truth, predictions):
+ if pred.has_bias_detected and gt.has_bias:
+ tp += 1
+ elif pred.has_bias_detected and not gt.has_bias:
+ fp += 1
+ elif not pred.has_bias_detected and gt.has_bias:
+ fn += 1
+ else: # not pred.has_bias_detected and not gt.has_bias
+ tn += 1
+
+ return self._calculate_metrics_from_counts(tp, fp, fn, tn)
+
+ def _calculate_category_metrics(
+ self,
+ ground_truth: List[GroundTruthSample],
+ predictions: List[BiasDetectionResult]
+ ) -> Dict[BiasCategory, EvaluationMetrics]:
+ """Calculate per-category evaluation metrics."""
+ # Group samples by category
+ category_data = defaultdict(list)
+
+ for gt, pred in zip(ground_truth, predictions):
+ category_data[gt.bias_category].append((gt, pred))
+
+ # Calculate metrics for each category
+ category_metrics = {}
+
+ for category, samples in category_data.items():
+ if category == BiasCategory.NONE:
+ continue # Skip non-biased samples for category metrics
+
+ tp = fp = fn = tn = 0
+
+ for gt, pred in samples:
+ if pred.has_bias_detected and gt.has_bias:
+ tp += 1
+ elif pred.has_bias_detected and not gt.has_bias:
+ fp += 1
+ elif not pred.has_bias_detected and gt.has_bias:
+ fn += 1
+ else:
+ tn += 1
+
+ category_metrics[category] = self._calculate_metrics_from_counts(tp, fp, fn, tn)
+
+ return category_metrics
+
+ def _calculate_metrics_from_counts(self, tp: int, fp: int, fn: int, tn: int) -> EvaluationMetrics:
+ """Calculate metrics from confusion matrix counts."""
+ precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+ recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+ f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+
+ return EvaluationMetrics(
+ precision=precision,
+ recall=recall,
+ f1_score=f1_score,
+ true_positives=tp,
+ false_positives=fp,
+ false_negatives=fn,
+ true_negatives=tn
+ )
+
+
+class MetricsFormatter:
+ """
+ Service for formatting evaluation metrics for display and export.
+
+ Provides methods to convert metrics objects into various output formats.
+ """
+
+ def format_for_csv(self, results: List[LanguageEvaluationResult]) -> List[Dict[str, str]]:
+ """
+ Format evaluation results for CSV export.
+
+ Args:
+ results: List of language evaluation results
+
+ Returns:
+ List of dictionaries suitable for CSV writing
+ """
+ csv_rows = []
+
+ for result in results:
+ lang_name = result.language.value.upper()
+
+ # Add overall metrics row
+ csv_rows.append({
+ 'Language': lang_name,
+ 'Category': 'OVERALL',
+ 'Precision': f"{result.overall_metrics.precision:.3f}",
+ 'Recall': f"{result.overall_metrics.recall:.3f}",
+ 'F1_Score': f"{result.overall_metrics.f1_score:.3f}",
+ 'TP': str(result.overall_metrics.true_positives),
+ 'FP': str(result.overall_metrics.false_positives),
+ 'FN': str(result.overall_metrics.false_negatives),
+ 'TN': str(result.overall_metrics.true_negatives)
+ })
+
+ # Add category-specific metrics rows
+ for category, metrics in result.category_metrics.items():
+ csv_rows.append({
+ 'Language': lang_name,
+ 'Category': category.value,
+ 'Precision': f"{metrics.precision:.3f}",
+ 'Recall': f"{metrics.recall:.3f}",
+ 'F1_Score': f"{metrics.f1_score:.3f}",
+ 'TP': str(metrics.true_positives),
+ 'FP': str(metrics.false_positives),
+ 'FN': str(metrics.false_negatives),
+ 'TN': str(metrics.true_negatives)
+ })
+
+ return csv_rows
+
+ def format_for_console(self, results: List[LanguageEvaluationResult]) -> str:
+ """
+ Format evaluation results for console display.
+
+ Args:
+ results: List of language evaluation results
+
+ Returns:
+ Formatted string for console output
+ """
+ output_lines = ["Running bias detection evaluation..."]
+
+ for result in results:
+ lang_name = "English" if result.language == Language.ENGLISH else "Swahili"
+
+ output_lines.extend([
+ f"Evaluating {result.language.value}...",
+ f"{lang_name} Results:",
+ f" Overall F1: {result.overall_metrics.f1_score:.3f}",
+ f" Precision: {result.overall_metrics.precision:.3f}",
+ f" Recall: {result.overall_metrics.recall:.3f}",
+ ""
+ ])
+
+ return "\n".join(output_lines)
\ No newline at end of file
diff --git a/eval/ml_detector.py b/eval/ml_detector.py
new file mode 100644
index 0000000000000000000000000000000000000000..e9763a807dd9ef0fb9e27ddbd115e9805f6d5427
--- /dev/null
+++ b/eval/ml_detector.py
@@ -0,0 +1,85 @@
+"""
+ML-based bias detector using transformer models for African languages
+"""
+import re
+from typing import Dict, List, Optional
+from .models import BiasDetectionResult, Language
+
+class MLBiasDetector:
+ """Machine learning bias detector using pre-trained models"""
+
+ def __init__(self):
+ self.models = self._load_models()
+
+ def _load_models(self) -> Dict[Language, str]:
+ """Load appropriate models for each language"""
+ return {
+ Language.ENGLISH: "distilbert-base-uncased",
+ Language.SWAHILI: "xlm-roberta-base",
+ Language.FRENCH: "xlm-roberta-base",
+ Language.GIKUYU: "xlm-roberta-base"
+ }
+
+ def detect_bias(self, text: str, language: Language) -> BiasDetectionResult:
+ """Detect bias using ML model (simplified implementation)"""
+ # Simulate ML model prediction
+ bias_score = self._predict_bias_score(text, language)
+
+ if bias_score > 0.7: # High confidence threshold
+ edits = self._extract_biased_terms(text, language)
+ return BiasDetectionResult(
+ text=text,
+ has_bias_detected=True,
+ detected_edits=edits
+ )
+
+ return BiasDetectionResult(
+ text=text,
+ has_bias_detected=False,
+ detected_edits=[]
+ )
+
+ def _predict_bias_score(self, text: str, language: Language) -> float:
+ """Simulate ML model bias prediction"""
+ # Simplified bias indicators for demo
+ bias_patterns = {
+ Language.ENGLISH: ['chairman', 'businessman', 'policeman', 'fireman'],
+ Language.SWAHILI: ['mwanaume', 'bwana'],
+ Language.FRENCH: ['président', 'directeur', 'policier'],
+ Language.GIKUYU: ['mũndũ mũrũme', 'mũrũme']
+ }
+
+ patterns = bias_patterns.get(language, [])
+ text_lower = text.lower()
+
+ # Simple scoring based on pattern matches
+ matches = sum(1 for pattern in patterns if pattern in text_lower)
+ return min(matches * 0.4, 1.0)
+
+ def _extract_biased_terms(self, text: str, language: Language) -> List[Dict[str, str]]:
+ """Extract biased terms and suggest corrections"""
+ corrections = {
+ Language.ENGLISH: {
+ 'chairman': 'chair',
+ 'businessman': 'businessperson',
+ 'policeman': 'police officer',
+ 'fireman': 'firefighter'
+ },
+ Language.SWAHILI: {
+ 'mwanaume': 'mtu',
+ 'bwana': 'mkuu'
+ }
+ }
+
+ lang_corrections = corrections.get(language, {})
+ edits = []
+
+ for biased_term, correction in lang_corrections.items():
+ if biased_term.lower() in text.lower():
+ edits.append({
+ 'from': biased_term,
+ 'to': correction,
+ 'severity': 'replace'
+ })
+
+ return edits
\ No newline at end of file
diff --git a/eval/ml_evaluation.py b/eval/ml_evaluation.py
new file mode 100644
index 0000000000000000000000000000000000000000..39e43e0f4a51e68048746355ca536123303942e1
--- /dev/null
+++ b/eval/ml_evaluation.py
@@ -0,0 +1,120 @@
+"""
+ML model evaluation comparing rules-based vs ML vs hybrid approaches
+"""
+import csv
+from typing import Dict, List
+from .bias_detector import BiasDetector
+from .ml_detector import MLBiasDetector
+from .hybrid_detector import HybridBiasDetector
+from .models import Language, EvaluationMetrics
+
+class MLEvaluationFramework:
+ """Evaluate and compare different detection approaches"""
+
+ def __init__(self):
+ self.rules_detector = BiasDetector()
+ self.ml_detector = MLBiasDetector()
+ self.hybrid_detector = HybridBiasDetector()
+
+ def run_comparative_evaluation(self) -> Dict:
+ """Run evaluation across all approaches and languages"""
+ results = {}
+
+ for language in Language:
+ print(f"\nEvaluating {language.value}...")
+
+ # Load ground truth
+ ground_truth = self._load_ground_truth(language)
+
+ # Evaluate each approach
+ rules_metrics = self._evaluate_approach(self.rules_detector, ground_truth, language)
+ ml_metrics = self._evaluate_approach(self.ml_detector, ground_truth, language)
+ hybrid_metrics = self._evaluate_approach(self.hybrid_detector, ground_truth, language)
+
+ results[language.value] = {
+ 'rules_based': rules_metrics,
+ 'ml_based': ml_metrics,
+ 'hybrid': hybrid_metrics,
+ 'sample_count': len(ground_truth)
+ }
+
+ # Print comparison
+ self._print_comparison(language, rules_metrics, ml_metrics, hybrid_metrics)
+
+ return results
+
+ def _evaluate_approach(self, detector, ground_truth: List, language: Language) -> EvaluationMetrics:
+ """Evaluate single detection approach"""
+ tp = fp = fn = tn = 0
+
+ for sample in ground_truth:
+ result = detector.detect_bias(sample['text'], language)
+ predicted = result.has_bias_detected
+ actual = sample['has_bias'] == 'True'
+
+ if predicted and actual:
+ tp += 1
+ elif predicted and not actual:
+ fp += 1
+ elif not predicted and actual:
+ fn += 1
+ else:
+ tn += 1
+
+ precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+ recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+ f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0
+
+ return EvaluationMetrics(
+ precision=precision,
+ recall=recall,
+ f1_score=f1,
+ true_positives=tp,
+ false_positives=fp,
+ false_negatives=fn,
+ true_negatives=tn
+ )
+
+ def _load_ground_truth(self, language: Language) -> List[Dict]:
+ """Load ground truth data for language"""
+ filename = f"eval/ground_truth_{language.value}.csv"
+ ground_truth = []
+
+ try:
+ with open(filename, 'r', encoding='utf-8') as f:
+ reader = csv.DictReader(f)
+ ground_truth = list(reader)
+ except FileNotFoundError:
+ print(f"Warning: Ground truth file {filename} not found")
+
+ return ground_truth
+
+ def _print_comparison(self, language: Language, rules: EvaluationMetrics,
+ ml: EvaluationMetrics, hybrid: EvaluationMetrics):
+ """Print comparison table for language"""
+ print(f"\n{language.value.upper()} COMPARISON:")
+ print("Approach | F1 | Precision | Recall")
+ print("-" * 40)
+ print(f"Rules-based | {rules.f1_score:.3f} | {rules.precision:.3f} | {rules.recall:.3f}")
+ print(f"ML-based | {ml.f1_score:.3f} | {ml.precision:.3f} | {ml.recall:.3f}")
+ print(f"Hybrid | {hybrid.f1_score:.3f} | {hybrid.precision:.3f} | {hybrid.recall:.3f}")
+
+if __name__ == "__main__":
+ evaluator = MLEvaluationFramework()
+ results = evaluator.run_comparative_evaluation()
+
+ print("\n" + "="*60)
+ print("SUMMARY: Best F1 Scores by Language")
+ print("="*60)
+
+ for lang, metrics in results.items():
+ best_f1 = max(
+ metrics['rules_based'].f1_score,
+ metrics['ml_based'].f1_score,
+ metrics['hybrid'].f1_score
+ )
+
+ best_approach = 'rules' if metrics['rules_based'].f1_score == best_f1 else \
+ 'ml' if metrics['ml_based'].f1_score == best_f1 else 'hybrid'
+
+ print(f"{lang}: {best_f1:.3f} ({best_approach})")
\ No newline at end of file
diff --git a/eval/models.py b/eval/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..f9c3c926ce3bdee095395e7483af97c2b9b2f0df
--- /dev/null
+++ b/eval/models.py
@@ -0,0 +1,207 @@
+"""
+Simplified data models for bias evaluation framework without external dependencies.
+
+This module defines the data structures used throughout the evaluation system
+using only standard library components.
+
+AI BRIDGE Compliance: Implements bias constructs from the AI BRIDGE guidelines
+including stereotype, counter-stereotype, derogation, and neutral classifications.
+"""
+from enum import Enum
+from typing import List, Dict, Any, Optional
+from dataclasses import dataclass, field
+
+
+class BiasCategory(str, Enum):
+ """Enumeration of bias categories for classification (detection mechanism)."""
+ OCCUPATION = "occupation"
+ PRONOUN_ASSUMPTION = "pronoun_assumption"
+ PRONOUN_GENERIC = "pronoun_generic"
+ HONORIFIC = "honorific"
+ MORPHOLOGY = "morphology"
+ NONE = "none"
+ STEREOTYPE="stereotype"
+
+
+class BiasLabel(str, Enum):
+ """
+ AI BRIDGE bias label classification.
+
+ Defines the type of representational bias present in text:
+ - stereotype: Reinforces common, often oversimplified beliefs about a group
+ - counter_stereotype: Challenges or contradicts common stereotypes
+ - derogation: Language that demeans or disparages a group
+ - neutral: No bias or stereotype present
+ """
+ STEREOTYPE = "stereotype"
+ COUNTER_STEREOTYPE = "counter-stereotype"
+ DEROGATION = "derogation"
+ NEUTRAL = "neutral"
+
+
+class StereotypeCategory(str, Enum):
+ """
+ AI BRIDGE stereotype category classification.
+
+ Thematic areas where gender stereotypes commonly manifest.
+ """
+ PROFESSION = "profession"
+ FAMILY_ROLE = "family_role"
+ LEADERSHIP = "leadership"
+ EDUCATION = "education"
+ RELIGION_CULTURE = "religion_culture"
+ PROVERB_IDIOM = "proverb_idiom"
+ DAILY_LIFE = "daily_life"
+ APPEARANCE = "appearance"
+ CAPABILITY = "capability"
+ NONE = "none"
+
+
+class TargetGender(str, Enum):
+ """
+ AI BRIDGE target gender classification.
+
+ Who is being talked about, referenced, or implied in the text.
+ """
+ FEMALE = "female"
+ MALE = "male"
+ NEUTRAL = "neutral"
+ MIXED = "mixed"
+ NONBINARY = "nonbinary"
+ UNKNOWN = "unknown"
+
+
+class Explicitness(str, Enum):
+ """
+ AI BRIDGE explicitness classification.
+
+ Whether the bias is directly stated or implied through context.
+ """
+ EXPLICIT = "explicit"
+ IMPLICIT = "implicit"
+
+
+class Sentiment(str, Enum):
+ """Emotional tone toward the gendered referent."""
+ POSITIVE = "positive"
+ NEUTRAL = "neutral"
+ NEGATIVE = "negative"
+
+
+class SafetyFlag(str, Enum):
+ """Content safety classification."""
+ SAFE = "safe"
+ SENSITIVE = "sensitive"
+ REJECT = "reject"
+
+
+class QAStatus(str, Enum):
+ """Quality assurance status for annotations."""
+ GOLD = "gold"
+ PASSED = "passed"
+ NEEDS_REVIEW = "needs_review"
+ REJECTED = "rejected"
+
+
+class Language(str, Enum):
+ """Supported languages for bias detection."""
+ ENGLISH = "en"
+ SWAHILI = "sw"
+ FRENCH = "fr"
+ GIKUYU = "ki"
+
+
+@dataclass
+class GroundTruthSample:
+ """
+ Single ground truth test case for evaluation.
+
+ Supports both legacy 4-field format and full AI BRIDGE 29-field format.
+ """
+ # Core required fields
+ text: str
+ has_bias: bool
+ bias_category: BiasCategory
+ expected_correction: str
+
+ # AI BRIDGE extended fields (optional for backward compatibility)
+ id: Optional[str] = None
+ language: Optional[str] = None
+ script: Optional[str] = None
+ country: Optional[str] = None
+ region_dialect: Optional[str] = None
+ source_type: Optional[str] = None
+ source_ref: Optional[str] = None
+ collection_date: Optional[str] = None
+ translation: Optional[str] = None
+ domain: Optional[str] = None
+ topic: Optional[str] = None
+ theme: Optional[str] = None
+ sensitive_characteristic: Optional[str] = None
+
+ # AI BRIDGE bias annotation fields
+ target_gender: Optional[TargetGender] = None
+ bias_label: Optional[BiasLabel] = None
+ stereotype_category: Optional[StereotypeCategory] = None
+ explicitness: Optional[Explicitness] = None
+ bias_severity: Optional[int] = None # 1-3 scale
+ sentiment_toward_referent: Optional[Sentiment] = None
+ device: Optional[str] = None # metaphor, proverb, sarcasm, etc.
+
+ # Quality and safety fields
+ safety_flag: Optional[SafetyFlag] = None
+ pii_removed: Optional[bool] = None
+ annotator_id: Optional[str] = None
+ qa_status: Optional[QAStatus] = None
+ approver_id: Optional[str] = None
+ cohen_kappa: Optional[float] = None
+ notes: Optional[str] = None
+ eval_split: Optional[str] = None # train, validation, test
+
+
+@dataclass
+class BiasDetectionResult:
+ """Result of bias detection on a single text sample."""
+ text: str
+ has_bias_detected: bool
+ detected_edits: List[Dict[str, str]]
+
+ # AI BRIDGE extended detection results
+ bias_label: Optional[BiasLabel] = None
+ stereotype_category: Optional[StereotypeCategory] = None
+ target_gender: Optional[TargetGender] = None
+ explicitness: Optional[Explicitness] = None
+ confidence: Optional[float] = None
+
+
+@dataclass
+class EvaluationMetrics:
+ """Evaluation metrics for bias detection performance."""
+ precision: float
+ recall: float
+ f1_score: float
+ true_positives: int
+ false_positives: int
+ false_negatives: int
+ true_negatives: int
+
+
+@dataclass
+class LanguageEvaluationResult:
+ """Complete evaluation results for a single language."""
+ language: Language
+ overall_metrics: EvaluationMetrics
+ category_metrics: Dict[BiasCategory, EvaluationMetrics]
+ total_samples: int
+
+
+@dataclass
+class FailureCase:
+ """Analysis of a failed prediction case."""
+ failure_type: str
+ input_text: str
+ expected: bool
+ predicted: bool
+ category: BiasCategory
+ diagnosis: str
+ language: Language
\ No newline at end of file
diff --git a/eval/mt5_corrector.py b/eval/mt5_corrector.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8ff5967b0289cad4bd049be2f3159e2d48fbc2f
--- /dev/null
+++ b/eval/mt5_corrector.py
@@ -0,0 +1,64 @@
+"""
+MT5-based bias correction using the generative approach from dev branch
+"""
+import time
+from typing import Dict, Any
+from .models import Language
+
+class MT5BiasCorrector:
+ """MT5-based bias correction system"""
+
+ def __init__(self):
+ self.model_id = "google/mt5-small"
+ self._tokenizer = None
+ self._model = None
+
+ def _ensure_model(self):
+ """Lazy load model to avoid import errors without transformers"""
+ if self._tokenizer is None:
+ try:
+ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+ import torch
+
+ self._tokenizer = AutoTokenizer.from_pretrained(self.model_id)
+ self._model = AutoModelForSeq2SeqLM.from_pretrained(self.model_id)
+ self._device = "cuda" if torch.cuda.is_available() else "cpu"
+ self._model.to(self._device)
+ self._model.eval()
+ except ImportError:
+ raise ImportError("transformers and torch required for MT5 correction")
+
+ def correct_bias(self, text: str, language: Language, num_candidates: int = 3) -> Dict[str, Any]:
+ """Generate bias-corrected versions of text"""
+ self._ensure_model()
+ start = time.time()
+
+ # Language-specific prompting
+ lang_code = language.value
+ prompt = f"Rewrite to remove gender bias while preserving meaning (language={lang_code}): {text}"
+
+ inputs = self._tokenizer(prompt, return_tensors="pt", truncation=True, padding=True).to(self._device)
+
+ outputs = self._model.generate(
+ **inputs,
+ max_new_tokens=64,
+ num_beams=max(2, num_candidates),
+ num_return_sequences=num_candidates,
+ early_stopping=True
+ )
+
+ candidates = [
+ self._tokenizer.decode(o, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+ for o in outputs
+ ]
+
+ latency_ms = int((time.time() - start) * 1000)
+
+ return {
+ "original": text,
+ "best_correction": candidates[0] if candidates else text,
+ "candidates": candidates,
+ "model": self.model_id,
+ "language": lang_code,
+ "latency_ms": latency_ms
+ }
\ No newline at end of file
diff --git a/eval/ngeli_tracker.py b/eval/ngeli_tracker.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a0ded18f324f6660032c3346c33a0ea12832177
--- /dev/null
+++ b/eval/ngeli_tracker.py
@@ -0,0 +1,285 @@
+"""
+Swahili noun class (ngeli) tracking module.
+
+This module provides utilities for tracking and analyzing Swahili noun classes,
+which is crucial for understanding agreement patterns and gender marking in Swahili.
+
+Swahili has 18 noun classes organized into pairs:
+- 1/2 (m-wa): People, animate beings (mtu/watu)
+- 3/4 (m-mi): Plants, body parts (mti/miti)
+- 5/6 (ji-ma): Fruits, paired items (jiwe/mawe)
+- 7/8 (ki-vi): Things, diminutives (kitu/vitu)
+- 9/10 (n-n): Animals, loanwords (ndege/ndege)
+- 11/10 (u-n): Abstract nouns (ukuta/kuta)
+- 15 (ku-): Infinitives (kukimbia)
+- 16/17/18 (pa-ku-mu): Locatives (mahali)
+"""
+
+from typing import Dict, List, Optional
+from dataclasses import dataclass
+from enum import Enum
+
+
+class NounClass(Enum):
+ """Swahili noun classes (ngeli)"""
+ M_WA = "1/2" # People, animate (mwalimu/walimu)
+ M_MI = "3/4" # Plants, natural objects (mti/miti)
+ JI_MA = "5/6" # Fruits, paired items (jiwe/mawe)
+ KI_VI = "7/8" # Things, diminutives (kitu/vitu)
+ N_N = "9/10" # Animals, loanwords (ndege/ndege)
+ U_N = "11/10" # Abstract nouns (ukuta/kuta)
+ KU = "15" # Infinitives (kukimbia)
+ PA = "16" # Locative (specific place)
+ KU_LOC = "17" # Locative (general)
+ MU_LOC = "18" # Locative (inside)
+ MA = "6" # Plural only (maji - water)
+
+
+@dataclass
+class NounClassInfo:
+ """Information about a noun's class"""
+ noun_class: NounClass
+ number: str # sg, pl, or both
+ prefix_singular: str
+ prefix_plural: str
+ agreement_pattern: str
+ examples: List[str]
+
+
+class NgeliTracker:
+ """
+ Tracks Swahili noun classes and agreement patterns.
+
+ This class provides utilities for:
+ - Identifying noun class from prefix
+ - Tracking subject-verb agreement
+ - Detecting possessive pronoun agreement
+ - Analyzing gender marking patterns
+ """
+
+ # Noun class patterns
+ NOUN_CLASS_PATTERNS = {
+ NounClass.M_WA: NounClassInfo(
+ noun_class=NounClass.M_WA,
+ number="sg/pl",
+ prefix_singular="m-, mw-, mu-",
+ prefix_plural="wa-, w-",
+ agreement_pattern="a-/wa- (subject), -ake/-ao (possessive)",
+ examples=["mwalimu/walimu", "mtu/watu", "mkulima/wakulima"]
+ ),
+ NounClass.M_MI: NounClassInfo(
+ noun_class=NounClass.M_MI,
+ number="sg/pl",
+ prefix_singular="m-, mw-",
+ prefix_plural="mi-",
+ agreement_pattern="u-/i- (subject), -ake/-ao (possessive)",
+ examples=["mti/miti", "mkono/mikono"]
+ ),
+ NounClass.JI_MA: NounClassInfo(
+ noun_class=NounClass.JI_MA,
+ number="sg/pl",
+ prefix_singular="ji-, j-, ø-",
+ prefix_plural="ma-",
+ agreement_pattern="li-/ya- (subject), -ake/-ao (possessive)",
+ examples=["jiwe/mawe", "gari/magari"]
+ ),
+ NounClass.KI_VI: NounClassInfo(
+ noun_class=NounClass.KI_VI,
+ number="sg/pl",
+ prefix_singular="ki-, ch-",
+ prefix_plural="vi-, vy-",
+ agreement_pattern="ki-/vi- (subject), -ake/-ao (possessive)",
+ examples=["kitu/vitu", "kitabu/vitabu"]
+ ),
+ NounClass.N_N: NounClassInfo(
+ noun_class=NounClass.N_N,
+ number="sg/pl",
+ prefix_singular="n-, ny-, m-, ø-",
+ prefix_plural="n-, ny-, m-, ø-",
+ agreement_pattern="i-/zi- (subject), -ake/-ao (possessive)",
+ examples=["ndege/ndege", "nyumba/nyumba"]
+ ),
+ NounClass.MA: NounClassInfo(
+ noun_class=NounClass.MA,
+ number="pl",
+ prefix_singular="",
+ prefix_plural="ma-",
+ agreement_pattern="ya- (subject), -ao (possessive)",
+ examples=["maji (water)", "maziwa (milk)"]
+ ),
+ }
+
+ # M-wa class prefixes (people/occupations - most relevant for gender bias)
+ M_WA_PREFIXES = {
+ 'singular': ['m', 'mw', 'mu'],
+ 'plural': ['wa', 'w']
+ }
+
+ # Possessive pronoun patterns by class
+ POSSESSIVE_PATTERNS = {
+ NounClass.M_WA: {
+ 'singular': ['wake', 'wako', 'wangu', 'wetu', 'wenu', 'wao'],
+ 'plural': ['wao', 'wako', 'wangu', 'wetu', 'wenu', 'wao']
+ },
+ # Add other classes as needed
+ }
+
+ def __init__(self):
+ """Initialize ngeli tracker"""
+ self.tracked_nouns: Dict[str, NounClass] = {}
+
+ def identify_class(self, noun: str) -> Optional[NounClass]:
+ """
+ Identify noun class from prefix.
+
+ Args:
+ noun: Swahili noun to analyze
+
+ Returns:
+ NounClass if identifiable, None otherwise
+ """
+ noun_lower = noun.lower().strip()
+
+ # M-wa class (people) - most important for bias detection
+ if any(noun_lower.startswith(prefix) for prefix in ['mw', 'mu', 'm']):
+ # Check if it's likely a person noun (occupation, role)
+ # This heuristic can be improved with corpus analysis
+ if any(marker in noun_lower for marker in ['limu', 'kulima', 'andishi', 'fanya']):
+ return NounClass.M_WA
+
+ # Wa- prefix indicates plural m-wa class
+ if any(noun_lower.startswith(prefix) for prefix in ['wa', 'w']):
+ return NounClass.M_WA
+
+ # Ma- prefix (class 6 plural or class 5/6)
+ if noun_lower.startswith('ma'):
+ return NounClass.JI_MA
+
+ # Ki-/Vi- prefix (class 7/8)
+ if noun_lower.startswith('ki') or noun_lower.startswith('ch'):
+ return NounClass.KI_VI
+ if noun_lower.startswith('vi') or noun_lower.startswith('vy'):
+ return NounClass.KI_VI
+
+ # N- prefix (class 9/10)
+ if noun_lower.startswith('n') or noun_lower.startswith('ny'):
+ return NounClass.N_N
+
+ return None
+
+ def is_m_wa_class(self, noun: str) -> bool:
+ """
+ Check if noun belongs to m-wa class (people).
+
+ This is the most important class for gender bias detection
+ as it includes all occupation and role nouns.
+
+ Args:
+ noun: Swahili noun to check
+
+ Returns:
+ True if noun is in m-wa class
+ """
+ noun_class = self.identify_class(noun)
+ return noun_class == NounClass.M_WA
+
+ def get_expected_agreement(self, noun: str, number: str = "sg") -> Optional[str]:
+ """
+ Get expected subject agreement prefix for a noun.
+
+ Args:
+ noun: Swahili noun
+ number: 'sg' or 'pl'
+
+ Returns:
+ Expected agreement prefix (e.g., 'a-' for m-wa singular)
+ """
+ noun_class = self.identify_class(noun)
+
+ if noun_class == NounClass.M_WA:
+ return 'a-' if number == 'sg' else 'wa-'
+ elif noun_class == NounClass.M_MI:
+ return 'u-' if number == 'sg' else 'i-'
+ elif noun_class == NounClass.JI_MA:
+ return 'li-' if number == 'sg' else 'ya-'
+ elif noun_class == NounClass.KI_VI:
+ return 'ki-' if number == 'sg' else 'vi-'
+ elif noun_class == NounClass.N_N:
+ return 'i-' if number == 'sg' else 'zi-'
+
+ return None
+
+ def track_noun(self, noun: str, noun_class: Optional[NounClass] = None):
+ """
+ Track a noun and its class.
+
+ Args:
+ noun: Swahili noun to track
+ noun_class: Optional explicit class (auto-detected if not provided)
+ """
+ if noun_class is None:
+ noun_class = self.identify_class(noun)
+
+ if noun_class:
+ self.tracked_nouns[noun] = noun_class
+
+ def get_statistics(self) -> Dict[str, int]:
+ """
+ Get statistics on tracked nouns by class.
+
+ Returns:
+ Dictionary mapping class names to counts
+ """
+ stats = {}
+ for noun_class in self.tracked_nouns.values():
+ class_name = noun_class.value
+ stats[class_name] = stats.get(class_name, 0) + 1
+
+ return stats
+
+ def analyze_text(self, text: str) -> Dict[str, any]:
+ """
+ Analyze text for noun class patterns.
+
+ Args:
+ text: Swahili text to analyze
+
+ Returns:
+ Dictionary with analysis results
+ """
+ words = text.split()
+ m_wa_nouns = []
+ other_nouns = []
+
+ for word in words:
+ # Remove punctuation
+ word_clean = word.strip('.,!?;:')
+ if len(word_clean) < 3:
+ continue
+
+ noun_class = self.identify_class(word_clean)
+ if noun_class == NounClass.M_WA:
+ m_wa_nouns.append(word_clean)
+ elif noun_class:
+ other_nouns.append((word_clean, noun_class.value))
+
+ return {
+ 'm_wa_nouns': m_wa_nouns,
+ 'm_wa_count': len(m_wa_nouns),
+ 'other_nouns': other_nouns,
+ 'total_nouns': len(m_wa_nouns) + len(other_nouns)
+ }
+
+
+def get_noun_class_info(noun_class: NounClass) -> NounClassInfo:
+ """
+ Get detailed information about a noun class.
+
+ Args:
+ noun_class: NounClass enum value
+
+ Returns:
+ NounClassInfo with patterns and examples
+ """
+ tracker = NgeliTracker()
+ return tracker.NOUN_CLASS_PATTERNS.get(noun_class)
diff --git a/eval/results/correction_eval_20251127_092129.json b/eval/results/correction_eval_20251127_092129.json
new file mode 100644
index 0000000000000000000000000000000000000000..795ed0137dad92f3e140fd553f2c3d503e110cb0
--- /dev/null
+++ b/eval/results/correction_eval_20251127_092129.json
@@ -0,0 +1,307 @@
+[
+ {
+ "language": "en",
+ "total_samples": 66,
+ "biased_samples": 34,
+ "overall_metrics": {
+ "pre_correction": {
+ "tp": 21,
+ "fp": 0,
+ "tn": 32,
+ "fn": 13,
+ "precision": 1.0,
+ "recall": 0.6176470588235294,
+ "f1_score": 0.7636363636363637
+ },
+ "post_correction": {
+ "tp": 0,
+ "fp": 0,
+ "tn": 32,
+ "fn": 34,
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removal_count": 21,
+ "detected_and_removed": 21
+ },
+ "category_metrics": {
+ "occupation": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.8636363636363636,
+ "f1_score": 0.9268292682926829
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 19,
+ "detected_count": 19
+ },
+ "pronoun_assumption": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.14285714285714285,
+ "f1_score": 0.25
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 1,
+ "detected_count": 1
+ },
+ "pronoun_generic": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.2,
+ "f1_score": 0.33333333333333337
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 1,
+ "detected_count": 1
+ }
+ },
+ "correction_quality": {
+ "meaning_preserved": 21,
+ "over_corrections": 0,
+ "successful_corrections": 21
+ }
+ },
+ {
+ "language": "sw",
+ "total_samples": 63,
+ "biased_samples": 31,
+ "overall_metrics": {
+ "pre_correction": {
+ "tp": 16,
+ "fp": 0,
+ "tn": 32,
+ "fn": 15,
+ "precision": 1.0,
+ "recall": 0.5161290322580645,
+ "f1_score": 0.6808510638297872
+ },
+ "post_correction": {
+ "tp": 14,
+ "fp": 0,
+ "tn": 32,
+ "fn": 17,
+ "precision": 1.0,
+ "recall": 0.45161290322580644,
+ "f1_score": 0.6222222222222222
+ },
+ "bias_removal_rate": 0.125,
+ "bias_removal_count": 2,
+ "detected_and_removed": 2
+ },
+ "category_metrics": {
+ "occupation": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.75,
+ "f1_score": 0.8571428571428571
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.65,
+ "f1_score": 0.787878787878788
+ },
+ "bias_removal_rate": 0.13333333333333333,
+ "bias_removed_count": 2,
+ "detected_count": 15
+ },
+ "pronoun_assumption": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.14285714285714285,
+ "f1_score": 0.25
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.14285714285714285,
+ "f1_score": 0.25
+ },
+ "bias_removal_rate": 0.0,
+ "bias_removed_count": 0,
+ "detected_count": 1
+ },
+ "pronoun_generic": {
+ "pre_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 0.0,
+ "bias_removed_count": 0,
+ "detected_count": 0
+ }
+ },
+ "correction_quality": {
+ "meaning_preserved": 2,
+ "over_corrections": 0,
+ "successful_corrections": 2
+ }
+ },
+ {
+ "language": "fr",
+ "total_samples": 50,
+ "biased_samples": 35,
+ "overall_metrics": {
+ "pre_correction": {
+ "tp": 16,
+ "fp": 0,
+ "tn": 15,
+ "fn": 19,
+ "precision": 1.0,
+ "recall": 0.45714285714285713,
+ "f1_score": 0.6274509803921569
+ },
+ "post_correction": {
+ "tp": 7,
+ "fp": 0,
+ "tn": 15,
+ "fn": 28,
+ "precision": 1.0,
+ "recall": 0.2,
+ "f1_score": 0.33333333333333337
+ },
+ "bias_removal_rate": 0.5625,
+ "bias_removal_count": 9,
+ "detected_and_removed": 9
+ },
+ "category_metrics": {
+ "occupation": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.30434782608695654,
+ "f1_score": 0.4666666666666667
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.043478260869565216,
+ "f1_score": 0.08333333333333333
+ },
+ "bias_removal_rate": 0.8571428571428571,
+ "bias_removed_count": 6,
+ "detected_count": 7
+ },
+ "pronoun_assumption": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.625,
+ "f1_score": 0.7692307692307693
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.375,
+ "f1_score": 0.5454545454545454
+ },
+ "bias_removal_rate": 0.4,
+ "bias_removed_count": 2,
+ "detected_count": 5
+ },
+ "pronoun_generic": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 1.0,
+ "f1_score": 1.0
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.75,
+ "f1_score": 0.8571428571428571
+ },
+ "bias_removal_rate": 0.25,
+ "bias_removed_count": 1,
+ "detected_count": 4
+ }
+ },
+ "correction_quality": {
+ "meaning_preserved": 12,
+ "over_corrections": 0,
+ "successful_corrections": 9
+ }
+ },
+ {
+ "language": "ki",
+ "total_samples": 33,
+ "biased_samples": 18,
+ "overall_metrics": {
+ "pre_correction": {
+ "tp": 10,
+ "fp": 0,
+ "tn": 15,
+ "fn": 8,
+ "precision": 1.0,
+ "recall": 0.5555555555555556,
+ "f1_score": 0.7142857142857143
+ },
+ "post_correction": {
+ "tp": 3,
+ "fp": 0,
+ "tn": 15,
+ "fn": 15,
+ "precision": 1.0,
+ "recall": 0.16666666666666666,
+ "f1_score": 0.2857142857142857
+ },
+ "bias_removal_rate": 0.7,
+ "bias_removal_count": 7,
+ "detected_and_removed": 7
+ },
+ "category_metrics": {
+ "pronoun_assumption": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 1.0,
+ "f1_score": 1.0
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.2222222222222222,
+ "f1_score": 0.3636363636363636
+ },
+ "bias_removal_rate": 0.7777777777777778,
+ "bias_removed_count": 7,
+ "detected_count": 9
+ },
+ "occupation": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.1111111111111111,
+ "f1_score": 0.19999999999999998
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.1111111111111111,
+ "f1_score": 0.19999999999999998
+ },
+ "bias_removal_rate": 0.0,
+ "bias_removed_count": 0,
+ "detected_count": 1
+ }
+ },
+ "correction_quality": {
+ "meaning_preserved": 9,
+ "over_corrections": 0,
+ "successful_corrections": 7
+ }
+ }
+]
\ No newline at end of file
diff --git a/eval/results/correction_evaluation_en_20251203_151228.json b/eval/results/correction_evaluation_en_20251203_151228.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d79579aa7dbe30c26e07cc8b358675f2cc8f749
--- /dev/null
+++ b/eval/results/correction_evaluation_en_20251203_151228.json
@@ -0,0 +1,1276 @@
+{
+ "language": "en",
+ "total_samples": 66,
+ "biased_samples": 34,
+ "overall_metrics": {
+ "pre_correction": {
+ "tp": 21,
+ "fp": 0,
+ "tn": 32,
+ "fn": 13,
+ "precision": 1.0,
+ "recall": 0.6176470588235294,
+ "f1_score": 0.7636363636363637
+ },
+ "post_correction": {
+ "tp": 0,
+ "fp": 0,
+ "tn": 32,
+ "fn": 34,
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removal_count": 21,
+ "detected_and_removed": 21,
+ "harmonic_score": 0.865979381443299
+ },
+ "semantic_preservation": {
+ "avg_bleu": 0.6162509448223734,
+ "avg_rouge_l": 0.7595795894115221,
+ "avg_token_overlap": 0.7650226757369614,
+ "avg_edit_similarity": 0.7283824640967499,
+ "avg_composite_score": 0.711430188236911,
+ "samples_analyzed": 21
+ },
+ "category_metrics": {
+ "occupation": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.8636363636363636,
+ "f1_score": 0.9268292682926829
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 19,
+ "detected_count": 19,
+ "harmonic_score": 0.9620253164556962,
+ "preservation": {
+ "avg_composite": 0.7025895062969367,
+ "avg_bleu": 0.602610693400167,
+ "samples": 19
+ }
+ },
+ "pronoun_assumption": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.14285714285714285,
+ "f1_score": 0.25
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 1,
+ "detected_count": 1,
+ "harmonic_score": 0.4,
+ "preservation": {
+ "avg_composite": 0.7925000000000001,
+ "avg_bleu": 0.775,
+ "samples": 1
+ }
+ },
+ "pronoun_generic": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.2,
+ "f1_score": 0.33333333333333337
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 1,
+ "detected_count": 1,
+ "harmonic_score": 0.5000000000000001,
+ "preservation": {
+ "avg_composite": 0.7983333333333333,
+ "avg_bleu": 0.7166666666666667,
+ "samples": 1
+ }
+ }
+ },
+ "correction_quality": {
+ "meaning_preserved": 21,
+ "over_corrections": 0,
+ "successful_corrections": 21,
+ "high_quality_corrections": 0
+ },
+ "samples": [
+ {
+ "original": "The chairman will lead the meeting",
+ "corrected": "The chair will lead the meeting",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "chairman",
+ "to": "chair",
+ "severity": "replace"
+ },
+ {
+ "from": "Chairman",
+ "to": "chair",
+ "severity": "replace"
+ },
+ {
+ "from": "the chairman",
+ "to": "chair",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7166666666666667,
+ "rouge_l_score": 0.8333333333333334,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.7916666666666667
+ }
+ },
+ {
+ "original": "She is a great nurse",
+ "corrected": "She is a great nurse",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The policeman arrested the suspect",
+ "corrected": "The police officer arrested the suspect",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "policeman",
+ "to": "police officer",
+ "severity": "replace"
+ },
+ {
+ "from": "Policeman",
+ "to": "police officer",
+ "severity": "replace"
+ },
+ {
+ "from": "the policeman",
+ "to": "police officer",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5333333333333333,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6615151515151516
+ }
+ },
+ {
+ "original": "Ask the waitress for the menu",
+ "corrected": "Ask the server for the menu",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "waitress",
+ "to": "server",
+ "severity": "replace"
+ },
+ {
+ "from": "Waitress",
+ "to": "server",
+ "severity": "replace"
+ },
+ {
+ "from": "the waitress",
+ "to": "server",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7166666666666667,
+ "rouge_l_score": 0.8333333333333334,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.7916666666666667
+ }
+ },
+ {
+ "original": "The businessman closed the deal",
+ "corrected": "The businessperson closed the deal",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "businessman",
+ "to": "businessperson",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.65,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7450000000000001
+ }
+ },
+ {
+ "original": "Every doctor should update his records",
+ "corrected": "Every doctor should update his records",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The fireman saved the cat",
+ "corrected": "The firefighter saved the cat",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "fireman",
+ "to": "firefighter",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.65,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7450000000000001
+ }
+ },
+ {
+ "original": "She works as a secretary",
+ "corrected": "She works as a secretary",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The mailman delivered the package",
+ "corrected": "The mail carrier delivered the package",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mailman",
+ "to": "mail carrier",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5333333333333333,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6615151515151516
+ }
+ },
+ {
+ "original": "The stewardess served drinks",
+ "corrected": "The flight attendant served drinks",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "stewardess",
+ "to": "flight attendant",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.425,
+ "rouge_l_score": 0.6666666666666665,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.6,
+ "composite_score": 0.5974999999999999
+ }
+ },
+ {
+ "original": "He is the best salesman",
+ "corrected": "He is the best salesman",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The cleaning lady comes on Fridays",
+ "corrected": "The cleaner comes on Fridays",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "cleaning lady",
+ "to": "cleaner",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.65,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.6666666666666666,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6798484848484849
+ }
+ },
+ {
+ "original": "Ask your congressman about the bill",
+ "corrected": "Ask your representative about the bill",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "congressman",
+ "to": "representative",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7166666666666667,
+ "rouge_l_score": 0.8333333333333334,
+ "token_overlap": 0.8333333333333334,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.7983333333333333
+ }
+ },
+ {
+ "original": "The weatherman predicted rain",
+ "corrected": "The meteorologist predicted rain",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "weatherman",
+ "to": "meteorologist",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5416666666666666,
+ "rouge_l_score": 0.75,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.75,
+ "composite_score": 0.6875
+ }
+ },
+ {
+ "original": "She is just a housewife",
+ "corrected": "She is just a housewife",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The repairman fixed the sink",
+ "corrected": "The repair technician fixed the sink",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "repairman",
+ "to": "repair technician",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5333333333333333,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6615151515151516
+ }
+ },
+ {
+ "original": "Every nurse knows her patients",
+ "corrected": "Every nurse knows her patients",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The doorman checked IDs",
+ "corrected": "The door attendant checked IDs",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "doorman",
+ "to": "door attendant",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.425,
+ "rouge_l_score": 0.6666666666666665,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.6,
+ "composite_score": 0.5974999999999999
+ }
+ },
+ {
+ "original": "She works as a receptionist",
+ "corrected": "She works as a receptionist",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The garbage man comes early",
+ "corrected": "The sanitation worker comes early",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "garbage man",
+ "to": "sanitation worker",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.425,
+ "rouge_l_score": 0.6,
+ "token_overlap": 0.6,
+ "edit_similarity": 0.6,
+ "composite_score": 0.5475
+ }
+ },
+ {
+ "original": "The anchorman read the news",
+ "corrected": "The news anchor read the news",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "anchorman",
+ "to": "news anchor",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7166666666666667,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.7165151515151515
+ }
+ },
+ {
+ "original": "Every teacher loves her students",
+ "corrected": "Every teacher loves her students",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The deliveryman was late",
+ "corrected": "The delivery driver was late",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "deliveryman",
+ "to": "delivery driver",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.425,
+ "rouge_l_score": 0.6666666666666665,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.6,
+ "composite_score": 0.5974999999999999
+ }
+ },
+ {
+ "original": "She is a talented seamstress",
+ "corrected": "She is a talented tailor",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "seamstress",
+ "to": "tailor",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.775,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7925000000000001
+ }
+ },
+ {
+ "original": "The handyman repaired the door",
+ "corrected": "The maintenance worker repaired the door",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "handyman",
+ "to": "maintenance worker",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5333333333333333,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6615151515151516
+ }
+ },
+ {
+ "original": "We need a strong policeman for this job",
+ "corrected": "We need a strong police officer for this job",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "policeman",
+ "to": "police officer",
+ "severity": "replace"
+ },
+ {
+ "from": "Policeman",
+ "to": "police officer",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7013888888888888,
+ "rouge_l_score": 0.823529411764706,
+ "token_overlap": 0.875,
+ "edit_similarity": 0.7777777777777778,
+ "composite_score": 0.788031045751634
+ }
+ },
+ {
+ "original": "The saleslady showed us the new products",
+ "corrected": "The saleslady showed us the new products",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Each businessman should review his portfolio",
+ "corrected": "Each businessperson should review his portfolio",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "businessman",
+ "to": "businessperson",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7166666666666667,
+ "rouge_l_score": 0.8333333333333334,
+ "token_overlap": 0.8333333333333334,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.7983333333333333
+ }
+ },
+ {
+ "original": "He's surprisingly good at nursing patients",
+ "corrected": "He's surprisingly good at nursing patients",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The new weathergirl is very professional",
+ "corrected": "The new weathergirl is very professional",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Every employee must submit his timesheet by Friday",
+ "corrected": "Every employee must submit his timesheet by Friday",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "She's very ambitious for a teacher",
+ "corrected": "She's very ambitious for a teacher",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Ask the cleaning lady to do the conference room",
+ "corrected": "Ask the cleaner to do the conference room",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "cleaning lady",
+ "to": "cleaner",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7946428571428572,
+ "rouge_l_score": 0.823529411764706,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.7777777777777778,
+ "composite_score": 0.7910072362278245
+ }
+ },
+ {
+ "original": "A good fireman must be physically strong",
+ "corrected": "A good firefighter must be physically strong",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "fireman",
+ "to": "firefighter",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7619047619047619,
+ "rouge_l_score": 0.8571428571428571,
+ "token_overlap": 0.8571428571428571,
+ "edit_similarity": 0.8571428571428572,
+ "composite_score": 0.8285714285714285
+ }
+ },
+ {
+ "original": "The table is wooden",
+ "corrected": "The table is wooden",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The meeting starts at 3pm",
+ "corrected": "The meeting starts at 3pm",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Please close the window",
+ "corrected": "Please close the window",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The doctor examined the patient carefully",
+ "corrected": "The doctor examined the patient carefully",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Our teacher explained the concept well",
+ "corrected": "Our teacher explained the concept well",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The engineer designed a new bridge",
+ "corrected": "The engineer designed a new bridge",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The nurse provided excellent care",
+ "corrected": "The nurse provided excellent care",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "A pilot flew the aircraft safely",
+ "corrected": "A pilot flew the aircraft safely",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The lawyer presented strong arguments",
+ "corrected": "The lawyer presented strong arguments",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Scientists discovered a new species",
+ "corrected": "Scientists discovered a new species",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The report is due tomorrow",
+ "corrected": "The report is due tomorrow",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Coffee tastes good",
+ "corrected": "Coffee tastes good",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The car needs gas",
+ "corrected": "The car needs gas",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "It is raining outside",
+ "corrected": "It is raining outside",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The book is interesting",
+ "corrected": "The book is interesting",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Turn left at the corner",
+ "corrected": "Turn left at the corner",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The phone is ringing",
+ "corrected": "The phone is ringing",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Water boils at 100 degrees",
+ "corrected": "Water boils at 100 degrees",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The train arrives at noon",
+ "corrected": "The train arrives at noon",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Please send the email",
+ "corrected": "Please send the email",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The computer is slow",
+ "corrected": "The computer is slow",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The door is locked",
+ "corrected": "The door is locked",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Time flies quickly",
+ "corrected": "Time flies quickly",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The sun is bright",
+ "corrected": "The sun is bright",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Music sounds beautiful",
+ "corrected": "Music sounds beautiful",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The project is complete",
+ "corrected": "The project is complete",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Food smells delicious",
+ "corrected": "Food smells delicious",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The road is bumpy",
+ "corrected": "The road is bumpy",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Plants need water",
+ "corrected": "Plants need water",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The sky is blue",
+ "corrected": "The sky is blue",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Numbers don't lie",
+ "corrected": "Numbers don't lie",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "The clock shows 5pm",
+ "corrected": "The clock shows 5pm",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/eval/results/correction_evaluation_fr_20251203_151228.json b/eval/results/correction_evaluation_fr_20251203_151228.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c4f38ef0f74696a3158939b886eb9a3b626e9f5
--- /dev/null
+++ b/eval/results/correction_evaluation_fr_20251203_151228.json
@@ -0,0 +1,1078 @@
+{
+ "language": "fr",
+ "total_samples": 50,
+ "biased_samples": 35,
+ "overall_metrics": {
+ "pre_correction": {
+ "tp": 14,
+ "fp": 0,
+ "tn": 15,
+ "fn": 21,
+ "precision": 1.0,
+ "recall": 0.4,
+ "f1_score": 0.5714285714285715
+ },
+ "post_correction": {
+ "tp": 5,
+ "fp": 0,
+ "tn": 15,
+ "fn": 30,
+ "precision": 1.0,
+ "recall": 0.14285714285714285,
+ "f1_score": 0.25
+ },
+ "bias_removal_rate": 0.6428571428571429,
+ "bias_removal_count": 9,
+ "detected_and_removed": 9,
+ "harmonic_score": 0.6050420168067228
+ },
+ "semantic_preservation": {
+ "avg_bleu": 0.5950892857142857,
+ "avg_rouge_l": 0.7341991341991342,
+ "avg_token_overlap": 0.8241071428571428,
+ "avg_edit_similarity": 0.6675595238095239,
+ "avg_composite_score": 0.6971198593073593,
+ "samples_analyzed": 12
+ },
+ "category_metrics": {
+ "occupation": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.30434782608695654,
+ "f1_score": 0.4666666666666667
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.043478260869565216,
+ "f1_score": 0.08333333333333333
+ },
+ "bias_removal_rate": 0.8571428571428571,
+ "bias_removed_count": 6,
+ "detected_count": 7,
+ "harmonic_score": 0.60431654676259,
+ "preservation": {
+ "avg_composite": 0.6438041125541126,
+ "avg_bleu": 0.555952380952381,
+ "samples": 6
+ }
+ },
+ "pronoun_assumption": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.5,
+ "f1_score": 0.6666666666666666
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.25,
+ "f1_score": 0.4
+ },
+ "bias_removal_rate": 0.5,
+ "bias_removed_count": 2,
+ "detected_count": 4,
+ "harmonic_score": 0.5714285714285714,
+ "preservation": {
+ "avg_composite": 0.7564353354978355,
+ "avg_bleu": 0.652827380952381,
+ "samples": 4
+ }
+ },
+ "pronoun_generic": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.75,
+ "f1_score": 0.8571428571428571
+ },
+ "post_correction": {
+ "precision": 1.0,
+ "recall": 0.5,
+ "f1_score": 0.6666666666666666
+ },
+ "bias_removal_rate": 0.3333333333333333,
+ "bias_removed_count": 1,
+ "detected_count": 3,
+ "harmonic_score": 0.48,
+ "preservation": {
+ "avg_composite": 0.7384361471861471,
+ "avg_bleu": 0.5970238095238095,
+ "samples": 2
+ }
+ }
+ },
+ "correction_quality": {
+ "meaning_preserved": 12,
+ "over_corrections": 0,
+ "successful_corrections": 9,
+ "high_quality_corrections": 1
+ },
+ "samples": [
+ {
+ "original": "Le président dirigera la réunion",
+ "corrected": "La présidence·e·e dirigera la réunion",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "président",
+ "to": "président·e",
+ "severity": "replace"
+ },
+ {
+ "from": "Président",
+ "to": "Président·e",
+ "severity": "replace"
+ },
+ {
+ "from": "le président",
+ "to": "la présidence",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.45238095238095233,
+ "rouge_l_score": 0.5,
+ "token_overlap": 0.6,
+ "edit_similarity": 0.4285714285714286,
+ "composite_score": 0.49142857142857144
+ }
+ },
+ {
+ "original": "Elle est une excellente infirmière",
+ "corrected": "Elle est une excellente personnel infirmier",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "infirmière",
+ "to": "personnel infirmier",
+ "severity": "replace"
+ },
+ {
+ "from": "Infirmière",
+ "to": "Personnel infirmier",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.6333333333333333,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.7015151515151514
+ }
+ },
+ {
+ "original": "Le policier a arrêté le suspect",
+ "corrected": "Le officier de police a arrêté le suspect",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "policier",
+ "to": "officier de police",
+ "severity": "replace"
+ },
+ {
+ "from": "Policier",
+ "to": "Officier de police",
+ "severity": "replace"
+ },
+ {
+ "from": "le policier",
+ "to": "l'officier de police",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5267857142857143,
+ "rouge_l_score": 0.7142857142857143,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.625,
+ "composite_score": 0.6573214285714286
+ }
+ },
+ {
+ "original": "Demandez au serveur le menu",
+ "corrected": "Demandez au serveur le menu",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "L'homme d'affaires a conclu l'accord",
+ "corrected": "L'personne d'affaires a conclu l'accord",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "homme",
+ "to": "personne",
+ "severity": "replace"
+ },
+ {
+ "from": "Homme",
+ "to": "Personne",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7946428571428572,
+ "rouge_l_score": 0.875,
+ "token_overlap": 0.8571428571428571,
+ "edit_similarity": 0.875,
+ "composite_score": 0.8473214285714286
+ }
+ },
+ {
+ "original": "Chaque médecin doit mettre à jour ses dossiers",
+ "corrected": "Chaque médecin doit mettre à jour ses dossiers",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le pompier a sauvé le chat",
+ "corrected": "Le pompier a sauvé le chat",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Elle travaille comme secrétaire",
+ "corrected": "Elle travaille comme secrétaire",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le facteur a livré le colis",
+ "corrected": "Le facteur a livré le colis",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "L'hôtesse a servi les boissons",
+ "corrected": "L'hôtesse a servi les boissons",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Il est le meilleur vendeur",
+ "corrected": "Il est le meilleur vendeur",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "La femme de ménage vient le vendredi",
+ "corrected": "La personne de ménage vient le vendredi",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "femme",
+ "to": "personne",
+ "severity": "replace"
+ },
+ {
+ "from": "Femme",
+ "to": "Personne",
+ "severity": "replace"
+ },
+ {
+ "from": "la femme",
+ "to": "la personne",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7619047619047619,
+ "rouge_l_score": 0.8571428571428571,
+ "token_overlap": 0.8571428571428571,
+ "edit_similarity": 0.8571428571428572,
+ "composite_score": 0.8285714285714285
+ }
+ },
+ {
+ "original": "Demandez à votre député au sujet du projet de loi",
+ "corrected": "Demandez à votre député au sujet du projet de loi",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le météorologue a prédit la pluie",
+ "corrected": "Le météorologue a prédit la pluie",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Elle n'est qu'une femme au foyer",
+ "corrected": "Elle n'est qu'une personne au foyer",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "femme",
+ "to": "personne",
+ "severity": "replace"
+ },
+ {
+ "from": "Femme",
+ "to": "Personne",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.7946428571428572,
+ "rouge_l_score": 0.875,
+ "token_overlap": 0.875,
+ "edit_similarity": 0.875,
+ "composite_score": 0.8508928571428572
+ }
+ },
+ {
+ "original": "Le réparateur a réparé l'évier",
+ "corrected": "Le réparateur a réparé l'évier",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Chaque infirmière connaît ses patients",
+ "corrected": "Chaque personnel infirmier connaît ses patients",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "infirmière",
+ "to": "personnel infirmier",
+ "severity": "replace"
+ },
+ {
+ "from": "Infirmière",
+ "to": "Personnel infirmier",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5333333333333333,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6715151515151516
+ }
+ },
+ {
+ "original": "Le portier a vérifié les cartes d'identité",
+ "corrected": "Le portier a vérifié les cartes d'identité",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Elle travaille comme réceptionniste",
+ "corrected": "Elle travaille comme réceptionniste",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le patron a pris la décision",
+ "corrected": "Le patron a pris la décision",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Chaque enseignant doit préparer ses cours",
+ "corrected": "Chaque enseignant·e·e doit préparer ses cours",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": true,
+ "post_detected": true,
+ "bias_removed": false,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "enseignant",
+ "to": "enseignant·e",
+ "severity": "replace"
+ },
+ {
+ "from": "Enseignant",
+ "to": "Enseignant·e",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [
+ {
+ "from": "enseignant",
+ "to": "enseignant·e",
+ "severity": "replace"
+ },
+ {
+ "from": "Enseignant",
+ "to": "Enseignant·e",
+ "severity": "replace"
+ }
+ ],
+ "preservation_metrics": {
+ "bleu_score": 0.6607142857142857,
+ "rouge_l_score": 0.8571428571428571,
+ "token_overlap": 1.0,
+ "edit_similarity": 0.75,
+ "composite_score": 0.8053571428571428
+ }
+ },
+ {
+ "original": "Le directeur général présidera",
+ "corrected": "La direction·rice·rice général présidera",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "directeur",
+ "to": "directeur·rice",
+ "severity": "replace"
+ },
+ {
+ "from": "Directeur",
+ "to": "Directeur·rice",
+ "severity": "replace"
+ },
+ {
+ "from": "le directeur",
+ "to": "la direction",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.26666666666666666,
+ "rouge_l_score": 0.4,
+ "token_overlap": 0.5,
+ "edit_similarity": 0.33333333333333337,
+ "composite_score": 0.3666666666666667
+ }
+ },
+ {
+ "original": "Elle est une bonne cuisinière",
+ "corrected": "Elle est une bonne cuisinière",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le gardien de nuit fait sa ronde",
+ "corrected": "Le gardien de nuit fait sa ronde",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": true,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [
+ {
+ "from": "sa",
+ "to": "leur",
+ "severity": "warn"
+ },
+ {
+ "from": "Sa",
+ "to": "Leur",
+ "severity": "warn"
+ }
+ ],
+ "post_edits": [
+ {
+ "from": "sa",
+ "to": "leur",
+ "severity": "warn"
+ },
+ {
+ "from": "Sa",
+ "to": "Leur",
+ "severity": "warn"
+ }
+ ]
+ },
+ {
+ "original": "Demandez au technicien de l'aide",
+ "corrected": "Demandez au technicien de l'aide",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le serveur a pris notre commande",
+ "corrected": "Le serveur a pris notre commande",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Elle veut devenir actrice",
+ "corrected": "Elle veut devenir actrice",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Chaque étudiant doit apporter son manuel",
+ "corrected": "Chaque étudiant doit apporter son manuel",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": true,
+ "post_detected": true,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [
+ {
+ "from": "son",
+ "to": "leur",
+ "severity": "warn"
+ },
+ {
+ "from": "Son",
+ "to": "Leur",
+ "severity": "warn"
+ }
+ ],
+ "post_edits": [
+ {
+ "from": "son",
+ "to": "leur",
+ "severity": "warn"
+ },
+ {
+ "from": "Son",
+ "to": "Leur",
+ "severity": "warn"
+ }
+ ]
+ },
+ {
+ "original": "Le mécanicien a réparé la voiture",
+ "corrected": "Le mécanicien a réparé la voiture",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "La serveuse était très gentille",
+ "corrected": "La serveur·euse était très gentille",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "serveuse",
+ "to": "serveur·euse",
+ "severity": "replace"
+ },
+ {
+ "from": "Serveuse",
+ "to": "Serveur·euse",
+ "severity": "replace"
+ },
+ {
+ "from": "la serveuse",
+ "to": "le personnel",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5333333333333333,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6715151515151516
+ }
+ },
+ {
+ "original": "Il travaille comme ingénieur",
+ "corrected": "Il travaille comme ingénieur·e·e",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": true,
+ "bias_removed": false,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "ingénieur",
+ "to": "ingénieur·e",
+ "severity": "replace"
+ },
+ {
+ "from": "Ingénieur",
+ "to": "Ingénieur·e",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [
+ {
+ "from": "ingénieur",
+ "to": "ingénieur·e",
+ "severity": "replace"
+ },
+ {
+ "from": "Ingénieur",
+ "to": "Ingénieur·e",
+ "severity": "replace"
+ }
+ ],
+ "preservation_metrics": {
+ "bleu_score": 0.6333333333333333,
+ "rouge_l_score": 0.8,
+ "token_overlap": 1.0,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.7633333333333332
+ }
+ },
+ {
+ "original": "Le conducteur a arrêté le bus",
+ "corrected": "Le conducteur a arrêté le bus",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Elle est avocat",
+ "corrected": "Elle est avocat·e·e",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": true,
+ "bias_removed": false,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "avocat",
+ "to": "avocat·e",
+ "severity": "replace"
+ },
+ {
+ "from": "Avocat",
+ "to": "Avocat·e",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [
+ {
+ "from": "avocat",
+ "to": "avocat·e",
+ "severity": "replace"
+ },
+ {
+ "from": "Avocat",
+ "to": "Avocat·e",
+ "severity": "replace"
+ }
+ ],
+ "preservation_metrics": {
+ "bleu_score": 0.55,
+ "rouge_l_score": 0.7499999999999999,
+ "token_overlap": 1.0,
+ "edit_similarity": 0.6,
+ "composite_score": 0.71
+ }
+ },
+ {
+ "original": "Le boucher a coupé la viande",
+ "corrected": "Le boucher a coupé la viande",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Demandez au bibliothécaire",
+ "corrected": "Demandez au bibliothécaire",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Cette personne gère l'équipe efficacement",
+ "corrected": "Cette personne gère l'équipe efficacement",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le personnel travaille dur",
+ "corrected": "Le personnel travaille dur",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "L'équipe a terminé le projet",
+ "corrected": "L'équipe a terminé le projet",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Chacun doit faire leur part",
+ "corrected": "Chacun doit faire leur part",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le groupe a voté",
+ "corrected": "Le groupe a voté",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Les gens attendent dehors",
+ "corrected": "Les gens attendent dehors",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "La communauté s'est réunie",
+ "corrected": "La communauté s'est réunie",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le comité a décidé",
+ "corrected": "Le comité a décidé",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "L'organisation a annoncé",
+ "corrected": "L'organisation a annoncé",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le département a approuvé",
+ "corrected": "Le département a approuvé",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Cette personne est qualifiée",
+ "corrected": "Cette personne est qualifiée",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "L'individu a réussi",
+ "corrected": "L'individu a réussi",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le candidat a gagné",
+ "corrected": "Le candidat a gagné",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Le participant a terminé",
+ "corrected": "Le participant a terminé",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "L'employé a travaillé",
+ "corrected": "L'employé a travaillé",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/eval/results/correction_evaluation_ki_20251203_151228.json b/eval/results/correction_evaluation_ki_20251203_151228.json
new file mode 100644
index 0000000000000000000000000000000000000000..0bf85924388a0e6cc677de1e59974492cd3703c0
--- /dev/null
+++ b/eval/results/correction_evaluation_ki_20251203_151228.json
@@ -0,0 +1,716 @@
+{
+ "language": "ki",
+ "total_samples": 33,
+ "biased_samples": 18,
+ "overall_metrics": {
+ "pre_correction": {
+ "tp": 9,
+ "fp": 0,
+ "tn": 15,
+ "fn": 9,
+ "precision": 1.0,
+ "recall": 0.5,
+ "f1_score": 0.6666666666666666
+ },
+ "post_correction": {
+ "tp": 0,
+ "fp": 0,
+ "tn": 15,
+ "fn": 18,
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removal_count": 9,
+ "detected_and_removed": 9,
+ "harmonic_score": 0.8
+ },
+ "semantic_preservation": {
+ "avg_bleu": 0.8537037037037037,
+ "avg_rouge_l": 0.8234086900753569,
+ "avg_token_overlap": 0.7833333333333334,
+ "avg_edit_similarity": 0.7833333333333334,
+ "avg_composite_score": 0.8164670514670516,
+ "samples_analyzed": 9
+ },
+ "category_metrics": {
+ "pronoun_assumption": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 1.0,
+ "f1_score": 1.0
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 9,
+ "detected_count": 9,
+ "harmonic_score": 1.0,
+ "preservation": {
+ "avg_composite": 0.8164670514670516,
+ "avg_bleu": 0.8537037037037037,
+ "samples": 9
+ }
+ },
+ "occupation": {
+ "pre_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 0.0,
+ "bias_removed_count": 0,
+ "detected_count": 0,
+ "harmonic_score": 0.0,
+ "preservation": {}
+ }
+ },
+ "correction_quality": {
+ "meaning_preserved": 9,
+ "over_corrections": 0,
+ "successful_corrections": 9,
+ "high_quality_corrections": 4
+ },
+ "samples": [
+ {
+ "original": "Mũrũgamĩrĩri ũcio nĩ mũndũ mũrũme",
+ "corrected": "Mũrũgamĩrĩri ũcio nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũndũ mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũndũ mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.888888888888889,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.8866666666666667
+ }
+ },
+ {
+ "original": "Mũrutani ũcio nĩ mũndũ mwega",
+ "corrected": "Mũrutani ũcio nĩ mũndũ mwega",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũthondeki wa nyũmba ũũkĩte",
+ "corrected": "Mũthondeki wa nyũmba ũũkĩte",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Daktari ũcio nĩ mũndũ mũũgĩ",
+ "corrected": "Daktari ũcio nĩ mũndũ mũũgĩ",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũrutani wa ciana nĩ mũndũ mũtheru",
+ "corrected": "Mũrutani wa ciana nĩ mũndũ mũtheru",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũthondeki nĩ mũrũme",
+ "corrected": "Mũthondeki nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.5833333333333333,
+ "rouge_l_score": 0.6666666666666666,
+ "token_overlap": 0.6666666666666666,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6416666666666666
+ }
+ },
+ {
+ "original": "Mũrũthia nĩ mũndũ mũrũme",
+ "corrected": "Mũrũthia nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũndũ mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũndũ mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.8571428571428571,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.75,
+ "composite_score": 0.8571428571428572
+ }
+ },
+ {
+ "original": "Mũruti wa thiomi nĩ mũndũ mwega",
+ "corrected": "Mũruti wa thiomi nĩ mũndũ mwega",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũroria wa mũtũrĩre nĩ mũrũme",
+ "corrected": "Mũroria wa mũtũrĩre nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.775,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7925000000000001
+ }
+ },
+ {
+ "original": "Mũnene wa kũũ nĩ mũndũ mũrũme",
+ "corrected": "Mũnene wa kũũ nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũndũ mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũndũ mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.9090909090909091,
+ "token_overlap": 0.8333333333333334,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.9060606060606062
+ }
+ },
+ {
+ "original": "Mũrutani ũcio ũrutaga wega",
+ "corrected": "Mũrutani ũcio ũrutaga wega",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũthondeki wa matũ nĩ mũrũme",
+ "corrected": "Mũthondeki wa matũ nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.775,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7925000000000001
+ }
+ },
+ {
+ "original": "Mũndũ ũcio nĩ mũruti mwega",
+ "corrected": "Mũndũ ũcio nĩ mũruti mwega",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Karani ũcio nĩ mũndũ mũrũme",
+ "corrected": "Karani ũcio nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũndũ mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũndũ mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.888888888888889,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.8866666666666667
+ }
+ },
+ {
+ "original": "Mũrũgamĩrĩri wa thirikari nĩ mũrũme",
+ "corrected": "Mũrũgamĩrĩri wa thirikari nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.775,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7925000000000001
+ }
+ },
+ {
+ "original": "Mũndũ ũcio nĩ mũrutani mwega",
+ "corrected": "Mũndũ ũcio nĩ mũrutani mwega",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũthondeki wa nyũmba nĩ mũrũme",
+ "corrected": "Mũthondeki wa nyũmba nĩ mũndũ",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "mũrũme",
+ "to": "mũndũ",
+ "severity": "replace"
+ },
+ {
+ "from": "Mũrũme",
+ "to": "Mũndũ",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.775,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7925000000000001
+ }
+ },
+ {
+ "original": "Mũrũgamĩrĩri nĩ mũndũ mwega",
+ "corrected": "Mũrũgamĩrĩri nĩ mũndũ mwega",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũndũ ũcio arutaga wega",
+ "corrected": "Mũndũ ũcio arutaga wega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Andũ acio nĩ arutani ega",
+ "corrected": "Andũ acio nĩ arutani ega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Gĩkundi kĩu kĩarutire wega",
+ "corrected": "Gĩkundi kĩu kĩarutire wega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũndũ nĩ mwega",
+ "corrected": "Mũndũ nĩ mwega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Andũ nĩ ega",
+ "corrected": "Andũ nĩ ega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Kĩrĩndĩ kĩu kĩrutaga wega",
+ "corrected": "Kĩrĩndĩ kĩu kĩrutaga wega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũndũ ũcio nĩ mũthondeki mwega",
+ "corrected": "Mũndũ ũcio nĩ mũthondeki mwega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Andũacio marutaga wega",
+ "corrected": "Andũacio marutaga wega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũndũ ũcio nĩ mũruti",
+ "corrected": "Mũndũ ũcio nĩ mũruti",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Gĩkundi kĩu kĩarutire wega mũno",
+ "corrected": "Gĩkundi kĩu kĩarutire wega mũno",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Andũ nĩ arutani ega",
+ "corrected": "Andũ nĩ arutani ega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũndũ ũcio nĩ mũthondeki",
+ "corrected": "Mũndũ ũcio nĩ mũthondeki",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Kĩrĩndĩ kĩu kĩrutaga",
+ "corrected": "Kĩrĩndĩ kĩu kĩrutaga",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mũndũ nĩ mũruti mwega",
+ "corrected": "Mũndũ nĩ mũruti mwega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Andũ acio nĩ athondeki ega",
+ "corrected": "Andũ acio nĩ athondeki ega",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/eval/results/correction_evaluation_sw_20251203_151228.json b/eval/results/correction_evaluation_sw_20251203_151228.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d2adf9f278f4b55bfea262ca4c7fa541b5a2d55
--- /dev/null
+++ b/eval/results/correction_evaluation_sw_20251203_151228.json
@@ -0,0 +1,1182 @@
+{
+ "language": "sw",
+ "total_samples": 63,
+ "biased_samples": 31,
+ "overall_metrics": {
+ "pre_correction": {
+ "tp": 16,
+ "fp": 0,
+ "tn": 32,
+ "fn": 15,
+ "precision": 1.0,
+ "recall": 0.5161290322580645,
+ "f1_score": 0.6808510638297872
+ },
+ "post_correction": {
+ "tp": 0,
+ "fp": 0,
+ "tn": 32,
+ "fn": 31,
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removal_count": 16,
+ "detected_and_removed": 16,
+ "harmonic_score": 0.810126582278481
+ },
+ "semantic_preservation": {
+ "avg_bleu": 0.8303819444444445,
+ "avg_rouge_l": 0.8086940836940837,
+ "avg_token_overlap": 0.7619791666666667,
+ "avg_edit_similarity": 0.734375,
+ "avg_composite_score": 0.7909936417748918,
+ "samples_analyzed": 16
+ },
+ "category_metrics": {
+ "occupation": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 0.25,
+ "f1_score": 0.4
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 5,
+ "detected_count": 5,
+ "harmonic_score": 0.5714285714285714,
+ "preservation": {
+ "avg_composite": 0.6869285714285714,
+ "avg_bleu": 0.6905555555555556,
+ "samples": 5
+ }
+ },
+ "pronoun_assumption": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 1.0,
+ "f1_score": 1.0
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 7,
+ "detected_count": 7,
+ "harmonic_score": 1.0,
+ "preservation": {
+ "avg_composite": 0.8542269635126779,
+ "avg_bleu": 0.9500000000000001,
+ "samples": 7
+ }
+ },
+ "pronoun_generic": {
+ "pre_correction": {
+ "precision": 1.0,
+ "recall": 1.0,
+ "f1_score": 1.0
+ },
+ "post_correction": {
+ "precision": 0.0,
+ "recall": 0.0,
+ "f1_score": 0.0
+ },
+ "bias_removal_rate": 1.0,
+ "bias_removed_count": 4,
+ "detected_count": 4,
+ "harmonic_score": 1.0,
+ "preservation": {
+ "avg_composite": 0.8104166666666668,
+ "avg_bleu": 0.7958333333333334,
+ "samples": 4
+ }
+ }
+ },
+ "correction_quality": {
+ "meaning_preserved": 16,
+ "over_corrections": 0,
+ "successful_corrections": 16,
+ "high_quality_corrections": 7
+ },
+ "samples": [
+ {
+ "original": "Mwalimu mkuu ataongoza mkutano",
+ "corrected": "Mwalimu mkuu ataongoza mkutano",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Yeye ni muuguzi mzuri",
+ "corrected": "Ni muuguzi mzuri",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "Yeye ni",
+ "to": "Ni",
+ "severity": "replace"
+ },
+ {
+ "from": "yeye ni",
+ "to": "ni",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.8571428571428571,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.75,
+ "composite_score": 0.8571428571428572
+ }
+ },
+ {
+ "original": "Askari alimkamata mshtakiwa",
+ "corrected": "Afisa wa usalama alimkamata mshtakiwa",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "askari",
+ "to": "afisa wa usalama",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.325,
+ "rouge_l_score": 0.5,
+ "token_overlap": 0.6666666666666666,
+ "edit_similarity": 0.4,
+ "composite_score": 0.4608333333333334
+ }
+ },
+ {
+ "original": "Muuliza mhudumu chakula",
+ "corrected": "Muuliza mhudumu chakula",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mfanyabiashara alifunga mikataba",
+ "corrected": "Mfanyabiashara alifunga mikataba",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Kila daktari anapaswa kusasisha rekodi zake",
+ "corrected": "Kila daktari anapaswa kusasisha rekodi zao",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "zake",
+ "to": "zao",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.8166666666666667,
+ "rouge_l_score": 0.8333333333333334,
+ "token_overlap": 0.8333333333333334,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.8283333333333334
+ }
+ },
+ {
+ "original": "Mzimamoto aliokoa paka",
+ "corrected": "Mzimamoto aliokoa paka",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Yeye anafanya kazi ya uandishi",
+ "corrected": "Anafanya kazi ya uandishi",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "Yeye anafanya",
+ "to": "Anafanya",
+ "severity": "replace"
+ },
+ {
+ "from": "yeye anafanya",
+ "to": "anafanya",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.888888888888889,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.8866666666666667
+ }
+ },
+ {
+ "original": "Mpeleka barua alipeleka kifurushi",
+ "corrected": "Mpeleka barua alipeleka kifurushi",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mhudumu wa ndege alihudumu vinywaji",
+ "corrected": "Mhudumu wa ndege alihudumu vinywaji",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Yeye ni muuzaji bora",
+ "corrected": "Ni muuzaji bora",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "Yeye ni",
+ "to": "Ni",
+ "severity": "replace"
+ },
+ {
+ "from": "yeye ni",
+ "to": "ni",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.8571428571428571,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.75,
+ "composite_score": 0.8571428571428572
+ }
+ },
+ {
+ "original": "Mama msafishaji anakuja Ijumaa",
+ "corrected": "Mama msafishaji anakuja Ijumaa",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Muulize mbunge kuhusu mswada",
+ "corrected": "Muulize mbunge kuhusu mswada",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mtabiri wa hali ya hewa alitabiri mvua",
+ "corrected": "Mtabiri wa hali ya hewa alitabiri mvua",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Yeye ni mama wa nyumbani tu",
+ "corrected": "Ni mtu wa nyumbani tu",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "Yeye ni",
+ "to": "Ni",
+ "severity": "replace"
+ },
+ {
+ "from": "yeye ni",
+ "to": "ni",
+ "severity": "replace"
+ },
+ {
+ "from": "mama wa nyumbani",
+ "to": "mtu wa nyumbani",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.65,
+ "rouge_l_score": 0.7272727272727272,
+ "token_overlap": 0.6666666666666666,
+ "edit_similarity": 0.6666666666666667,
+ "composite_score": 0.6798484848484849
+ }
+ },
+ {
+ "original": "Fundi alirekebishe bomba",
+ "corrected": "Fundi alirekebishe bomba",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Kila muuguzi anajua wagonjwa wake",
+ "corrected": "Kila muuguzi anajua wagonjwa wao",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "wake",
+ "to": "wao",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.775,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7925000000000001
+ }
+ },
+ {
+ "original": "Mlezi wa mlango alikagua vitambulisho",
+ "corrected": "Mlezi wa mlango alikagua vitambulisho",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Yeye anafanya kazi ya upokeaji",
+ "corrected": "Anafanya kazi ya upokeaji",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "Yeye anafanya",
+ "to": "Anafanya",
+ "severity": "replace"
+ },
+ {
+ "from": "yeye anafanya",
+ "to": "anafanya",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.888888888888889,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.8866666666666667
+ }
+ },
+ {
+ "original": "Mchuuzi wa taka alikuja mapema",
+ "corrected": "Mchuuzi wa taka alikuja mapema",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mwandishi wa habari alisoma habari",
+ "corrected": "Mwandishi wa habari alisoma habari",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Kila mwalimu anapenda wanafunzi wake",
+ "corrected": "Kila mwalimu anapenda wanafunzi wao",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "wake",
+ "to": "wao",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.775,
+ "rouge_l_score": 0.8000000000000002,
+ "token_overlap": 0.8,
+ "edit_similarity": 0.8,
+ "composite_score": 0.7925000000000001
+ }
+ },
+ {
+ "original": "Mpeleka mizigo alichelewa",
+ "corrected": "Mpeleka mizigo alichelewa",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Yeye ni mshonaji hodari",
+ "corrected": "Ni mshonaji hodari",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "Yeye ni",
+ "to": "Ni",
+ "severity": "replace"
+ },
+ {
+ "from": "yeye ni",
+ "to": "ni",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.8571428571428571,
+ "token_overlap": 0.75,
+ "edit_similarity": 0.75,
+ "composite_score": 0.8571428571428572
+ }
+ },
+ {
+ "original": "Fundi wa nyumba alirekebishe mlango",
+ "corrected": "Fundi wa nyumba alirekebishe mlango",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Tunah itaji askari mwenye nguvu kwa kazi hii",
+ "corrected": "Tunah itaji afisa wa usalama mwenye nguvu kwa kazi hii",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "askari",
+ "to": "afisa wa usalama",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.6277777777777778,
+ "rouge_l_score": 0.7777777777777777,
+ "token_overlap": 0.875,
+ "edit_similarity": 0.7,
+ "composite_score": 0.7366666666666667
+ }
+ },
+ {
+ "original": "Kila mfanyakazi anapaswa kuwasilisha kadi yake",
+ "corrected": "Kila mfanyakazi anapaswa kuwasilisha kadi yao",
+ "is_biased": true,
+ "category": "pronoun_generic",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "yake",
+ "to": "yao",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.8166666666666667,
+ "rouge_l_score": 0.8333333333333334,
+ "token_overlap": 0.8333333333333334,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.8283333333333334
+ }
+ },
+ {
+ "original": "Yeye ni mama mzuri wa nyumbani",
+ "corrected": "Ni mama mzuri wa nyumbani",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "Yeye ni",
+ "to": "Ni",
+ "severity": "replace"
+ },
+ {
+ "from": "yeye ni",
+ "to": "ni",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.9090909090909091,
+ "token_overlap": 0.8333333333333334,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.9060606060606062
+ }
+ },
+ {
+ "original": "Mwalimu wa kike alifundisha vizuri",
+ "corrected": "Mwalimu alifundisha vizuri",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "wa kike",
+ "to": "",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.75,
+ "rouge_l_score": 0.7499999999999999,
+ "token_overlap": 0.6,
+ "edit_similarity": 0.6,
+ "composite_score": 0.69
+ }
+ },
+ {
+ "original": "Daktari wa kiume alipima mgonjwa",
+ "corrected": "Daktari alipima mgonjwa",
+ "is_biased": true,
+ "category": "occupation",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "wa kiume",
+ "to": "",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 0.75,
+ "rouge_l_score": 0.7499999999999999,
+ "token_overlap": 0.6,
+ "edit_similarity": 0.6,
+ "composite_score": 0.69
+ }
+ },
+ {
+ "original": "Yeye anafanya vizuri kwa mtoto-mlea",
+ "corrected": "Anafanya vizuri kwa mtoto-mlea",
+ "is_biased": true,
+ "category": "pronoun_assumption",
+ "pre_detected": true,
+ "post_detected": false,
+ "bias_removed": true,
+ "text_changed": true,
+ "text_changed_eval": true,
+ "pre_edits": [
+ {
+ "from": "Yeye anafanya",
+ "to": "Anafanya",
+ "severity": "replace"
+ },
+ {
+ "from": "yeye anafanya",
+ "to": "anafanya",
+ "severity": "replace"
+ }
+ ],
+ "post_edits": [],
+ "preservation_metrics": {
+ "bleu_score": 1.0,
+ "rouge_l_score": 0.9090909090909091,
+ "token_overlap": 0.8333333333333334,
+ "edit_similarity": 0.8333333333333334,
+ "composite_score": 0.9060606060606062
+ }
+ },
+ {
+ "original": "Meza ni ya mbao",
+ "corrected": "Meza ni ya mbao",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mkutano unaanza saa tisa",
+ "corrected": "Mkutano unaanza saa tisa",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Tafadhali funga dirisha",
+ "corrected": "Tafadhali funga dirisha",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Daktari alipima mgonjwa kwa uangalifu",
+ "corrected": "Daktari alipima mgonjwa kwa uangalifu",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mwalimu wetu alieleza dhana vizuri",
+ "corrected": "Mwalimu wetu alieleza dhana vizuri",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mhandisi alibuni daraja jipya",
+ "corrected": "Mhandisi alibuni daraja jipya",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Muuguzi alitoa huduma nzuri",
+ "corrected": "Muuguzi alitoa huduma nzuri",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Rubani aliruka ndege kwa usalama",
+ "corrected": "Rubani aliruka ndege kwa usalama",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mwanasheria aliwasilisha hoja madhubuti",
+ "corrected": "Mwanasheria aliwasilisha hoja madhubuti",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Wanasayansi waligundua spishi mpya",
+ "corrected": "Wanasayansi waligundua spishi mpya",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Ripoti inahitajika kesho",
+ "corrected": "Ripoti inahitajika kesho",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Kahawa ina ladha nzuri",
+ "corrected": "Kahawa ina ladha nzuri",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Gari linahitaji mafuta",
+ "corrected": "Gari linahitaji mafuta",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Inanyesha nje",
+ "corrected": "Inanyesha nje",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Kitabu ni cha kuvutia",
+ "corrected": "Kitabu ni cha kuvutia",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Geuka kushoto kwenye kona",
+ "corrected": "Geuka kushoto kwenye kona",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Simu inalia",
+ "corrected": "Simu inalia",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Maji yanachemka kwa nyuzi 100",
+ "corrected": "Maji yanachemka kwa nyuzi 100",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Treni inafika adhuhuri",
+ "corrected": "Treni inafika adhuhuri",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Tafadhali tuma barua pepe",
+ "corrected": "Tafadhali tuma barua pepe",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Kompyuta ni polepole",
+ "corrected": "Kompyuta ni polepole",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mlango umefungwa",
+ "corrected": "Mlango umefungwa",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Wakati unaruka haraka",
+ "corrected": "Wakati unaruka haraka",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Jua linang'aa",
+ "corrected": "Jua linang'aa",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Muziki unasikika vizuri",
+ "corrected": "Muziki unasikika vizuri",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mradi umekamilika",
+ "corrected": "Mradi umekamilika",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Chakula kinanuka vizuri",
+ "corrected": "Chakula kinanuka vizuri",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Barabara ni mbovu",
+ "corrected": "Barabara ni mbovu",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Mimea inahitaji maji",
+ "corrected": "Mimea inahitaji maji",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Anga ni la buluu",
+ "corrected": "Anga ni la buluu",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Nambari hazidanganyi",
+ "corrected": "Nambari hazidanganyi",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ },
+ {
+ "original": "Saa inaonyesha saa kumi na moja",
+ "corrected": "Saa inaonyesha saa kumi na moja",
+ "is_biased": false,
+ "category": "none",
+ "pre_detected": false,
+ "post_detected": false,
+ "bias_removed": false,
+ "text_changed": false,
+ "text_changed_eval": false,
+ "pre_edits": [],
+ "post_edits": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/eval/results/correction_report_en_20251203_151228.txt b/eval/results/correction_report_en_20251203_151228.txt
new file mode 100644
index 0000000000000000000000000000000000000000..78339b844f5742f9105a090ec1a9ccf3099926f4
--- /dev/null
+++ b/eval/results/correction_report_en_20251203_151228.txt
@@ -0,0 +1,47 @@
+
+================================================================================
+ENHANCED CORRECTION EFFECTIVENESS REPORT - EN
+================================================================================
+
+Dataset: 66 samples (34 biased)
+
+PRE-CORRECTION DETECTION:
+ Precision: 1.000
+ Recall: 0.618
+ F1 Score: 0.764
+ Confusion: TP=21, FP=0, FN=13, TN=32
+
+POST-CORRECTION DETECTION:
+ Precision: 0.000
+ Recall: 0.000
+ F1 Score: 0.000
+ Confusion: TP=0, FP=0, FN=34, TN=32
+
+BIAS REMOVAL EFFECTIVENESS:
+ Bias Removal Rate: 100.0%
+ Successfully Neutralized: 21 / 21 detected
+ HarmonicScore (F1 ⊗ Removal): 0.866
+ → Assessment: EXCELLENT (≥0.75)
+
+SEMANTIC PRESERVATION (Token-Level Analysis):
+ Samples Analyzed: 21
+ BLEU Score: 0.616
+ ROUGE-L Score: 0.760
+ Token Overlap: 0.765
+ Edit Similarity: 0.728
+ Composite Score: 0.711
+ → Assessment: GOOD preservation
+
+CORRECTION QUALITY:
+ Successful Corrections: 21
+ High-Quality Corrections: 0
+ Over-Corrections: 0
+ Meaning Preserved (manual): 21 samples
+
+CATEGORY BREAKDOWN:
+Category Pre-F1 Post-F1 Removal% Harmonic Status Detd Cortd
+--------------------------------------------------------------------------------
+occupation 0.927 0.000 100.0% 0.962 ✓ Effective 19 19
+pronoun_assumption 0.250 0.000 100.0% 0.400 ⚠ Review 1 1
+pronoun_generic 0.333 0.000 100.0% 0.500 ⚠ Review 1 1
+
diff --git a/eval/results/correction_report_fr_20251203_151228.txt b/eval/results/correction_report_fr_20251203_151228.txt
new file mode 100644
index 0000000000000000000000000000000000000000..589c2974bec139e244f775072776dbf3f3049847
--- /dev/null
+++ b/eval/results/correction_report_fr_20251203_151228.txt
@@ -0,0 +1,47 @@
+
+================================================================================
+ENHANCED CORRECTION EFFECTIVENESS REPORT - FR
+================================================================================
+
+Dataset: 50 samples (35 biased)
+
+PRE-CORRECTION DETECTION:
+ Precision: 1.000
+ Recall: 0.400
+ F1 Score: 0.571
+ Confusion: TP=14, FP=0, FN=21, TN=15
+
+POST-CORRECTION DETECTION:
+ Precision: 1.000
+ Recall: 0.143
+ F1 Score: 0.250
+ Confusion: TP=5, FP=0, FN=30, TN=15
+
+BIAS REMOVAL EFFECTIVENESS:
+ Bias Removal Rate: 64.3%
+ Successfully Neutralized: 9 / 14 detected
+ HarmonicScore (F1 ⊗ Removal): 0.605
+ → Assessment: GOOD
+
+SEMANTIC PRESERVATION (Token-Level Analysis):
+ Samples Analyzed: 12
+ BLEU Score: 0.595
+ ROUGE-L Score: 0.734
+ Token Overlap: 0.824
+ Edit Similarity: 0.668
+ Composite Score: 0.697
+ → Assessment: Moderate preservation, review needed
+
+CORRECTION QUALITY:
+ Successful Corrections: 9
+ High-Quality Corrections: 1
+ Over-Corrections: 0
+ Meaning Preserved (manual): 12 samples
+
+CATEGORY BREAKDOWN:
+Category Pre-F1 Post-F1 Removal% Harmonic Status Detd Cortd
+--------------------------------------------------------------------------------
+occupation 0.467 0.083 85.7% 0.604 ⚠ Review 7 6
+pronoun_assumption 0.667 0.400 50.0% 0.571 ⚠ Review 4 2
+pronoun_generic 0.857 0.667 33.3% 0.480 ⚠ Review 3 1
+
diff --git a/eval/results/correction_report_ki_20251203_151228.txt b/eval/results/correction_report_ki_20251203_151228.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6faf80eb42b1dbcb2d7e2447f4aae6b502af41d1
--- /dev/null
+++ b/eval/results/correction_report_ki_20251203_151228.txt
@@ -0,0 +1,46 @@
+
+================================================================================
+ENHANCED CORRECTION EFFECTIVENESS REPORT - KI
+================================================================================
+
+Dataset: 33 samples (18 biased)
+
+PRE-CORRECTION DETECTION:
+ Precision: 1.000
+ Recall: 0.500
+ F1 Score: 0.667
+ Confusion: TP=9, FP=0, FN=9, TN=15
+
+POST-CORRECTION DETECTION:
+ Precision: 0.000
+ Recall: 0.000
+ F1 Score: 0.000
+ Confusion: TP=0, FP=0, FN=18, TN=15
+
+BIAS REMOVAL EFFECTIVENESS:
+ Bias Removal Rate: 100.0%
+ Successfully Neutralized: 9 / 9 detected
+ HarmonicScore (F1 ⊗ Removal): 0.800
+ → Assessment: EXCELLENT (≥0.75)
+
+SEMANTIC PRESERVATION (Token-Level Analysis):
+ Samples Analyzed: 9
+ BLEU Score: 0.854
+ ROUGE-L Score: 0.823
+ Token Overlap: 0.783
+ Edit Similarity: 0.783
+ Composite Score: 0.816
+ → Assessment: GOOD preservation
+
+CORRECTION QUALITY:
+ Successful Corrections: 9
+ High-Quality Corrections: 4
+ Over-Corrections: 0
+ Meaning Preserved (manual): 9 samples
+
+CATEGORY BREAKDOWN:
+Category Pre-F1 Post-F1 Removal% Harmonic Status Detd Cortd
+--------------------------------------------------------------------------------
+pronoun_assumption 1.000 0.000 100.0% 1.000 ✓ Effective 9 9
+occupation 0.000 0.000 0.0% 0.000 ⚠ Review 0 0
+
diff --git a/eval/results/correction_report_sw_20251203_151228.txt b/eval/results/correction_report_sw_20251203_151228.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c4d9769d084ec2d5969e7d2f7f12714204cb670d
--- /dev/null
+++ b/eval/results/correction_report_sw_20251203_151228.txt
@@ -0,0 +1,47 @@
+
+================================================================================
+ENHANCED CORRECTION EFFECTIVENESS REPORT - SW
+================================================================================
+
+Dataset: 63 samples (31 biased)
+
+PRE-CORRECTION DETECTION:
+ Precision: 1.000
+ Recall: 0.516
+ F1 Score: 0.681
+ Confusion: TP=16, FP=0, FN=15, TN=32
+
+POST-CORRECTION DETECTION:
+ Precision: 0.000
+ Recall: 0.000
+ F1 Score: 0.000
+ Confusion: TP=0, FP=0, FN=31, TN=32
+
+BIAS REMOVAL EFFECTIVENESS:
+ Bias Removal Rate: 100.0%
+ Successfully Neutralized: 16 / 16 detected
+ HarmonicScore (F1 ⊗ Removal): 0.810
+ → Assessment: EXCELLENT (≥0.75)
+
+SEMANTIC PRESERVATION (Token-Level Analysis):
+ Samples Analyzed: 16
+ BLEU Score: 0.830
+ ROUGE-L Score: 0.809
+ Token Overlap: 0.762
+ Edit Similarity: 0.734
+ Composite Score: 0.791
+ → Assessment: GOOD preservation
+
+CORRECTION QUALITY:
+ Successful Corrections: 16
+ High-Quality Corrections: 7
+ Over-Corrections: 0
+ Meaning Preserved (manual): 16 samples
+
+CATEGORY BREAKDOWN:
+Category Pre-F1 Post-F1 Removal% Harmonic Status Detd Cortd
+--------------------------------------------------------------------------------
+occupation 0.400 0.000 100.0% 0.571 ⚠ Review 5 5
+pronoun_assumption 1.000 0.000 100.0% 1.000 ✓ Effective 7 7
+pronoun_generic 1.000 0.000 100.0% 1.000 ✓ Effective 4 4
+
diff --git a/eval/results/f1_report_20251127_194705.csv b/eval/results/f1_report_20251127_194705.csv
new file mode 100644
index 0000000000000000000000000000000000000000..9e5507e8b73eb1664e9564208c9e0894b961fa5b
--- /dev/null
+++ b/eval/results/f1_report_20251127_194705.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.750,0.857,15,0,5,0
+SW,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+SW,pronoun_generic,0.000,0.000,0.000,0,0,4,0
+FR,OVERALL,1.000,0.457,0.627,16,0,19,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.625,0.769,5,0,3,0
+FR,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+KI,OVERALL,1.000,0.556,0.714,10,0,8,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,1.000,0.111,0.200,1,0,8,0
diff --git a/eval/results/f1_report_20251127_195719.csv b/eval/results/f1_report_20251127_195719.csv
new file mode 100644
index 0000000000000000000000000000000000000000..9e5507e8b73eb1664e9564208c9e0894b961fa5b
--- /dev/null
+++ b/eval/results/f1_report_20251127_195719.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.750,0.857,15,0,5,0
+SW,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+SW,pronoun_generic,0.000,0.000,0.000,0,0,4,0
+FR,OVERALL,1.000,0.457,0.627,16,0,19,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.625,0.769,5,0,3,0
+FR,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+KI,OVERALL,1.000,0.556,0.714,10,0,8,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,1.000,0.111,0.200,1,0,8,0
diff --git a/eval/results/f1_report_20251127_200123.csv b/eval/results/f1_report_20251127_200123.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a0922fc8b1508baacd6340f37ee89450572be08c
--- /dev/null
+++ b/eval/results/f1_report_20251127_200123.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.935,0.967,29,0,2,32
+SW,occupation,1.000,0.900,0.947,18,0,2,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.457,0.627,16,0,19,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.625,0.769,5,0,3,0
+FR,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+KI,OVERALL,1.000,0.556,0.714,10,0,8,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,1.000,0.111,0.200,1,0,8,0
diff --git a/eval/results/f1_report_20251127_202137.csv b/eval/results/f1_report_20251127_202137.csv
new file mode 100644
index 0000000000000000000000000000000000000000..6085360582fa94fd96710d6bef46185e15b22b75
--- /dev/null
+++ b/eval/results/f1_report_20251127_202137.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,1.000,1.000,31,0,0,32
+SW,occupation,1.000,1.000,1.000,20,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.457,0.627,16,0,19,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.625,0.769,5,0,3,0
+FR,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+KI,OVERALL,1.000,0.556,0.714,10,0,8,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,1.000,0.111,0.200,1,0,8,0
diff --git a/eval/results/f1_report_20251127_202452.csv b/eval/results/f1_report_20251127_202452.csv
new file mode 100644
index 0000000000000000000000000000000000000000..6085360582fa94fd96710d6bef46185e15b22b75
--- /dev/null
+++ b/eval/results/f1_report_20251127_202452.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,1.000,1.000,31,0,0,32
+SW,occupation,1.000,1.000,1.000,20,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.457,0.627,16,0,19,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.625,0.769,5,0,3,0
+FR,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+KI,OVERALL,1.000,0.556,0.714,10,0,8,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,1.000,0.111,0.200,1,0,8,0
diff --git a/eval/results/f1_report_20251127_203747.csv b/eval/results/f1_report_20251127_203747.csv
new file mode 100644
index 0000000000000000000000000000000000000000..fe7ae5b2dec2db2ab1cbedee7b869f7b205b9a02
--- /dev/null
+++ b/eval/results/f1_report_20251127_203747.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,0.298,1.000,0.459,345,812,0,61
+SW,morphology,1.000,1.000,1.000,120,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,51,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,154,0,0,0
+SW,occupation,1.000,1.000,1.000,20,0,0,0
+FR,OVERALL,1.000,0.457,0.627,16,0,19,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.625,0.769,5,0,3,0
+FR,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+KI,OVERALL,1.000,0.556,0.714,10,0,8,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,1.000,0.111,0.200,1,0,8,0
diff --git a/eval/results/f1_report_20251127_203855.csv b/eval/results/f1_report_20251127_203855.csv
new file mode 100644
index 0000000000000000000000000000000000000000..8b873d514157ea2a7e5218b1beba5b0d1666dfde
--- /dev/null
+++ b/eval/results/f1_report_20251127_203855.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.957,0.978,330,0,15,873
+SW,morphology,1.000,1.000,1.000,120,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,51,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,154,0,0,0
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251127_203952.csv b/eval/results/f1_report_20251127_203952.csv
new file mode 100644
index 0000000000000000000000000000000000000000..8b873d514157ea2a7e5218b1beba5b0d1666dfde
--- /dev/null
+++ b/eval/results/f1_report_20251127_203952.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.957,0.978,330,0,15,873
+SW,morphology,1.000,1.000,1.000,120,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,51,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,154,0,0,0
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251127_205117.csv b/eval/results/f1_report_20251127_205117.csv
new file mode 100644
index 0000000000000000000000000000000000000000..8b873d514157ea2a7e5218b1beba5b0d1666dfde
--- /dev/null
+++ b/eval/results/f1_report_20251127_205117.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.957,0.978,330,0,15,873
+SW,morphology,1.000,1.000,1.000,120,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,51,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,154,0,0,0
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251127_205133.csv b/eval/results/f1_report_20251127_205133.csv
new file mode 100644
index 0000000000000000000000000000000000000000..8b873d514157ea2a7e5218b1beba5b0d1666dfde
--- /dev/null
+++ b/eval/results/f1_report_20251127_205133.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.957,0.978,330,0,15,873
+SW,morphology,1.000,1.000,1.000,120,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,51,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,154,0,0,0
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251127_223023.csv b/eval/results/f1_report_20251127_223023.csv
new file mode 100644
index 0000000000000000000000000000000000000000..034cc1227332fbeb4e3763487a74f383bf5bf2d0
--- /dev/null
+++ b/eval/results/f1_report_20251127_223023.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.998,0.999,9302,0,15,35295
+SW,morphology,1.000,1.000,1.000,8700,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,439,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,158,0,0,0
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251127_223031.csv b/eval/results/f1_report_20251127_223031.csv
new file mode 100644
index 0000000000000000000000000000000000000000..034cc1227332fbeb4e3763487a74f383bf5bf2d0
--- /dev/null
+++ b/eval/results/f1_report_20251127_223031.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.998,0.999,9302,0,15,35295
+SW,morphology,1.000,1.000,1.000,8700,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,439,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,158,0,0,0
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251203_140921.csv b/eval/results/f1_report_20251203_140921.csv
new file mode 100644
index 0000000000000000000000000000000000000000..034cc1227332fbeb4e3763487a74f383bf5bf2d0
--- /dev/null
+++ b/eval/results/f1_report_20251203_140921.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.998,0.999,9302,0,15,35295
+SW,morphology,1.000,1.000,1.000,8700,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,439,0,0,0
+SW,pronoun_assumption,1.000,1.000,1.000,158,0,0,0
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251203_141342.csv b/eval/results/f1_report_20251203_141342.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ca9798f7d622c9bf7080f912728bf14d9e8adf94
--- /dev/null
+++ b/eval/results/f1_report_20251203_141342.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251203_152419.csv b/eval/results/f1_report_20251203_152419.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ca9798f7d622c9bf7080f912728bf14d9e8adf94
--- /dev/null
+++ b/eval/results/f1_report_20251203_152419.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251203_152526.csv b/eval/results/f1_report_20251203_152526.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ca9798f7d622c9bf7080f912728bf14d9e8adf94
--- /dev/null
+++ b/eval/results/f1_report_20251203_152526.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251204_173336.csv b/eval/results/f1_report_20251204_173336.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ca9798f7d622c9bf7080f912728bf14d9e8adf94
--- /dev/null
+++ b/eval/results/f1_report_20251204_173336.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251211_151304.csv b/eval/results/f1_report_20251211_151304.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ca9798f7d622c9bf7080f912728bf14d9e8adf94
--- /dev/null
+++ b/eval/results/f1_report_20251211_151304.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20251211_151426.csv b/eval/results/f1_report_20251211_151426.csv
new file mode 100644
index 0000000000000000000000000000000000000000..2a55acff7bc142ef537606cb69c280190f1c60af
--- /dev/null
+++ b/eval/results/f1_report_20251211_151426.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.882,0.938,90,0,12,50
+KI,pronoun_assumption,1.000,1.000,1.000,56,0,0,0
+KI,occupation,1.000,0.739,0.850,34,0,12,0
diff --git a/eval/results/f1_report_20251211_153452.csv b/eval/results/f1_report_20251211_153452.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ca9798f7d622c9bf7080f912728bf14d9e8adf94
--- /dev/null
+++ b/eval/results/f1_report_20251211_153452.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.250,0.400,5,0,15,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.500,0.667,9,0,9,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,0.000,0.000,0.000,0,0,9,0
diff --git a/eval/results/f1_report_20260122_113633.csv b/eval/results/f1_report_20260122_113633.csv
new file mode 100644
index 0000000000000000000000000000000000000000..9e5507e8b73eb1664e9564208c9e0894b961fa5b
--- /dev/null
+++ b/eval/results/f1_report_20260122_113633.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.618,0.764,21,0,13,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.516,0.681,16,0,15,32
+SW,occupation,1.000,0.750,0.857,15,0,5,0
+SW,pronoun_assumption,1.000,0.143,0.250,1,0,6,0
+SW,pronoun_generic,0.000,0.000,0.000,0,0,4,0
+FR,OVERALL,1.000,0.457,0.627,16,0,19,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.625,0.769,5,0,3,0
+FR,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+KI,OVERALL,1.000,0.556,0.714,10,0,8,15
+KI,pronoun_assumption,1.000,1.000,1.000,9,0,0,0
+KI,occupation,1.000,0.111,0.200,1,0,8,0
diff --git a/eval/results/f1_report_20260122_122155.csv b/eval/results/f1_report_20260122_122155.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_122155.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260122_122921.csv b/eval/results/f1_report_20260122_122921.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_122921.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260122_130256.csv b/eval/results/f1_report_20260122_130256.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_130256.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260122_131022.csv b/eval/results/f1_report_20260122_131022.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_131022.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260122_132237.csv b/eval/results/f1_report_20260122_132237.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_132237.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260122_151601.csv b/eval/results/f1_report_20260122_151601.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_151601.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260122_151626.csv b/eval/results/f1_report_20260122_151626.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_151626.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260122_151648.csv b/eval/results/f1_report_20260122_151648.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_151648.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260122_151912.csv b/eval/results/f1_report_20260122_151912.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a32b558eeaeb9e893fd831c5e70072c71a9b26ea
--- /dev/null
+++ b/eval/results/f1_report_20260122_151912.csv
@@ -0,0 +1,16 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,1.000,0.473,0.643,231,0,257,254
+KI,occupation,1.000,0.083,0.153,14,0,155,0
+KI,pronoun_assumption,1.000,0.680,0.810,217,0,102,0
diff --git a/eval/results/f1_report_20260204_175444.csv b/eval/results/f1_report_20260204_175444.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a88d9c96f378ee7a61a784dbf1eb05ef6aab54d4
--- /dev/null
+++ b/eval/results/f1_report_20260204_175444.csv
@@ -0,0 +1,17 @@
+Language,Category,Precision,Recall,F1_Score,TP,FP,FN,TN
+EN,OVERALL,1.000,0.647,0.786,22,0,12,32
+EN,occupation,1.000,0.864,0.927,19,0,3,0
+EN,pronoun_assumption,1.000,0.286,0.444,2,0,5,0
+EN,pronoun_generic,1.000,0.200,0.333,1,0,4,0
+SW,OVERALL,1.000,0.548,0.708,17,0,14,32
+SW,occupation,1.000,0.300,0.462,6,0,14,0
+SW,pronoun_assumption,1.000,1.000,1.000,7,0,0,0
+SW,pronoun_generic,1.000,1.000,1.000,4,0,0,0
+FR,OVERALL,1.000,0.400,0.571,14,0,21,15
+FR,occupation,1.000,0.304,0.467,7,0,16,0
+FR,pronoun_assumption,1.000,0.500,0.667,4,0,4,0
+FR,pronoun_generic,1.000,0.750,0.857,3,0,1,0
+KI,OVERALL,0.814,0.155,0.260,253,58,1383,3506
+KI,occupation,1.000,0.041,0.078,22,0,520,0
+KI,pronoun_assumption,1.000,0.326,0.491,223,0,462,0
+KI,stereotype,1.000,0.020,0.038,8,0,401,0
diff --git "a/eval/results/reporting/Bias Correction Evaluation \342\200\223 Kikuyu (JuaKazi)_15Jan26.pdf" "b/eval/results/reporting/Bias Correction Evaluation \342\200\223 Kikuyu (JuaKazi)_15Jan26.pdf"
new file mode 100644
index 0000000000000000000000000000000000000000..f8b03279882ef983299fa4af6c033a4a92b9ee48
--- /dev/null
+++ "b/eval/results/reporting/Bias Correction Evaluation \342\200\223 Kikuyu (JuaKazi)_15Jan26.pdf"
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32e3722ba85ea749d3a04235cafaced0cf5fa23ed4daa78507f495b357366e3e
+size 173530
diff --git "a/eval/results/reporting/Bias Correction Evaluation \342\200\223 Kikuyu (JuaKazi)_19Dec2025.pdf" "b/eval/results/reporting/Bias Correction Evaluation \342\200\223 Kikuyu (JuaKazi)_19Dec2025.pdf"
new file mode 100644
index 0000000000000000000000000000000000000000..8decc2bbef161422efd1ac10f89246574acc67c3
--- /dev/null
+++ "b/eval/results/reporting/Bias Correction Evaluation \342\200\223 Kikuyu (JuaKazi)_19Dec2025.pdf"
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cfb319dbab69fad407a2c968caa72c7dde87e75bd6e44f1a5d53df2569c88bd
+size 107456
diff --git "a/eval/results/reporting/Bias Correction Evaluation \342\200\223 Swahili (JuaKazi)_12Jan2026.pdf" "b/eval/results/reporting/Bias Correction Evaluation \342\200\223 Swahili (JuaKazi)_12Jan2026.pdf"
new file mode 100644
index 0000000000000000000000000000000000000000..6f7e59859beae0fa41fd5bdcea0803ba0047706c
--- /dev/null
+++ "b/eval/results/reporting/Bias Correction Evaluation \342\200\223 Swahili (JuaKazi)_12Jan2026.pdf"
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a74436d25debd7adcee9aaddde2e0600c5cdd7e1823e6d61eb8828dbdacd4c4
+size 105854
diff --git "a/eval/results/reporting/Kikuyu Bias Correction \342\200\223 Comparative Evaluation Report (JuaKazi)_15Jan2026.pdf" "b/eval/results/reporting/Kikuyu Bias Correction \342\200\223 Comparative Evaluation Report (JuaKazi)_15Jan2026.pdf"
new file mode 100644
index 0000000000000000000000000000000000000000..34fcf25d61988152a701aac66801d43cbaf46ed2
Binary files /dev/null and "b/eval/results/reporting/Kikuyu Bias Correction \342\200\223 Comparative Evaluation Report (JuaKazi)_15Jan2026.pdf" differ
diff --git a/eval/results/reporting/juakazi_kikuyu_eval_comparison_report_15Jan2026.html b/eval/results/reporting/juakazi_kikuyu_eval_comparison_report_15Jan2026.html
new file mode 100644
index 0000000000000000000000000000000000000000..26422f5cc0af56423a27d055120a96944d15abb3
--- /dev/null
+++ b/eval/results/reporting/juakazi_kikuyu_eval_comparison_report_15Jan2026.html
@@ -0,0 +1,327 @@
+
+
+
+
+
+Kikuyu Bias Correction – Comparative Evaluation Report (JuaKazi)
+
+
+
+
+
+
+
+
Comparative Bias Correction Evaluation – Kikuyu (KI)
+
+
+Project: JuaKazi Gender Bias Detection & Correction System
+Languages: Kikuyu (KI)
+Comparison Scope: Small-scale vs Large-scale Evaluation
+
+
+
1. Evaluation Scope Overview
+
+
+This report compares two evaluation runs of the JuaKazi bias detection and
+correction system for Kikuyu. The first evaluation was conducted on a small,
+manually curated dataset, while the second represents a large-scale evaluation
+intended to stress-test detection coverage, correction reliability, and semantic
+preservation.
+
+
+
+
+
+| Metric |
+Run A (Small-Scale) |
+Run B (Large-Scale) |
+
+
+
+
+| Total Samples |
+33 |
+742 |
+
+
+| Biased Samples |
+18 |
+488 |
+
+
+| Detected Biases |
+9 |
+231 |
+
+
+
+
+
2. Pre-Correction Detection Performance Comparison
+
+
+Detection performance reflects the system’s ability to identify biased text prior
+to any intervention. Both runs were evaluated under a precision-first operating
+point.
+
+
+
+
+
+| Metric |
+Run A |
+Run B |
+
+
+
+
+| Precision |
+1.000 |
+1.000 |
+
+
+| Recall |
+0.500 |
+0.473 |
+
+
+| F1 Score |
+0.667 |
+0.643 |
+
+
+
+
+
+
+
+Detection quality remains stable as scale increases, with only a marginal drop in
+recall and F1. This indicates good generalization of the detector, though coverage
+limitations persist.
+
+
+
3. Bias Removal Effectiveness Comparison
+
+
+Bias removal effectiveness measures whether detected bias is successfully
+neutralized.
+
+
+
+
+
+| Metric |
+Run A |
+Run B |
+
+
+
+
+| Detected Biases |
+9 |
+231 |
+
+
+| Successfully Neutralized |
+9 |
+231 |
+
+
+| Removal Rate |
+100.0% |
+100.0% |
+
+
+| HarmonicScore |
+0.800 |
+0.782 |
+
+
+
+
+
+Perfect removal is maintained at scale. The slight reduction in HarmonicScore in
+Run B is attributable solely to reduced detection F1, not correction quality.
+
+
+
4. Semantic Preservation Comparison
+
+
+Semantic preservation metrics assess how well meaning is retained after bias
+neutralization.
+
+
+
+
+
+| Metric |
+Run A |
+Run B |
+
+
+
+
+| BLEU |
+0.854 |
+0.703 |
+
+
+| ROUGE-L |
+0.823 |
+0.824 |
+
+
+| Token Overlap |
+0.783 |
+0.824 |
+
+
+| Edit Similarity |
+0.783 |
+0.824 |
+
+
+| Composite |
+0.816 |
+0.787 |
+
+
+
+
+
+At scale, BLEU decreases due to increased lexical diversity, while ROUGE-L and
+token overlap remain stable or improve. This suggests that corrections remain
+structurally faithful, even when surface-level wording varies.
+
+
+
5. Areas of Success
+
+
+The following strengths are consistently observed across both evaluations:
+
+
+
+- Zero false positives across all datasets, ensuring safe deployment.
+- Perfect correction reliability once bias is detected.
+- High semantic preservation, even at large scale.
+- Strong pronoun-bias handling, with high pre-F1 and full neutralization.
+- Scalable correction logic unaffected by dataset size.
+
+
+
6. Areas Requiring Improvement
+
+
+Despite strong correction performance, several limitations remain:
+
+
+
+- Detection recall remains below 0.50, limiting end-to-end impact.
+- Occupation bias detection shows consistently low pre-F1.
+- HarmonicScore ceiling is constrained by detection rather than correction.
+- BLEU sensitivity at scale suggests a need for embedding-based semantic metrics.
+
+
+
+Future work should prioritize expanding detection coverage—particularly for
+occupation-related bias—while preserving the system’s conservative precision
+profile.
+
+
+
+© JuaKazi Team — Comparative Bias Correction Analysis
+
+
+
+
+
Bias Detection and Correction Evaluation for Kikuyu
+
+
+Project: JuaKazi Gender Bias Detection & Correction System
+Language: Kikuyu (KI)
+Evaluation Date: 15 January 2026
+
+
+
1. Dataset Description
+
+
+The evaluation was conducted on a large-scale Kikuyu dataset consisting of
+742 sentences. A total of 488 samples were annotated as containing gender bias,
+covering both pronoun assumption and occupation-related bias
+categories.
+
+
+
+
+
+| Total Samples |
+Biased Samples |
+Detected (Pre-Correction) |
+
+
+
+
+| 742 |
+488 |
+231 |
+
+
+
+
+
2. Pre-Correction Bias Detection Performance
+
+
+This section reports bias detection performance on the original Kikuyu text prior
+to any correction. The detector operates under a precision-first configuration to
+avoid unintended semantic distortion.
+
+
+
+
+Precision
+1.000
+
+
+Recall
+0.473
+
+
+F1 Score
+0.643
+
+
+
+
+Perfect precision indicates zero false positives across 742 samples. Recall of
+0.473 shows that 231 of 488 biased instances were detected. This behavior confirms
+the system’s conservative bias-identification strategy.
+
+
+
+
+
3. Bias Removal Effectiveness
+
+
+
+
+| Detected Biases |
+Neutralized |
+Removal Rate |
+
+
+
+
+| 231 |
+231 |
+100.0% |
+
+
+
+
+
+All detected biased instances were successfully neutralized. End-to-end system
+performance is summarized using the HarmonicScore, computed as the
+harmonic mean of pre-correction F1 and bias removal rate.
+
+
+
+
+HarmonicScore
+0.782
+
+
+
+
+A HarmonicScore of 0.782 reflects strong overall effectiveness, balancing moderate
+detection coverage with perfect correction reliability at scale.
+
+
+
4. Semantic Preservation Analysis
+
+
+Semantic preservation was evaluated for all 231 successfully corrected sentences.
+Token-level similarity metrics quantify the degree to which original meaning was
+maintained after bias neutralization.
+
+
+
+
+
+| Metric |
+Score |
+Interpretation |
+
+
+
+
+| BLEU |
+0.703 |
+Moderate local lexical similarity |
+
+
+| ROUGE-L |
+0.824 |
+Strong sequence-level alignment |
+
+
+| Token Overlap |
+0.824 |
+High content retention |
+
+
+| Edit Similarity |
+0.824 |
+Edits were minimal and localized |
+
+
+| Composite |
+0.787 |
+Meaning preserved across corrections |
+
+
+
+
+
+
+
5. Category-Level Performance
+
+
+
+
+| Category |
+Pre-F1 |
+Removal % |
+Harmonic |
+Detected |
+Corrected |
+
+
+
+
+| Occupation |
+0.153 |
+100.0% |
+0.265 |
+14 |
+14 |
+
+
+| Pronoun Assumption |
+0.810 |
+100.0% |
+0.895 |
+217 |
+217 |
+
+
+
+
+
6. Technical Interpretation
+
+
+At scale, the JuaKazi system demonstrates consistent and reliable bias correction
+for Kikuyu text. Pronoun-based bias dominates both detection and correction volume,
+with high pre-correction F1 and complete neutralization.
+
+
+
+Occupation-related bias remains under-detected, reflected in low pre-F1 despite
+perfect correction once identified. This indicates that future improvements
+should prioritize expanding detection coverage rather than correction logic.
+
+
+
+
+
+
+
Bias Detection and Correction Evaluation for Kikuyu
+
+
+Project: JuaKazi Gender Bias Detection & Correction System
+Language: Kikuyu (KI)
+Evaluation Date: 19 December 2025
+
+
+
1. Dataset Description
+
+
+The evaluation was conducted on a manually curated Kikuyu dataset consisting of
+33 sentences. Of these, 18 instances were annotated as containing gender bias.
+Bias categories include pronoun assumption and
+occupation-related bias.
+
+
+
+
+
+| Total Samples |
+Biased Samples |
+Detected (Pre-Correction) |
+
+
+
+
+| 33 |
+18 |
+9 |
+
+
+
+
+
2. Pre-Correction Bias Detection Performance
+
+
+This section evaluates the performance of the bias detector on original,
+uncorrected text. These metrics quantify the system’s ability to identify
+biased content prior to intervention.
+
+
+
+
+Precision
+1.000
+
+
+Recall
+0.500
+
+
+F1 Score
+0.667
+
+
+
+
+Perfect precision indicates the absence of false positives, while a recall of
+0.50 shows that half of the biased instances were identified. This reflects a
+deliberate precision-first strategy appropriate for automated correction
+pipelines where false corrections are costly.
+
+
+
+
+
+
3. Bias Removal Effectiveness
+
+
+
+
+| Detected Biases |
+Neutralized |
+Removal Rate |
+
+
+
+
+| 9 |
+9 |
+100.0% |
+
+
+
+
+
+All detected biased instances were successfully neutralized. The
+HarmonicScore, defined as the harmonic mean of pre-correction F1
+and bias removal rate, summarizes end-to-end system effectiveness.
+
+
+
+
+HarmonicScore
+0.800
+
+
+
+
+A HarmonicScore of 0.800 indicates strong combined performance, balancing
+moderate detection coverage with perfect correction reliability.
+
+
+
4. Semantic Preservation Analysis
+
+
+Semantic preservation evaluates whether bias mitigation alters the original
+meaning. Token-level similarity metrics were computed between original and
+corrected sentences for all successfully neutralized samples.
+
+
+
+
+
+| Metric |
+Score |
+Interpretation |
+
+
+
+
+| BLEU |
+0.854 |
+High local lexical similarity |
+
+
+| ROUGE-L |
+0.823 |
+Strong sequence-level alignment |
+
+
+| Token Overlap |
+0.783 |
+Most content words preserved |
+
+
+| Edit Similarity |
+0.783 |
+Limited, targeted edits |
+
+
+| Composite |
+0.816 |
+Overall semantic integrity retained |
+
+
+
+
+
+
+
+
5. Category-Level Performance
+
+
+
+
+| Category |
+Pre-F1 |
+Removal % |
+Harmonic |
+Detected |
+Corrected |
+
+
+
+
+| Pronoun Assumption |
+1.000 |
+100.0% |
+1.000 |
+9 |
+9 |
+
+
+| Occupation |
+0.000 |
+0.0% |
+0.000 |
+0 |
+0 |
+
+
+
+
+
6. Technical Interpretation
+
+
+Results indicate that the JuaKazi correction pipeline achieves complete bias
+neutralization for detected Kikuyu pronoun-based bias, while preserving
+semantic content. The correction strategy relies on minimal lexical substitution,
+which is particularly suitable for low-resource and morphologically rich
+languages.
+
+
+
+Detection recall remains the primary limitation, especially for occupation-based
+bias categories, suggesting future work should focus on expanding detection
+coverage without compromising precision.
+
+
+
+
+
+
+
Bias Detection and Correction Evaluation for Swahili
+
+
+Project: JuaKazi Gender Bias Detection & Correction System
+Language: Swahili (SW)
+Evaluation Date: 12 January 2026
+
+
+
1. Dataset Description
+
+
+The evaluation was conducted on a manually curated Swahili dataset consisting of
+63 sentences. Of these, 31 instances were annotated as containing gender bias.
+Bias categories include pronoun assumption, pronoun generic,
+and occupation-related bias.
+
+
+
+
+
+| Total Samples |
+Biased Samples |
+Detected (Pre-Correction) |
+
+
+
+
+| 63 |
+31 |
+16 |
+
+
+
+
+
2. Pre-Correction Bias Detection Performance
+
+
+This section evaluates the performance of the bias detector on original,
+uncorrected text. These metrics quantify the system’s ability to identify
+biased content prior to intervention.
+
+
+
+
+Precision
+1.000
+
+
+Recall
+0.516
+
+
+F1 Score
+0.681
+
+
+
+
+Perfect precision indicates no false positives, while a recall of 0.516 shows
+that slightly over half of the biased instances were identified. This reflects
+a conservative detection approach suitable for automated correction pipelines.
+
+
+
+
+
3. Bias Removal Effectiveness
+
+
+
+
+| Detected Biases |
+Neutralized |
+Removal Rate |
+
+
+
+
+| 16 |
+16 |
+100.0% |
+
+
+
+
+
+
+HarmonicScore
+0.810
+
+
+
+
+All detected biased instances were successfully neutralized. The HarmonicScore,
+the harmonic mean of pre-correction F1 and bias removal rate, confirms strong
+end-to-end correction effectiveness.
+
+
+
4. Semantic Preservation Analysis
+
+
+Semantic preservation evaluates whether bias mitigation alters the original
+meaning. Token-level similarity metrics were computed between original and
+corrected sentences.
+
+
+
+
+
+| Metric |
+Score |
+Interpretation |
+
+
+
+
+| BLEU |
+0.830 |
+High local lexical similarity |
+
+
+| ROUGE-L |
+0.809 |
+Strong sequence-level alignment |
+
+
+| Token Overlap |
+0.762 |
+Most content words preserved |
+
+
+| Edit Similarity |
+0.734 |
+Limited, targeted edits |
+
+
+| Composite |
+0.791 |
+Overall semantic integrity retained |
+
+
+
+
+
+
+
5. Category-Level Performance
+
+
+
+
+| Category |
+Pre-F1 |
+Removal % |
+Harmonic |
+Detected |
+Corrected |
+
+
+
+
+| Occupation |
+0.400 |
+100.0% |
+0.571 |
+5 |
+5 |
+
+
+| Pronoun Assumption |
+1.000 |
+100.0% |
+1.000 |
+7 |
+7 |
+
+
+| Pronoun Generic |
+1.000 |
+100.0% |
+1.000 |
+4 |
+4 |
+
+
+
+
+
6. Technical Interpretation
+
+
+Results indicate that the JuaKazi correction pipeline achieves complete bias
+neutralization for all detected Swahili bias categories, while preserving
+semantic integrity. Token-level analysis confirms that edits are minimal and targeted,
+ensuring content fidelity.
+
+
+
+Detection recall is lower for certain categories, such as occupation-related bias,
+indicating the need for improved detection coverage without sacrificing precision.
+
+
+
+
+