Spaces:
Runtime error
Runtime error
| """ | |
| Evaluate a single pre-computed output against gold labels. | |
| Usage: | |
| python eval_single_output.py --input evaluation_dataset.json --output hybrid_output.json --outlet "Newfoundland Independent" | |
| Where hybrid_output.json contains the raw_output from the hybrid run. | |
| """ | |
| import json | |
| import argparse | |
| import logging | |
| from metrics import MetricsCalculator | |
| logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') | |
| logger = logging.getLogger(__name__) | |
| def evaluate_output(gold_item: dict, generated: dict) -> dict: | |
| """Compute all metrics for a single outlet given gold and generated data.""" | |
| metrics_calc = MetricsCalculator() | |
| # Construct text blocks (same logic as run_benchmark.py evaluate_single) | |
| gen_text = " ".join(filter(None, [ | |
| generated.get('bias_category_description'), | |
| generated.get('overall_summary'), | |
| generated.get('analysis'), | |
| generated.get('history'), | |
| generated.get('ownership'), | |
| ])).strip() | |
| gold_text_full = "\n\n".join(filter(None, [ | |
| gold_item.get('bias_category_description', ''), | |
| gold_item.get('overall_summary', ''), | |
| gold_item.get('history', ''), | |
| gold_item.get('analysis', ''), | |
| gold_item.get('ownership', ''), | |
| ])) | |
| gold_summary = " ".join(filter(None, [ | |
| gold_item.get('bias_category_description', ''), | |
| gold_item.get('overall_summary', ''), | |
| gold_item.get('analysis', ''), | |
| gold_item.get('history', ''), | |
| gold_item.get('ownership', ''), | |
| ])) | |
| # A. Text Overlap | |
| logger.info("Computing ROUGE-L and METEOR...") | |
| text_m = metrics_calc.calculate_text_metrics([gold_summary], [gen_text]) | |
| # B. FactScore Precision: Generated -> Gold | |
| logger.info("Computing FACTScore Precision...") | |
| fs_precision = metrics_calc.check_fact_recall(gold_text_full, gen_text) | |
| # C. Fact Recall: Gold -> Generated | |
| logger.info("Computing FACTScore Recall...") | |
| fs_recall = metrics_calc.check_gold_fact_recall(gold_text_full, gen_text) | |
| # D. Fact Check Detection | |
| logger.info("Computing Fact Check Detection...") | |
| gold_fc = gold_item.get('failed_fact_checks', []) | |
| gen_fc_list = generated.get('failed_fact_checks', []) | |
| fc_m = MetricsCalculator.evaluate_fact_checks(gold_fc, gen_text, gen_fc_list) | |
| result = { | |
| "name": gold_item['name'], | |
| "source_url": gold_item.get('source_url', ''), | |
| "gold_bias": gold_item.get('bias_score', 0), | |
| "pred_bias": generated.get('bias_score'), | |
| "gold_factuality": gold_item.get('factual_score', 0), | |
| "pred_factuality": generated.get('factual_score'), | |
| "rougeL": text_m['rougeL'], | |
| "meteor": text_m['meteor'], | |
| "factscore_precision": fs_precision.get('factscore', 0.0), | |
| "error_rate": fs_precision.get('error_rate', 0.0), | |
| "factscore_recall": fs_recall.get('gold_recall', 0.0), | |
| "fact_check_hit": fc_m, | |
| } | |
| return result | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Evaluate a single pre-computed output") | |
| parser.add_argument("--input", type=str, default="evaluation_dataset.json", | |
| help="Path to evaluation dataset JSON") | |
| parser.add_argument("--output", type=str, required=True, | |
| help="Path to JSON file with the generated output (raw_output)") | |
| parser.add_argument("--outlet", type=str, required=True, | |
| help="Name of the outlet to evaluate") | |
| args = parser.parse_args() | |
| # Load gold dataset | |
| with open(args.input) as f: | |
| dataset = json.load(f) | |
| # Find gold item | |
| gold_item = None | |
| for item in dataset: | |
| if item['name'].lower() == args.outlet.lower(): | |
| gold_item = item | |
| break | |
| if not gold_item: | |
| logger.error(f"Outlet '{args.outlet}' not found in {args.input}") | |
| return | |
| # Load generated output | |
| with open(args.output) as f: | |
| generated = json.load(f) | |
| # If the file has a 'raw_output' key (from benchmark results), use that | |
| if 'raw_output' in generated: | |
| generated = generated['raw_output'] | |
| # If wrapped in 'output' key (from hybrid runner results) | |
| if 'output' in generated: | |
| generated = generated['output'] | |
| logger.info(f"Evaluating: {gold_item['name']}") | |
| logger.info(f" Gold bias: {gold_item.get('bias_score')}, Pred bias: {generated.get('bias_score')}") | |
| logger.info(f" Gold factuality: {gold_item.get('factual_score')}, Pred factuality: {generated.get('factual_score')}") | |
| result = evaluate_output(gold_item, generated) | |
| # Print results | |
| print("\n" + "=" * 60) | |
| print(f"EVALUATION RESULTS: {result['name']}") | |
| print("=" * 60) | |
| print(f" Bias: gold={result['gold_bias']}, pred={result['pred_bias']}") | |
| print(f" Factuality: gold={result['gold_factuality']}, pred={result['pred_factuality']}") | |
| print(f" ROUGE-L: {result['rougeL']:.4f}") | |
| print(f" METEOR: {result['meteor']:.4f}") | |
| print(f" FACTScore Prec: {result['factscore_precision']:.4f}") | |
| print(f" Error Rate: {result['error_rate']:.4f}") | |
| print(f" FACTScore Recall: {result['factscore_recall']:.4f}") | |
| print(f" Fact Check Hit: {result['fact_check_hit']}") | |
| print("=" * 60) | |
| # Save full result | |
| out_path = args.output.replace('.json', '_evaluated.json') | |
| with open(out_path, 'w') as f: | |
| json.dump(result, f, indent=2, default=str) | |
| print(f"\nFull result saved to: {out_path}") | |
| if __name__ == "__main__": | |
| main() | |