File size: 4,520 Bytes
3e9c053 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | import csv
import json
import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from tqdm import tqdm
def get_domain_weight(domain):
credible = ["reuters.com", "apnews.com", "bbc.com", "politifact.com", "snopes.com", "factcheck.org"]
unreliable = ["freedomtruthblog.net", "theonion.com", "randomnews.org", "infowars.com"]
if any(d in domain for d in credible):
return 1.5
elif any(d in domain for d in unreliable):
return 0.2
return 1.0
def run_genuine_ablation():
print("Loading test_claims_dataset.csv for Empirical Grid Search Ablation...")
y_true = []
prob_fake_list = []
evidence_data_list = []
with open("test_claims_dataset.csv", mode='r', encoding='utf-8') as f:
reader = csv.DictReader(f)
for row in reader:
y_true.append(1 if row["true_label"] == "Fake" else 0)
prob_fake_list.append(float(row["linguistic_prob_fake"]))
evidence_data_list.append(json.loads(row["evidence"]))
print(f"Successfully loaded {len(y_true)} claims into memory.")
print("Sweeping through 101 model configurations (Linguistic Weight 0.00 to 1.00)...\n")
results = []
weights = np.linspace(0.0, 1.0, 101)
for ling_weight in tqdm(weights, desc="Evaluating Pipeline Configurations"):
evid_weight = 1.0 - ling_weight
y_pred = []
for i in range(len(y_true)):
risk_score = prob_fake_list[i] * 100
evidence_list = evidence_data_list[i]
if len(evidence_list) > 0:
total_stance_score = 0
total_weight = 0
has_strong_debunk = False
for ev in evidence_list:
weight = get_domain_weight(ev["domain"])
# confidence that it is PRO (supports claim)
if ev["stance"] == "PRO":
prob_pro = ev["confidence"]
else:
prob_pro = 1.0 - ev["confidence"]
if weight >= 1.4 and ev["has_debunk_keywords"] and ev["confidence"] >= 0.75:
has_strong_debunk = True
total_stance_score += (prob_pro * weight)
total_weight += weight
if has_strong_debunk:
final_risk = max(risk_score, 90.0)
else:
avg_pro = total_stance_score / total_weight
evidence_risk = (1.0 - avg_pro) * 100
final_risk = (risk_score * ling_weight) + (evidence_risk * evid_weight)
else:
final_risk = risk_score
final_risk = min(max(final_risk, 0), 100)
y_pred.append(1 if final_risk > 50 else 0)
acc = accuracy_score(y_true, y_pred)
prec = precision_score(y_true, y_pred, zero_division=0)
rec = recall_score(y_true, y_pred, zero_division=0)
f1 = f1_score(y_true, y_pred, zero_division=0)
results.append({
"Linguistic_Weight": round(ling_weight, 2),
"Evidence_Weight": round(evid_weight, 2),
"Accuracy": acc,
"Precision": prec,
"Recall": rec,
"F1_Score": f1
})
df_results = pd.DataFrame(results)
output_csv = "genuine_ablation_results.csv"
df_results.to_csv(output_csv, index=False)
print("\n\nGrid Search Ablation Study Complete!")
print(f"Metrics saved to {output_csv}")
best_idx = df_results['F1_Score'].idxmax()
best_config = df_results.iloc[best_idx]
print("\n==============================================")
print("OPTIMAL EMPIRICAL CONFIGURATION DISCOVERED")
print("==============================================")
print(f"Linguistic Weight: {best_config['Linguistic_Weight']:.2f} ({int(best_config['Linguistic_Weight']*100)}%)")
print(f"Evidence Weight: {best_config['Evidence_Weight']:.2f} ({int(best_config['Evidence_Weight']*100)}%)")
print("-" * 46)
print(f"Peak F1-Score: {best_config['F1_Score']:.4f}")
print(f"Peak Accuracy: {best_config['Accuracy']:.4f}")
print("==============================================")
if __name__ == "__main__":
run_genuine_ablation()
|