File size: 3,643 Bytes
3998131 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
import fitz # pymupdf
import re
from typing import List
from transformers import pipeline
import torch
PDF_FILE_PATH = "module_b/file_2.pdf"
def extract_nepali_sentences_from_pdf(pdf_path: str) -> List[str]:
"""
Extracts clean Nepali sentences from a searchable PDF using PyMuPDF.
"""
print(f"Opening PDF: {pdf_path}")
doc = fitz.open(pdf_path)
full_text = ""
for page in doc:
text = page.get_text("text")
full_text += text + "\n"
doc.close()
if not full_text.strip():
print("Warning: No text found. PDF might be scanned (image-based). Use OCR version instead.")
return []
# Clean whitespace
text = full_text.replace('\n', ' ')
text = re.sub(r'\s+', ' ', text).strip()
# Split sentences intelligently
sentences = re.split(r'(?<=[।.!?])\s+(?=[अ-हँ-ॿअ-ह])|(?<=[।.!?])(?=$)', text)
if len(sentences) <= 1: # fallback
sentences = re.split(r'(?<=[।.!?])\s+', text)
# Final cleaning
cleaned = [s.strip(' ।.!?').strip() for s in sentences if len(s.strip()) > 5]
print(f"Successfully extracted {len(cleaned)} clean sentences.\n")
return cleaned
print("Loading your model from Hugging Face...")
model_name = "sangy1212/distilbert-base-nepali-fine-tuned"
classifier = pipeline(
"text-classification",
model=model_name,
tokenizer=model_name,
device=0 if torch.cuda.is_available() else -1,
batch_size=16
)
print("Model loaded and ready!\n")
id_to_label = {
"LABEL_0": "neutral",
"LABEL_1": "gender",
"LABEL_2": "religional",
"LABEL_3": "caste",
"LABEL_4": "religion",
"LABEL_5": "appearence",
"LABEL_6": "socialstatus",
"LABEL_7": "amiguity",
"LABEL_8": "political",
"LABEL_9": "Age",
"LABEL_10": "Disablity"
}
def predict_bias_on_sentences(sentences: List[str], confidence_threshold: float = 0.7):
"""
Runs batch prediction and prints results with nice formatting.
"""
if not sentences:
print("No sentences to analyze.")
return
print(f"Running bias detection on {len(sentences)} sentences...\n")
# Batch inference
results = classifier(sentences)
print("="*100)
print("BIAS DETECTION RESULTS")
print("="*100)
biased_count = 0
for sent, res in zip(sentences, results):
label_id = res['label']
category = id_to_label.get(label_id, "unknown")
confidence = res['score']
if category != "neutral" and confidence >= confidence_threshold:
mark = " BIAS DETECTED"
biased_count += 1
else:
mark = "✓ neutral / low confidence"
print(f"{mark}")
print(f" Category : {category.upper()}")
print(f" Confidence : {confidence:.3f}")
print(f" Sentence : {sent}")
print("-" * 80)
print(f"\nSummary: {biased_count}/{len(sentences)} sentences contain detectable bias (confidence ≥ {confidence_threshold})")
if __name__ == "__main__":
pdf_file_path = PDF_FILE_PATH
import os
if os.path.exists(pdf_file_path):
print(f"Using PDF file at: {pdf_file_path}\n")
else:
print(f"PDF file not found at: {pdf_file_path}. Please check the path.")
exit(1)
# Step 1: Extract sentences
sentences = extract_nepali_sentences_from_pdf(pdf_file_path)
# Step 2: Run batch prediction
if sentences:
predict_bias_on_sentences(sentences, confidence_threshold=0.7)
print("\nDone! Your bias detection is complete.") |