Spaces:
Sleeping
Sleeping
| import pandas as pd | |
| import re | |
| import os | |
| import json | |
| from transformers import MarianMTModel, MarianTokenizer | |
| # --- 1. SETUP & DATA LOADING --- | |
| tr_model_name = "Helsinki-NLP/opus-mt-en-ml" | |
| tr_tokenizer = MarianTokenizer.from_pretrained(tr_model_name) | |
| tr_model = MarianMTModel.from_pretrained(tr_model_name) | |
| def load_medical_glossary(csv_path='medical_glossary.csv'): | |
| # Get the directory where nlp_utils.py is located | |
| base_path = os.path.dirname(__file__) | |
| full_path = os.path.join(base_path, csv_path) | |
| if os.path.exists(full_path): | |
| return pd.read_csv(full_path, encoding='utf-8') | |
| else: | |
| # Debug print to see where it's looking in logs | |
| print(f"GLOSSARY ERROR: Could not find file at {full_path}") | |
| return None | |
| GLOSSARY_DF = load_medical_glossary() | |
| # --- 2. THE CLEANING LOGIC --- | |
| def post_process_translation(mal_text, glossary_df): | |
| if mal_text is None: return "" | |
| # 1. Tone Fixes (Stay the same) | |
| tone_fixes = {r'അവൾക്ക്': 'ഇവർക്ക്', r'അവൾ': 'ഇവർ', r'അവളുടെ': 'ഇവരുടെ', r'അവൻ': 'ഇദ്ദേഹം'} | |
| for pattern, replacement in tone_fixes.items(): | |
| mal_text = mal_text.replace(pattern, replacement) | |
| # 2. Advanced Glossary Mapping | |
| if glossary_df is not None: | |
| # Sort by length (longest first) to prevent partial matching | |
| glossary_df['match_len'] = glossary_df['English_Term'].astype(str).str.len() | |
| sorted_df = glossary_df.sort_values('match_len', ascending=False) | |
| for _, row in sorted_df.iterrows(): | |
| eng_phrase = str(row['English_Term']).strip() # This is the variable name | |
| mal_phrase = str(row['Malayalam_Translation']).strip() | |
| pattern = re.compile(re.escape(eng_phrase), re.IGNORECASE) | |
| mal_text = pattern.sub(mal_phrase, mal_text) | |
| # 3. Clean 'Management Management' bug | |
| mal_text = re.sub(r'(\b\w+\b)( \1)+', r'\1', mal_text) | |
| return mal_text | |
| # --- 3. THE MAIN WORKFLOW --- | |
| def translate_to_malayalam(text): | |
| sentences = re.split(r'(?<=[.!?;]) +', text) | |
| output = [] | |
| for sent in sentences: | |
| sent = sent.strip() | |
| if not sent: continue | |
| # STEP A: Generate Raw Translation | |
| inputs = tr_tokenizer(sent, return_tensors="pt", truncation=True) | |
| translated = tr_model.generate(**inputs, max_new_tokens=150) | |
| raw_malayalam = tr_tokenizer.decode(translated[0], skip_special_tokens=True) | |
| # STEP B: Run the Cleaning Function (The "Workflow") | |
| cleaned_malayalam = post_process_translation(raw_malayalam, GLOSSARY_DF) | |
| output.append(cleaned_malayalam) | |
| return " ".join(output) | |
| def get_highlighted_summary(text, highlight_list): | |
| """ | |
| Handles multi-word highlighting for the Gradio UI. | |
| """ | |
| if not text: return "" | |
| # Sort: Longest phrases first to avoid partial matching | |
| sorted_terms = sorted(highlight_list, key=len, reverse=True) | |
| highlighted = text | |
| for term in sorted_terms: | |
| pattern = re.compile(r'\b(' + re.escape(term) + r')\b', re.IGNORECASE) | |
| highlighted = pattern.sub(r'**\1**', highlighted) | |
| return highlighted |