Upload 2 files

app.py

@@ -0,0 +1,355 @@
+import os
+import time
+import re
+import requests
+from flask import Flask, request, render_template, send_file, jsonify
+import fitz  # PyMuPDF
+from werkzeug.utils import secure_filename
+
+app = Flask(__name__)
+
+# Configuration
+UPLOAD_FOLDER = 'uploads'
+OUTPUT_FOLDER = 'outputs'
+os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+os.makedirs(OUTPUT_FOLDER, exist_ok=True)
+app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
+app.config['OUTPUT_FOLDER'] = OUTPUT_FOLDER
+
+# Hugging Face API Configuration
+HF_API_KEY = 
+HF_API_URL = "https://api-inference.huggingface.co/models/facebook/nllb-200-3.3B"
+HEADERS = {
+    "Authorization": f"Bearer {HF_API_KEY}",
+    "X-Wait-For-Model": "180",  # Wait up to 3 minutes
+    "X-Use-Cache": "0"
+}
+
+# Update token IDs for 1.3B model:
+LANGUAGES = {
+    "Hindi": {"token_id": 256047, "code": "hin_Deva", "iso": "hi"},
+    "Tamil": {"token_id": 256157, "code": "tam_Taml", "iso": "ta"},
+    "Telugu": {"token_id": 256082, "code": "tel_Telu", "iso": "te"}
+}
+MAX_LENGTH_DEFAULT = 512
+
+DIGIT_MAP = {
+    "Hindi": "०१२३४५६७८९",
+    "Tamil": "௦௧௨௩௪௫௬௭௮௯",
+    "Telugu": "౦౧౨౩౪౫౬౭౮౯"
+}
+LATIN_DIGITS = "0123456789"
+
+# Utility functions
+def parse_user_entities(user_input):
+    return sorted({e.strip() for e in user_input.split(',') if e.strip()}, key=len, reverse=True)
+
+def parse_user_languages(user_input):
+    selected = [lang.strip().capitalize() for lang in user_input.split(',')]
+    valid = [lang for lang in selected if lang in LANGUAGES]
+    return valid or list(LANGUAGES.keys())
+
+def replace_with_placeholders(text, entities):
+    placeholder_map = {}
+    modified_text = text
+    patterns = [
+        (re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'), "emails"),
+        (re.compile(r'https?://\S+|www\.\S+'), "URLs"),
+        (re.compile(r'@\w+'), "usernames"),
+        (re.compile(r'[\$£€%#@&*]\s*\d+(?:\.\d+)?|\d+(?:\.\d+)?[\$£€%#@&*]'), "symbols_with_numbers"),
+        (re.compile(r'[\$£€%#@&*]'), "symbols")
+    ]
+    
+    for pattern, _ in patterns:
+        for match in pattern.findall(modified_text):
+            placeholder = f"__PRESERVE{len(placeholder_map):03d}__"
+            placeholder_map[placeholder] = match
+            modified_text = modified_text.replace(match, placeholder)
+
+    for entity in entities:
+        pattern = re.compile(re.escape(entity), re.IGNORECASE)
+        modified_text = pattern.sub(lambda m: f"__PRESERVE{len(placeholder_map):03d}__", modified_text)
+        placeholder_map[f"__PRESERVE{len(placeholder_map):03d}__"] = entity
+
+    return modified_text, placeholder_map
+
+def convert_numbers_to_script(text, target_lang):
+    digit_map = DIGIT_MAP[target_lang]
+    return re.sub(r'\d+', lambda m: ''.join(digit_map[int(d)] for d in m.group()), text)
+
+def translate_batch(texts, target_lang, fast_mode=False):
+    if not texts:
+        return []
+    
+    translated_texts = []
+    batch_size = 2
+    max_retries = 3
+    lang_data = LANGUAGES[target_lang]
+
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        max_length = max(MAX_LENGTH_DEFAULT, max(len(t.split()) for t in batch) * 2)
+        payload = {
+            "inputs": batch,
+            "parameters": {
+                "max_length": max_length,
+                "forced_bos_token_id": lang_data["token_id"],
+                "src_lang": "eng_Latn",
+                "tgt_lang": lang_data["code"]
+            }
+        }
+
+        for attempt in range(max_retries):
+            try:
+                response = requests.post(HF_API_URL, headers=HEADERS, json=payload)
+                response.raise_for_status()
+                result = response.json()
+
+                if isinstance(result, list):
+                    translated = [r.get("translation_text", "") for r in result]
+                    translated_texts.extend([re.sub(r'^\s*…|\.+$', '', t.strip()) for t in translated])
+                    break
+                else:
+                    if "estimated_time" in result:
+                        wait_time = result["estimated_time"] + 5
+                        print(f"Model loading, retrying in {wait_time}s...")
+                        time.sleep(wait_time)
+                        continue
+                    raise ValueError(f"Unexpected response format: {result}")
+
+            except requests.exceptions.HTTPError as e:
+                if e.response.status_code == 503 and attempt < max_retries - 1:
+                    wait_time = 10 * (attempt + 1)
+                    print(f"Server overloaded, retrying in {wait_time}s...")
+                    time.sleep(wait_time)
+                    continue
+                raise
+
+            time.sleep(1)
+
+    return translated_texts
+
+# PDF processing functions (extract_pdf_components, split_block_into_subblocks, 
+# ... [Keep previous configuration and constants] ...
+
+def extract_pdf_components(pdf_path):
+    print(f"\n📄 Extracting components from {pdf_path}...")
+    doc = fitz.open(pdf_path)
+    components = []
+    for page_num, page in enumerate(doc):
+        blocks = page.get_text("dict")["blocks"]
+        text_blocks = []
+        for b in blocks:
+            if b["type"] == 0:  # Text block
+                lines = []
+                for line in b["lines"]:
+                    if line["spans"]:
+                        text = join_spans(line["spans"])
+                        if text.strip():
+                            lines.append({
+                                "text": text,
+                                "y_pos": line["spans"][0]["origin"][1],
+                                "x_pos": line["spans"][0]["origin"][0],
+                                "font_size": line["spans"][0]["size"],
+                                "line_bbox": line["bbox"]
+                            })
+                if lines:
+                    text_blocks.append({"bbox": b["bbox"], "lines": lines})
+        components.append({"page_num": page_num, "text_blocks": text_blocks, "size": (page.rect.width, page.rect.height)})
+    doc.close()
+    return components
+
+def join_spans(spans):
+    if not spans:
+        return ""
+    spans = sorted(spans, key=lambda s: s["bbox"][0])
+    text_parts = [spans[0]["text"].strip()]
+    for i in range(1, len(spans)):
+        span1, span2 = spans[i - 1], spans[i]
+        d = span2["bbox"][0] - span1["bbox"][2]
+        text2 = span2["text"].strip()
+        if not text2:
+            continue
+        if d < 0.5 * min((span1["bbox"][2] - span1["bbox"][0])/len(span1["text"]), 
+                        (span2["bbox"][2] - span2["bbox"][0])/len(text2)):
+            text_parts.append(text2)
+        else:
+            text_parts.append(" " + text2)
+    return "".join(text_parts)
+
+def split_block_into_subblocks(block):
+    lines = block["lines"]
+    subblocks = []
+    current_subblock = {"text": "", "lines": [], "is_short": False}
+    max_words = 50
+    
+    for i, line in enumerate(lines):
+        text = line["text"].strip()
+        if not text:
+            continue
+            
+        is_short = len(text.split()) <= 3
+        font_size = line["font_size"]
+        current_words = current_subblock["text"].split()
+        
+        if len(current_words) + len(text.split()) > max_words or font_size > 20:
+            subblocks.append(current_subblock)
+            current_subblock = {"text": "", "lines": [], "is_short": False}
+            
+        if i > 0:
+            gap = lines[i]["y_pos"] - lines[i-1]["y_pos"] - lines[i-1]["font_size"]
+            x_shift = abs(line["x_pos"] - lines[i-1]["x_pos"])
+            
+            if gap > font_size * 0.5 or x_shift > 10:
+                subblocks.append(current_subblock)
+                current_subblock = {"text": "", "lines": [], "is_short": False}
+                
+        current_subblock["text"] += " " + text if current_subblock["text"] else text
+        current_subblock["lines"].append(line)
+        current_subblock["is_short"] = is_short
+        
+    if current_subblock["text"]:
+        subblocks.append(current_subblock)
+    return subblocks
+
+def translate_chunk(chunk, entities, target_lang, fast_mode=False):
+    all_subblocks = []
+    for page in chunk:
+        for block in page["text_blocks"]:
+            subblocks = split_block_into_subblocks(block)
+            block["subblocks"] = subblocks
+            all_subblocks.extend(subblocks)
+            
+    texts = []
+    placeholder_maps = []
+    for subblock in all_subblocks:
+        if not subblock["text"].strip():
+            continue
+        modified_text, ph_map = replace_with_placeholders(subblock["text"], entities)
+        texts.append(modified_text)
+        placeholder_maps.append(ph_map)
+        
+    translated_texts = translate_batch(texts, target_lang, fast_mode)
+    
+    for i, subblock in enumerate(all_subblocks):
+        if i >= len(translated_texts):
+            continue
+        translated = translated_texts[i]
+        ph_map = placeholder_maps[i]
+        
+        for placeholder, original in ph_map.items():
+            translated = translated.replace(placeholder, original)
+        translated = convert_numbers_to_script(translated, target_lang)
+        subblock["translated_text"] = translated
+
+def rebuild_pdf(components, target_lang, output_path, original_pdf_path):
+    doc = fitz.open(original_pdf_path)
+    lang_iso = LANGUAGES[target_lang]["iso"]
+    
+    for page_data in components:
+        page = doc[page_data["page_num"]]
+        links = page.get_links()
+        
+        for block in page_data["text_blocks"]:
+            page.add_redact_annot(block["bbox"])
+            page.apply_redactions()
+            
+            for subblock in block.get("subblocks", []):
+                if not subblock.get("translated_text", "").strip():
+                    continue
+                
+                translated_lines = redistribute_translated_text(
+                    subblock["translated_text"], 
+                    subblock["lines"]
+                )
+                
+                for line, translated in zip(subblock["lines"], translated_lines):
+                    rect = fitz.Rect(line["line_bbox"])
+                    html = f'<p lang="{lang_iso}">{translated}</p>'
+                    page.insert_htmlbox(rect, html)
+        
+        for link in links:
+            page.insert_link(link)
+    
+    doc.save(output_path, garbage=4, deflate=True)
+    doc.close()
+
+def redistribute_translated_text(translated_text, original_lines):
+    words = translated_text.split()
+    lines = []
+    current_line = []
+    current_width = 0
+    
+    for line in original_lines:
+        if not words:
+            break
+            
+        max_width = line["line_bbox"][2] - line["line_bbox"][0]
+        font_size = line["font_size"]
+        font = fitz.Font("helv")
+        
+        while words:
+            word_width = font.text_length(words[0] + " ", fontsize=font_size)
+            if current_width + word_width <= max_width:
+                current_line.append(words.pop(0))
+                current_width += word_width
+            else:
+                break
+                
+        lines.append(" ".join(current_line))
+        current_line = []
+        current_width = 0
+    
+    if words:
+        lines[-1] += " " + " ".join(words)
+    
+    return lines + [""]*(len(original_lines)-len(lines))
+
+# ... [Keep the Flask routes from previous code] ...
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/translate', methods=['POST'])
+def translate_pdf():
+    if 'pdf_file' not in request.files:
+        return jsonify({'error': 'No PDF file uploaded'}), 400
+
+    pdf_file = request.files['pdf_file']
+    if pdf_file.filename == '':
+        return jsonify({'error': 'No file selected'}), 400
+
+    filename = secure_filename(pdf_file.filename)
+    pdf_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
+    pdf_file.save(pdf_path)
+
+    entities = parse_user_entities(request.form.get('entities', ''))
+    languages = parse_user_languages(request.form.get('languages', 'Hindi,Tamil,Telugu'))
+
+    try:
+        components = extract_pdf_components(pdf_path)
+        output_files = []
+
+        for lang in languages:
+            start_time = time.time()
+            translate_chunk(components, entities, lang, fast_mode=len(components) <= 5)
+            output_path = os.path.join(app.config['OUTPUT_FOLDER'], f"translated_{lang}_{filename}")
+            rebuild_pdf(components, lang, output_path, pdf_path)
+            output_files.append(output_path)
+            print(f"{lang} translation completed in {time.time()-start_time:.2f}s")
+
+        return jsonify({
+            'message': f"Translation completed for {', '.join(languages)}",
+            'files': [os.path.basename(f) for f in output_files]
+        })
+
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+@app.route('/download/<filename>')
+def download_file(filename):
+    file_path = os.path.join(app.config['OUTPUT_FOLDER'], filename)
+    return send_file(file_path, as_attachment=True) if os.path.exists(file_path) else (jsonify({'error': 'File not found'}), 404)
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5000)), debug=False)

requirements.txt

@@ -0,0 +1,5 @@
+Flask
+pymupdf
+requests
+werkzeug
+googletrans==4.0.0-rc1